diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 06430dc31..a700b3b9e 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -7,8 +7,10 @@ name: Claude Auto Review # enforcement lives in claude-review-guard.yml, whose verdict arm fails while the newest verdict # is changes-requested, so a substantive finding becomes a red check instead of a comment that # auto-merge outruns (the #3470-#3473 train shipped six findings in one night that way; one was -# real). It no-ops cleanly until the CLAUDE_CODE_OAUTH_TOKEN repo secret is set, and on fork PRs -# (which do not receive secrets), so neither case shows a failed check. +# real). Since #3650 the job also fails ITSELF when the run submitted no verdict, with the cause in +# its log and the transcript attached -- a green here means a verdict exists, and the guard is the +# second line. It no-ops cleanly until the CLAUDE_CODE_OAUTH_TOKEN repo secret is set, and on fork +# PRs (which do not receive secrets), so neither case shows a failed check. on: pull_request: types: [opened, synchronize, reopened] @@ -37,7 +39,19 @@ jobs: with: fetch-depth: 1 + # #3650: the verdict check at the bottom counts the bot's formal reviews submitted AFTER this + # instant, so a verdict left by an earlier run on the same PR cannot vouch for this one. Read + # once, here, before the action installs anything -- GitHub stamps submitted_at in the same + # ISO-8601 UTC shape (2026-09-18T22:00:00Z), so the comparison below is a plain string one. + - name: Open the verdict window + id: window + if: ${{ env.CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: echo "start=$(date -u +%FT%TZ)" >> "$GITHUB_OUTPUT" + + # The step NAME is read by claude-review-guard.yml (REVIEW_STEP) to tell a clean no-op from a + # run that said nothing; renaming it blinds the guard. The id is for the steps below. - name: Claude review + id: review if: ${{ env.CLAUDE_CODE_OAUTH_TOKEN != '' }} uses: anthropics/claude-code-action@v1 with: @@ -78,5 +92,139 @@ jobs: does not enable, and no gate needs it. The guard's rule is newest-verdict-wins: on a re-review after new commits, review the NEW diff and submit a fresh verdict, and a clean fresh verdict clears an earlier changes-requested by itself. + # #3650: the prompt above says the branch is checked out and asks for a correctness, + # parity and security review -- an invitation to read code -- while the allowlist used to + # permit only the four gh verbs and the inline-comment tool. Every Read, Grep, Glob and + # git call the reviewer reached for was a permission denial, and the swallowed runs' own + # result blocks put the ratio on record: 50 turns / 18 denials, 39 / 21, 18 / 17, each + # ending subtype=success with NOTHING posted. After enough denials the session ends + # without ever reaching the verdict protocol, so a paid run leaves no trace (three PRs, + # about thirteen runs, one night). Read-only tools are enough: the reviewer needs to open + # the files the diff touches, find a symbol's other callers and read a parity twin, and + # nothing about reviewing needs a write. Nothing here can edit, commit, push or post + # outside the gh verbs already listed; the git verbs are the read-only three. claude_args: | - --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr review:*)" + --allowedTools "mcp__github_inline_comment__create_inline_comment,Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr review:*),Read,Grep,Glob,Bash(git diff:*),Bash(git log:*),Bash(git show:*)" + + # #3650: a review that posted nothing used to finish GREEN. The prompt mandates one formal + # review per run -- changes-requested on a substantive finding, a "LGTM" comment review + # otherwise -- so on this repo a run that submitted no verdict has broken its contract every + # time; a clean review is never silent, and a zero here is never legitimate. This step turns + # that into the job's own colour, with the diagnosis in the log, so the guard (#2229/#3492, + # still the REQUIRED check, still enforcing newest-verdict-wins and the drift arm) becomes the + # second line rather than the first. Counted: reviews by the bot, submitted inside this run's + # window, with a NON-EMPTY body. The body test is load-bearing: the inline-comment tool files + # each comment inside a review of its own with an empty body (#3647 carried four bot reviews, + # two of them bodiless carriers), and both verdict shapes must carry one (gh and the API both + # refuse a bodiless comment/changes-requested review) -- so "posted inline notes, never a + # verdict" is the #3470 shape and fails here, as it should. + # Two runs end with zero verdicts and both are defects on this repo, told apart by whether + # Claude ran at all: the action exits SUCCESS in seconds without running when this file + # differs from the default branch's copy (cause 1 in the guard's header), and then it sets no + # execution_file and writes no transcript. Expected on a PR that edits this file; a repo-wide + # outage otherwise -- the guard grades which, because it can read the PR's file list from a + # workflow that is free to change. The step's own outcome is left to speak for itself when it + # is not success (a failed step already reds the job; a cancelled one is a superseded push). + # A gh lookup failure is NOT a verdict on the review (#2309): it warns and stands down rather + # than forcing a paid re-run of the whole job to clear a transient API error -- the guard's + # own tally, a separate and free-to-rerun workflow, stays the enforcement. + - name: Verify the review posted a verdict + if: ${{ always() && env.CLAUDE_CODE_OAUTH_TOKEN != '' }} + env: + GH_TOKEN: ${{ github.token }} + R: ${{ github.repository }} + PR: ${{ github.event.pull_request.number }} + SINCE: ${{ steps.window.outputs.start }} + REVIEW_OUTCOME: ${{ steps.review.outcome }} + # Set by the action only after Claude actually ran; empty when it refused at validation. + EXECUTION_FILE: ${{ steps.review.outputs.execution_file }} + # Author of every artifact the review leaves behind (the action's bot_name default). + REVIEW_BOT: claude[bot] + run: | + set -euo pipefail + summary() { echo "$*" >> "$GITHUB_STEP_SUMMARY"; } + + if [ "$REVIEW_OUTCOME" != "success" ]; then + echo "::notice title=Verdict check skipped::The review step ended '$REVIEW_OUTCOME'; its own"\ + "outcome carries the story, so this step has nothing to add." + exit 0 + fi + + transcript="${EXECUTION_FILE:-$RUNNER_TEMP/claude-execution-output.json}" + + errfile=$(mktemp 2>/dev/null || echo /dev/null) + if ! matched=$(gh api --paginate "repos/$R/pulls/$PR/reviews?per_page=100" \ + --jq ".[] | select(.user.login == env.REVIEW_BOT + and .submitted_at != null + and .submitted_at >= env.SINCE + and ((.body // \"\") | length) > 0) + | \"\\(.state)\\t\\(.submitted_at)\\t\\(.html_url)\"" 2>"$errfile"); then + err=$(cat "$errfile" 2>/dev/null || true) + [ "$errfile" != /dev/null ] && rm -f "$errfile" || true + echo "::warning title=Verdict check could not read the PR's reviews::gh api"\ + "repos/$R/pulls/$PR/reviews failed: ${err:-no stderr}. A lookup failure is not a"\ + "review verdict (#2309), so the review is UNCONFIRMED here; the guard's own tally"\ + "decides, or read the PR by eye." + summary "- Verdict check: lookup failed; review UNCONFIRMED (#2309)." + exit 0 + fi + [ "$errfile" != /dev/null ] && rm -f "$errfile" || true + + count=$(printf '%s' "$matched" | grep -c . || true) + echo "verdict reviews by $REVIEW_BOT on PR #$PR since $SINCE: $count" + if [ -n "$matched" ]; then printf '%s\n' "$matched"; fi + + if [ "$count" -gt 0 ]; then + summary "### Claude review posted $count verdict review(s) since $SINCE" + exit 0 + fi + + summary '### Claude review posted NO verdict' + + if [ -z "$EXECUTION_FILE" ] && [ ! -s "$transcript" ]; then + echo "::error title=Claude never ran::claude-code-action exited success without running"\ + "Claude -- no execution file, no transcript. That is what it does when this branch's"\ + ".github/workflows/claude-review.yml differs from the default branch's copy (#2229,"\ + "cause 1). Expected on a PR that edits this file; on any other PR it means the file"\ + "has drifted and EVERY PR in the repo is going unreviewed -- see the guard's verdict"\ + "on this PR. Either way, do not read this PR as reviewed." + summary '- Claude never ran (workflow validation refused). Not reviewed.' + exit 1 + fi + + echo "::error title=Review ran and posted no verdict::The review ran and posted no"\ + "verdict: $REVIEW_BOT submitted no formal review on PR #$PR since $SINCE, and the"\ + "prompt promises one every run even when clean. Real money was spent and the output"\ + "vanished (#3650; the #2229 failure shape). Do NOT read this PR as reviewed. The"\ + "transcript is attached to this run as the claude-review-transcript artifact; its"\ + "result block follows." + summary '- The review ran and submitted no formal review. Not reviewed (#3650).' + if [ -s "$transcript" ]; then + echo "--- result block of $transcript ---" + jq -c '.[-1] | {type, subtype, is_error, num_turns, permission_denials_count, + total_cost_usd, duration_ms, + denied: [.permission_denials[]? | .tool_name]}' "$transcript" \ + 2>/dev/null || true + echo "--- tail of $transcript ---" + tail -c 4000 "$transcript" || true + echo + else + echo "(no transcript at $transcript)" + fi + exit 1 + + # #3650: the per-turn transcript is what proves WHY a run said nothing, and it used to die + # with the runner -- the denial-ratio evidence above had to be inferred from result blocks in + # the step log. Seven days is long enough to diagnose the next swallowed run in one click and + # short enough that nothing accumulates. The reviewer's tools are read-only over a public + # tree and gh reads of public PR data, so the transcript holds nothing that is not already + # public; keep the allowlist that way, because this artifact is readable by anyone who can + # read the repo. Missing file (the action refused to run, or was skipped) is not an error. + - name: Retain the review transcript + if: ${{ always() && env.CLAUDE_CODE_OAUTH_TOKEN != '' }} + uses: actions/upload-artifact@v6 + with: + name: claude-review-transcript + path: ${{ steps.review.outputs.execution_file || format('{0}/claude-execution-output.json', runner.temp) }} + if-no-files-found: ignore + retention-days: 7 diff --git a/CHANGELOG.md b/CHANGELOG.md index ab726a132..707d8cb9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,32 @@ cut it is archived and compacted like every other version: Releases before 3.0.0 are not archived: those entries carry no prose to move. +## [Unreleased] + +### Fixed + +- **Each generated store TLS root certificate carries a unique per-generation name** ([#3557]) - Windows caches every root a viewer's TLS stack ever sees into the user's intermediate-CA store, one per certificate rotation; with all rotations sharing one name, that cache eventually breaks certificate chain building outright on long-running viewer machines. Distinct names per rotation mean the pile can never form. +- **The by-CPU tools now actually rank by CPU** ([#3523]) - get_top_queries_by_cpu and get_top_procedures_by_cpu ordered by summed elapsed time in both SKUs, so on a wait-bound server the real CPU consumers could be missing from the page entirely - and attributed_cpu_ratio read as "hidden CPU" when it actually meant "wrong sort key". Every ranking site now orders by worker time, including the over-fetch cut that could drop a CPU-heavy query before the final sort ever saw it. The viewer's Duration grids keep their elapsed ranking, which is what they promise. +- **analyze_server no longer answers "all metrics are within normal ranges" when the analysis window collected nothing** ([#3524]) - The analysis gate passes on lifetime history, so a server whose collection died still reached the all-clear path with an empty window. Both SKUs' analysis services now flag the zero-facts window and analyze_server returns the "unavailable" envelope pointing at get_collection_health; the genuine all-clear (facts collected, zero findings) is unchanged. +- **The Performance Calendar, daily summary, and fleet sweep band deadlocks as a measured per-hour rate, not any-deadlock-is-Critical** ([#3525]) - The shared daily classifier routed the Deadlocks signal through the Overview card's store-backed rate tiers (#3368, V120) with each surface's real window as the denominator, so one deadlock no longer paints a calendar day red, sweep verdicts stop scaling with the cadence knob, sub-hour spans fall to Warning instead of a multiplied rate, and the day tooltip/reasons report the rate beside the count. The still-forming day clamps its window to the elapsed portion, so an active storm bands on its true in-progress rate instead of diluting against hours that have not happened yet. +- **Perfmon rates are honest per-second values in analysis** ([#3527]) - The PERFMON_*_SEC facts, the batch-request anomaly window, and both SKUs' batch-request baselines read the per-collection-interval delta as if per-second (60-300x overstatement); every read now divides by the measured sample_interval_seconds (Darling's baseline derives it from collection-time gaps, since the continuous aggregate stores no interval), and interval-0 rows - where no delta was knowable - are skipped instead of read as rates. +- **Floor the SQL count thresholds, give Store Disk Pressure a GB floor, and count measured metrics in the fleet Healthy label** ([#3528]) - The SQL Server deadlock and blocking count thresholds now floor at 1 on read like their PostgreSQL twins, so a store row hand-edited to 0 can't fire on a quiet server; Store Disk Pressure gains a self_alerts.disk_free_warn_gb floor (default 50, 0 disables) so a large store volume at a low percent stops paging CRITICAL with hundreds of GB of runway; and fleet cards carry measured_metric_count/metric_count so a Healthy label built off one measured metric of six says so ("1 of 6 measured" on the web fleet page). +- **get_memory_trend stops reporting granted memory as a hardcoded zero** ([#3529]) - The MCP payload shipped a literal total_granted_mb = 0.0 on both SKUs, steering agents away from memory grants during grant-pressure investigations. The field became an explicit null with the envelope naming get_memory_grants as the grants series' source, and both tool descriptions stopped promising granted memory. +- **Lite's Query Store time slicer reads physical reads from its own column** ([#3530]) - The reader mapped both read fields to the logical-reads ordinal and never read the SELECT's total_physical_reads column, so the physical aggregate was computed and dropped. Pinned with a test whose fixture rows carry distinct values per I/O column. +- **LCK_M_IS advice carries the same RCSI caveats as its LCK_M_S twin** ([#3531]) - The intent-shared-lock advice handed out the READ_COMMITTED_SNAPSHOT ALTER with no caveats. Both lock twins now name the brief exclusive lock the ALTER takes, the tempdb version-store cost, and the test-on-a-copy warning for NOLOCK-dependent code. +- **Collector schedules refuse cadences that would fabricate quiet** ([#3532]) - Delta-family collectors (wait/latch/spinlock/query/procedure/file I/O/memory-grant/perfmon stats, and the PostgreSQL wait/statement collectors) now cap at 30 minutes in both Lite's and Darling's schedule editors, with the store-side resolver ignoring out-of-policy rows: past the shared 60-minute delta gap policy every cycle re-baselined and recorded zeros forever, so the charts flatlined green precisely because collection stopped measuring. +- **get_pg_plans finds the plan you asked for, not just the plans in the top page** ([#3533]) - the queryid filter ran client-side over a fetched top-duration page: the reader had no query_id predicate, so the tool pulled the top limit x 10 shapes by total time and filtered them in C#, which made any plan ranked below that page unfindable at every window size - and the filtered-empty branch then told the caller capture was working, the plan was never captured, and the statement was "not the query to look at", while the plan sat in the store. The predicate now runs in the store's SQL over every capture in the window, the over-fetch is gone, and the miss text says what was actually searched and what a miss can mean, with get_pg_top_queries and get_pg_plan_capture_readiness cross-references. The web dashboard's read dispatch already passed query_id through, so it gains the server-side search with no wiring change. +- **get_pg_autovacuum_health classifies severity from the same axis it ranks by** ([#3534]) - The reader ranks tables by GREATEST(dead ratio, insert ratio) but severity only read the dead side, so an append-only worst_table ten times past its insert-vacuum threshold reported "ok"; severity now comes from the worse of the two ratios, the insert-side ratio is published as insert_threshold_ratio, a one-sample window reports growth as unknown instead of "flat" (with first_seen_at showing the window), and the page-scoped summary counts carry the sibling limit_reached flag. +- **get_pg_replication_slots headlines the worst-classified slot and never spells unknown WAL growth as stable** ([#3535]) - worst_slot was picked by retained size, so an active 45 GB keeping-pace slot ("ok") outranked an inactive growing orphan ("critical_orphan_filling_disk"); slots now rank by severity with size only breaking ties, growth is null when either endpoint is the collector's -1 sentinel or the window holds one sample, new unknown-growth severities say so explicitly, and raw retained_wal_bytes nulls the sentinel like its _gb sibling. +- **get_pg_io_stats no longer renders track_io_timing=off as an impossibly fast disk** ([#3536]) - track_io_timing is OFF by default in PostgreSQL, so on a stock server the read and write time counters are never populated - and this read divided the zeros out to 0.000 ms latencies while its trend sibling had shipped the honest contract all along. The single-window read now mirrors that contract exactly: the setting is read from the collected pg_server_config bounded by the window's end (inferred from the window's data when never collected, with io_timing_source saying which), io_timing_tracked and timing_note are published, and every time-derived figure is null when untracked rather than zero, while the operation counts, hit ratios and byte figures stand untouched. The new busiest_basis field states which key decided the "busiest" ranking. +- **The PostgreSQL xmin-horizon alert catches rotating holders and stops firing on single observations** ([#3537]) - The persistence gate gains a horizon arm that fires when the horizon sits past the age threshold in a majority of the window's real captures (counted from the collector's own collection_log runs) regardless of who holds it, naming the rotating-holder pattern under a stable dedup subject; a minimum-observations floor stops the first holder after quiet hours from reading 1-of-1 as chronic. +- **QueryStore slicer's physical-reads sort plots the physical series** ([#3547]) - Sorting the Query Store grid by physical reads relabeled the slicer but kept plotting logical reads, and the selected-row overlay had no physical series to draw at all. Bars and overlay both plot the real physical-reads aggregate now. +- **get_memory_trend joins the grants series so total_granted_mb carries real data** ([#3548]) - Completing #3529's honest null: both SKUs join the memory-grant series the viewers already chart into the trend payload, matching each memory point to the nearest grants snapshot within 30 seconds. A snapshot measuring nothing granted is a genuine 0.0, an uncovered point stays null, and the granted_note appears only when there is a gap to explain. +- **Viewer Recommendations tabs stop saying "All clear" when the analysis window collected nothing** ([#3551]) - Both viewers only branched on the insufficient-data message, so a server whose collection died still rendered the all-clear on a zero-finding read. Lite's Generate now consumes the analysis service's window-empty state directly; the Darling service persists it into the analysis-state marker (false-with-a-message, no schema change) so the viewer's read and Generate now doors both render a "nothing was measured - check Collection Health" notice. The genuine all-clear (facts measured, zero findings) is unchanged. +- **Viewer pass for the Store Disk Pressure GB floor and fleet measured-metric qualifier** ([#3563]) - The Settings window can now edit the Store Disk Pressure warning's GB floor (V126, 0 disables) next to its percent sibling, and the WPF fleet card's band label carries the web fleet page's "N of M measured" qualifier when a band folded over unmeasured metrics, so an Unknown-heavy server no longer reads as an unqualified green. +- **Deprecated Dashboard analysis reads perfmon counters as true per-second rates** ([#3561]) - The Dashboard's analysis facts, batch-request anomaly window, and batch-request baseline read the per-collection-interval perfmon delta as if it were per-second, overstating by 60-300x at common cadences; all three now divide by the row's measured sample interval and skip rows with no knowable delta, matching the Lite/Darling fix in #3560. +- **Sorting the Procedures or Query Store grid by physical reads plots the physical series, in both apps** ([#3556]) - In Lite and the Darling viewer, the Procedures grid's physical-reads sort plotted the logical aggregate under a physical label, and the Query Store slicer labeled execution-weighted slice totals "Avg". The Darling viewer's Query Store grid also still carried the original #3547 physical/logical swap, and none of its grids re-projected a selected row's overlay when the sort metric changed, so the bars and the overlay could show different metrics. Both apps now plot the series the sorted column names, label slice totals as totals, and keep the overlay on the bars' metric - pinned by distinct-per-column reader tests and source-text pins over the sorting handlers in both apps. + ## [3.8.0] - 2026-09-17 Full entries: [docs/changelog/3.8.md](docs/changelog/3.8.md) @@ -1227,6 +1253,27 @@ Full entries: [docs/changelog/3.0.md](docs/changelog/3.0.md) - **Failed SQL Agent job alert** ([#749]) - **Installer: optional custom data/log file locations** ([#768]) +[#3523]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3523 +[#3524]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3524 +[#3525]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3525 +[#3527]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3527 +[#3528]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3528 +[#3529]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3529 +[#3530]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3530 +[#3531]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3531 +[#3532]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3532 +[#3533]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3533 +[#3534]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3534 +[#3535]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3535 +[#3536]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3536 +[#3537]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3537 +[#3547]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3547 +[#3548]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3548 +[#3551]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3551 +[#3556]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3556 +[#3561]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3561 +[#3563]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3563 +[#3557]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3557 [#3514]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3514 [#3477]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3477 [#3495]: https://github.com/erikdarlingdata/PerformanceMonitor/issues/3495 diff --git a/Darling/Darling.Tests/AlertEngineTests.cs b/Darling/Darling.Tests/AlertEngineTests.cs index ac94fffaf..d3bdd3b30 100644 --- a/Darling/Darling.Tests/AlertEngineTests.cs +++ b/Darling/Darling.Tests/AlertEngineTests.cs @@ -69,6 +69,7 @@ test switches on exactly the check it pins (a disabled check must not even fetch public int DiskCriticalFreePercent { get; set; } = 3; public int DiskCriticalFreeGb { get; set; } = 2; public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int SelfDiskFreeWarnGb { get; set; } = 50; public int CollectionStaleMinutes { get; set; } = 30; public int CollectionFailureThreshold { get; set; } = 10; /* #1984: DarlingConfig defaults (40% / 1 GB); enable stays the class's opt-in OFF. */ @@ -92,7 +93,10 @@ private sealed class FakeReadAdapter : IAlertReadAdapter { public List Blocking { get; } = new(); public List Deadlocks { get; } = new(); - public List PoisonWaits { get; } = new(); + /* #3539 A4: the accumulation rows the engine grades. Whatever a test puts here comes back + unfiltered — the seam contract is a dumb window sum; the engine does the thresholding. */ + public List PoisonWaits { get; } = new(); + public int PoisonWaitWindowMinutesAsked { get; private set; } public List LongRunning { get; } = new(); public List Volumes { get; } = new(); public List PvsDatabases { get; } = new(); @@ -126,9 +130,11 @@ public Task> GetRecentDeadlocksAsync(string serverKey, in return Task.FromResult(new List(Deadlocks)); } - public Task> GetPoisonWaitDeltasAsync(string serverKey, double thresholdMs, CancellationToken cancellationToken = default) => - /* The seam contract: fetch-then-filter client-side, like Lite's loop. */ - Task.FromResult(PoisonWaits.FindAll(w => w.AvgMsPerWait >= thresholdMs)); + public Task> GetPoisonWaitAccumulationAsync(string serverKey, int windowMinutes, CancellationToken cancellationToken = default) + { + PoisonWaitWindowMinutesAsked = windowMinutes; + return Task.FromResult(new List(PoisonWaits)); + } /* #3495's degrade arm: the CPU card's fire-time maintenance probe rides this same seam, and the pin that a failed probe costs the annotation and never the alert needs a read that faults. */ @@ -151,11 +157,22 @@ public Task> GetLongRunningQueriesAsync( public Task> GetVolumeFreeSpaceAsync(string serverKey, CancellationToken cancellationToken = default) => Task.FromResult(new List(Volumes)); - /* #2349: empty on purpose. These tests exercise other alerts, and a fabricated file would - make the file-growth gate fire inside an unrelated scenario. */ + /* #2349: EMPTY by default. These tests mostly exercise other alerts, and a fabricated file would + make the file-growth gate fire inside an unrelated scenario; the #3539 A8c pins below plant rows + and read back the lookback the engine asked for. #3636: a fetch counter beside it, like the + forced-plan seam, so the once-per-observation pins can assert both what fired and that the read + still happened on every pass. */ + public List Files { get; } = new(); + public int? FileGrowthLookbackAsked { get; private set; } + public int FileGrowthFetches { get; private set; } + public Task> GetDatabaseFileGrowthAsync( - string serverKey, int lookbackMinutes, CancellationToken cancellationToken = default) => - Task.FromResult(new List()); + string serverKey, int lookbackMinutes, CancellationToken cancellationToken = default) + { + FileGrowthLookbackAsked = lookbackMinutes; + FileGrowthFetches++; + return Task.FromResult(new List(Files)); + } public Task GetTempDbSpaceAsync(string serverKey, CancellationToken cancellationToken = default) => Task.FromResult(TempDb); @@ -326,6 +343,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } /// One engine + fakes + a controllable clock per test. @@ -339,6 +365,10 @@ private sealed class Harness public List FailedJobs { get; } = new(); public int FailedJobFetches { get; private set; } public bool Muted { get; set; } + + /* #3539: an optional mute PROBE for pins that need to see which AlertMuteContext the engine asked + about (the poison-wait mute keys on the worst wait type); null keeps the flat Muted answer. */ + public Func? IsMuted { get; set; } public DateTime Now { get; set; } = new(2026, 7, 1, 12, 0, 0, DateTimeKind.Utc); /* #3013: the swallowed-read counter is an OPTIONAL harness input, defaulting to null, so every @@ -355,7 +385,7 @@ into the process-wide AlertReadFailureCounter.Shared. */ public AlertEngine Build(bool withFailedJobsFetcher = false, bool withAgentJobResolver = false) => new( Settings, Adapter, StateStore, Deliverer, - isAlertMuted: _ => Muted, + isAlertMuted: ctx => IsMuted?.Invoke(ctx) ?? Muted, failedJobsFetcher: withFailedJobsFetcher ? (_, _, _) => { FailedJobFetches++; return Task.FromResult(new List(FailedJobs)); } : null, @@ -1367,78 +1397,287 @@ whose processes ALL ran in excluded databases is dropped from the count. */ Assert.Equal("1", Assert.Single(h.Deliverer.Outcomes).CurrentValue); } - /* ---------------- poison waits ---------------- */ + /* ---------------- poison waits (#3539 A4: the accumulation shape) ---------------- */ + + private static readonly DateTime PoisonCollected = new(2026, 9, 18, 12, 0, 0, DateTimeKind.Unspecified); + + private static PoisonWaitAccumulation Poison( + long accumulatedMs, string waitType = "THREADPOOL", long waits = 1, long observed = 10, DateTime? collected = null) + => new(waitType, accumulatedMs, waits, observed, collected ?? PoisonCollected); + /// + /// The storm the retired shape could not see, in the engine: 300,000 THREADPOOL waits of 2 ms each is + /// 600 seconds of worker starvation inside ten minutes — one task continuously starved for the whole + /// window — and the old avg-ms-per-wait bar (500) read it as 2 ms and slept. It fires Warning, with the + /// PostgreSQL twin's exact numeric pair (accumulated ms against the breached bar in ms), the severity on + /// BOTH the outcome and the context override, one detail item carrying the remedy, and no incidents (the + /// metric-level cooldown shape IncidentDeliveryFilter documents for this metric). + /// [Fact] - public async Task PoisonWait_FiresWithWorstWaitNumerics_AndResolvesWhenGone() + public async Task PoisonWait_FiresOnAccumulatedStarvation_NotPerWaitAverage() { - /* Lite AlertEngine.cs:274-333. */ var h = new Harness(); h.Settings.PoisonWaitEnabled = true; + h.Settings.PoisonWaitThresholdMs = 500; var engine = h.Build(); - h.Adapter.PoisonWaits.Add(new PoisonWaitDelta { WaitType = "THREADPOOL", DeltaMs = 100000, DeltaTasks = 50, AvgMsPerWait = 2000 }); + h.Adapter.PoisonWaits.Add(Poison(600_000, waits: 300_000)); await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(PoisonWaitEvaluator.WindowMinutes, h.Adapter.PoisonWaitWindowMinutesAsked); var fired = Assert.Single(h.Deliverer.Outcomes); Assert.Equal("Poison Wait", fired.MetricName); - Assert.Equal("THREADPOOL (2000ms)", fired.CurrentValue); /* :286 */ - Assert.Equal("500ms avg", fired.ThresholdValue); /* :314 */ - Assert.Equal(2000d, fired.NumericCurrentValue); /* :317 */ - Assert.Equal(500d, fired.NumericThresholdValue); /* :318 */ + Assert.Equal("THREADPOOL (600s in 10m)", fired.CurrentValue); + Assert.Equal("600s accumulated over 10m (an average of 1 task(s) continuously waiting)", fired.ThresholdValue); + Assert.Equal(600_000d, fired.NumericCurrentValue); + Assert.Equal(600_000d, fired.NumericThresholdValue); + Assert.Equal(AlertSeverityLevel.Warning, fired.Severity); + Assert.Equal( + "[THREADPOOL] 600s of wait accumulated in the last 10 minutes across 300,000 waits — on average 1.0 task(s) continuously stuck", + fired.ShortMessage); + + Assert.NotNull(fired.Context); + Assert.Equal(AlertSeverityLevel.Warning, fired.Context!.SeverityOverride); + Assert.Null(fired.Context.Incidents); + var detail = Assert.Single(fired.Context.Details); + Assert.Equal("THREADPOOL", detail.Heading); + Assert.Contains(detail.Fields, f => f.Item1 == "Remedy" && f.Item2 == PoisonWaitEvaluator.SqlServerRemedyFor("THREADPOOL")); + Assert.Contains(detail.Fields, f => f.Item1 == "Accumulated wait" && f.Item2 == "600 s over the last 10 min"); + Assert.NotNull(fired.DetailText); + Assert.Contains("Remedy", fired.DetailText, StringComparison.Ordinal); + } + + /// + /// The false page the retired shape produced: ONE wait of 600 ms is 600 ms of accumulated wait — 0.001 + /// average tasks stuck — and the old bar (avg 600 >= 500) paged CRITICAL on it. Silent now, and so is + /// every shape the 43-server fleet measurement actually produced: the worst ten-minute bucket anywhere + /// (5,795 ms of THREADPOOL), the 703-task 8.2 ms row (5,779 ms), and the daily compile burst just under + /// the old bar (8 tasks, 3,154 ms at 394 ms average). + /// + [Theory] + [InlineData(600, 1)] + [InlineData(5_795, 703)] + [InlineData(5_779, 703)] + [InlineData(3_154, 8)] + [InlineData(599_999, 100_000)] + public async Task PoisonWait_OneSlowWait_AndEveryMeasuredFleetBucket_StaySilent(long accumulatedMs, long waits) + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(accumulatedMs, waits: waits)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.Resolutions); + } + + /// + /// Graded, matching the PostgreSQL twin's boundaries exactly: Critical at ten tasks continuously stuck + /// (6,000,000 ms), and the threshold text names the bar that was crossed rather than always the Warning + /// one. The map arm for "Poison Wait" is CRITICAL for override-less renders (pre-#3539 history rows, + /// which WERE presence-flat critical); a live Critical fire agrees with it, a live Warning fire overrides + /// it through the context. + /// + [Fact] + public async Task PoisonWait_GradesCritical_AtTenTasksContinuouslyStuck() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(6_000_000, waitType: "RESOURCE_SEMAPHORE", waits: 4_000)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + Assert.Equal(AlertSeverityLevel.Critical, fired.Context!.SeverityOverride); + Assert.Equal("6,000s accumulated over 10m (an average of 10 task(s) continuously waiting)", fired.ThresholdValue); + Assert.Equal(6_000_000d, fired.NumericThresholdValue); + Assert.Contains(fired.Context.Details.Single().Fields, f => f.Item1 == "Severity" && f.Item2 == "CRITICAL"); + } + + /// + /// Per wait type, worst-first: a Critical RESOURCE_SEMAPHORE (10.0 avg tasks stuck) leads a Warning + /// THREADPOOL (9.8 — just under the Critical bar), the mute key follows the worst type (Lite's documented + /// limitation, unchanged), the alert's severity is the worst type's, and a third type under the Warning + /// bar neither fires nor rides along — it is not in CurrentValue and has no detail item. + /// + [Fact] + public async Task PoisonWait_JudgesEachWaitTypeIndependently_WorstFirst_MutesOnTheWorst() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + AlertMuteContext? muteAsked = null; + h.IsMuted = ctx => { muteAsked = ctx; return false; }; + var engine = h.Build(); + h.Adapter.PoisonWaits.Add(Poison(5_900_000, waitType: "THREADPOOL", waits: 1_000_000)); /* Warning */ + h.Adapter.PoisonWaits.Add(Poison(6_000_000, waitType: "RESOURCE_SEMAPHORE", waits: 100)); /* Critical */ + h.Adapter.PoisonWaits.Add(Poison(5_795, waitType: "RESOURCE_SEMAPHORE_QUERY_COMPILE", waits: 8)); /* under the bar */ + await engine.EvaluateServerAsync(Harness.Snapshot()); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("RESOURCE_SEMAPHORE (6,000s in 10m), THREADPOOL (5,900s in 10m)", fired.CurrentValue); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + Assert.Equal("RESOURCE_SEMAPHORE", muteAsked!.WaitType); + Assert.Equal("Poison Wait", muteAsked.MetricName); + Assert.Equal(2, fired.Context!.Details.Count); + Assert.DoesNotContain("RESOURCE_SEMAPHORE_QUERY_COMPILE", fired.CurrentValue, StringComparison.Ordinal); + } + + /// + /// The clear arm needs an OBSERVATION. Active, then a sweep whose read returns a row under the bar + /// (the window has aged the wait out — "observed and quiet") clears with the windowed message. But + /// FIRST, a sweep whose read returns NO rows at all (the collector delivered nothing in ten minutes) + /// neither clears nor fires: an absent measurement is not evidence of quiet (#3282's rule for CPU), + /// where the retired shape announced "Poison Waits Cleared" on that same silence. + /// + [Fact] + public async Task PoisonWait_HoldsOnAnEmptyRead_AndClearsOnlyOnAnObservedQuietWindow() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(900_000, waits: 30_000)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* Collector silent: no wait_stats rows for any poison type inside the window. Hold. */ h.Adapter.PoisonWaits.Clear(); + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + Assert.Empty(h.Resolutions); + + /* Rows again, summing to zero across ten observed intervals — the window aged the storm out. A + zero sum is what a genuinely idle window AND a window of the calculator's (0, 0) "unknowable" + markers both read as; the sum treats them identically (nothing added either way), and the + observation that lets the flag clear is the collector delivering rows, not the zero itself. */ + h.Adapter.PoisonWaits.Add(Poison(0, waits: 0, observed: 10, collected: PoisonCollected.AddMinutes(12))); + h.Now = h.Now.AddMinutes(6); await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); var resolution = Assert.Single(h.Resolutions); - Assert.Equal("Poison Waits Cleared", resolution.Title); /* :329 */ - Assert.Equal("SRV-A: Poison wait avg below threshold", resolution.Message); /* :330 */ + Assert.Equal("Poison Waits Cleared", resolution.Title); + Assert.Equal("SRV-A: Poison wait accumulated over the last 10 minutes back below threshold", resolution.Message); + + /* Cleared is an edge: the next quiet sweep says nothing more. */ + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Resolutions); + } + + /// + /// A sub-bar row that is not zero also clears — the measured fleet's own worst bucket (5,795 ms) is + /// "quiet" by this alert's definition — and the retired knob plays no part in that judgement either: + /// a threshold of 1 ms, which under the old shape made every row a poison wait, changes nothing. + /// + [Fact] + public async Task PoisonWait_ClearsOnASubBarWindow_RegardlessOfTheRetiredKnob() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + h.Settings.PoisonWaitThresholdMs = 1; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(600_000, waits: 300_000)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Adapter.PoisonWaits.Clear(); + h.Adapter.PoisonWaits.Add(Poison(5_795, waits: 703, collected: PoisonCollected.AddMinutes(11))); + h.Now = h.Now.AddMinutes(11); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + Assert.Single(h.Resolutions); } [Fact] public async Task PoisonWait_DoesNotRefire_OnTheSameCollectionTime_EvenAfterCooldownElapses() { - /* The read adapter's own "newest row within 10 minutes" window can hand back the SAME - wait_stats row across multiple sweeps when the collector's delivered cadence lags the - alert cooldown — observed live as byte-identical "Poison Wait" alerts ~5-7 minutes - apart on the same server. Cooldown elapsing is not proof a fresh observation exists; - re-firing on an unrefreshed collection_time reports the same event twice. */ + /* #2704, unchanged in substance by the accumulation shape: the read adapter's window can hand + back the SAME newest row across multiple sweeps when the collector's delivered cadence lags + the alert cooldown — observed live as byte-identical "Poison Wait" alerts ~5-7 minutes apart + on the same server. Cooldown elapsing is not proof a fresh observation exists; re-firing on an + unrefreshed collection_time reports the same window twice. */ var h = new Harness(); h.Settings.PoisonWaitEnabled = true; var engine = h.Build(); var firstCollection = new DateTime(2026, 8, 31, 6, 0, 0, DateTimeKind.Utc); - h.Adapter.PoisonWaits.Add(new PoisonWaitDelta - { - WaitType = "RESOURCE_SEMAPHORE_QUERY_COMPILE", - DeltaMs = 113997, - DeltaTasks = 134, - AvgMsPerWait = 850.7, - CollectionTime = firstCollection - }); + h.Adapter.PoisonWaits.Add(Poison(700_000, waitType: "RESOURCE_SEMAPHORE_QUERY_COMPILE", waits: 134, collected: firstCollection)); await engine.EvaluateServerAsync(Harness.Snapshot()); Assert.Single(h.Deliverer.Outcomes); /* Cooldown (5 min default) elapses, but the collector has not produced a new row yet — - the adapter still hands back the identical collection_time. Must NOT re-fire. */ + the adapter still hands back the identical newest collection_time. Must NOT re-fire. */ h.Now = h.Now.AddMinutes(6); await engine.EvaluateServerAsync(Harness.Snapshot()); Assert.Single(h.Deliverer.Outcomes); - /* A genuinely new collection — even with the identical wait-type/value shape — is a - fresh observation of the condition and must fire. */ + /* A genuinely new collection — even with the identical wait-type/value shape — is a fresh + observation of the standing condition and must fire. */ h.Now = h.Now.AddMinutes(6); h.Adapter.PoisonWaits.Clear(); - h.Adapter.PoisonWaits.Add(new PoisonWaitDelta - { - WaitType = "RESOURCE_SEMAPHORE_QUERY_COMPILE", - DeltaMs = 113997, - DeltaTasks = 134, - AvgMsPerWait = 850.7, - CollectionTime = firstCollection.AddMinutes(7) - }); + h.Adapter.PoisonWaits.Add(Poison(700_000, waitType: "RESOURCE_SEMAPHORE_QUERY_COMPILE", waits: 134, collected: firstCollection.AddMinutes(7))); await engine.EvaluateServerAsync(Harness.Snapshot()); Assert.Equal(2, h.Deliverer.Outcomes.Count); } + /// + /// The freshness stamp is taken over the FIRING types only: a quiet type's newer row is not a new + /// observation of the type that is over the bar, so it cannot unlock a re-fire on an unrefreshed sum. + /// + [Fact] + public async Task PoisonWait_FreshnessFollowsTheFiringTypes_NotAQuietSibling() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(700_000, waitType: "THREADPOOL", collected: PoisonCollected)); + h.Adapter.PoisonWaits.Add(Poison(100, waitType: "RESOURCE_SEMAPHORE", collected: PoisonCollected)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Now = h.Now.AddMinutes(6); + h.Adapter.PoisonWaits.Clear(); + h.Adapter.PoisonWaits.Add(Poison(700_000, waitType: "THREADPOOL", collected: PoisonCollected)); + h.Adapter.PoisonWaits.Add(Poison(100, waitType: "RESOURCE_SEMAPHORE", collected: PoisonCollected.AddMinutes(5))); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + } + + /// + /// Suppression (acknowledged / silenced server) evaluates but does not deliver, and it tracks the + /// condition: the active flag still rises, so the later clear is still an edge; and a suppressed clear + /// is silent too — exactly the shape every other family in this engine has. + /// + [Fact] + public async Task PoisonWait_Suppressed_TracksTheConditionWithoutDelivering() + { + var h = new Harness(); + h.Settings.PoisonWaitEnabled = true; + var engine = h.Build(); + + h.Adapter.PoisonWaits.Add(Poison(700_000)); + await engine.EvaluateServerAsync(Harness.Snapshot(suppressed: true)); + Assert.Empty(h.Deliverer.Outcomes); + + h.Adapter.PoisonWaits.Clear(); + h.Adapter.PoisonWaits.Add(Poison(0, waits: 0, collected: PoisonCollected.AddMinutes(11))); + await engine.EvaluateServerAsync(Harness.Snapshot(suppressed: true)); + Assert.Empty(h.Resolutions); + + /* Unsuppressed and still quiet: the flag already fell, so nothing is announced late. */ + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Empty(h.Resolutions); + Assert.Empty(h.Deliverer.Outcomes); + } + /* ---------------- long-running queries ---------------- */ [Fact] @@ -1890,6 +2129,367 @@ public async Task LowDisk_GradesCriticallyLowBreaches_AndAStandingBreachDoesNotR Assert.Equal(3, h.Deliverer.Outcomes.Count); } + /* ---------------- database file growth (#2349; #3539 A8c: the rise is MB per HOUR) ---------------- */ + + private static DatabaseFileGrowthInfo GrowingFile(double growthMb, double windowMinutes) => new() + { + DatabaseName = "tempdb", FileName = "tempdev", PhysicalName = @"D:\data\tempdev.mdf", FileTypeDesc = "ROWS", + TotalSizeMb = 90_000, GrowthMb = growthMb, GrowthWindowMinutes = windowMinutes, + VolumeMountPoint = @"D:\", VolumeTotalMb = 4_000_000, VolumeFreeMb = 3_000_000, + }; + + /// + /// Through the ENGINE: the same growth rate pages on a five-minute lookback and on a one-day lookback, and + /// the threshold line on what fired names the rate, the window it was averaged over, and the megabytes + /// that rate amounts to inside the window — in the unit phrase both Settings windows use. The lookback the + /// engine hands the read is the configured one, so the store read and the bar cannot disagree about the + /// window. + /// + [Theory] + [InlineData(5, 853.34, "rise ≥ 10240 MB/hr averaged over 5 min (≥ 853 MB in the window) or file ≥ 60% of volume")] + [InlineData(1440, 245_760, "rise ≥ 10240 MB/hr averaged over 1440 min (≥ 245760 MB in the window) or file ≥ 60% of volume")] + public async Task FileGrowth_TheRiseIsPerHour_SoTheSameRatePagesOnAnyLookback_AndTheThresholdSaysSo( + int lookbackMinutes, double growthMb, string expectedThreshold) + { + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.FileGrowthLookbackMinutes = lookbackMinutes; + Assert.Equal(10_240, h.Settings.FileGrowthRiseMb); + var engine = h.Build(); + + h.Adapter.Files.Add(GrowingFile(growthMb, lookbackMinutes)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(lookbackMinutes, h.Adapter.FileGrowthLookbackAsked); + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("Database File Growth", fired.MetricName); + Assert.Equal(expectedThreshold, fired.ThresholdValue); + Assert.Contains(AlertContextBuilders.FileGrowthRiseUnit, fired.ThresholdValue, StringComparison.Ordinal); + } + + /// + /// The other half of the same property: a rate a tenth under the bar is silent on both lookbacks — including + /// the day-long one, where the per-window reading paged on 12 GB in a day because 12,288 is more than 10,240. + /// + [Theory] + [InlineData(5, 768)] + [InlineData(1440, 12_288)] + public async Task FileGrowth_ARateUnderTheBar_IsSilentOnAnyLookback(int lookbackMinutes, double growthMb) + { + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.FileGrowthLookbackMinutes = lookbackMinutes; + var engine = h.Build(); + + h.Adapter.Files.Add(GrowingFile(growthMb, lookbackMinutes)); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Empty(h.Deliverer.Outcomes); + } + + /* ---------------- #3636: the rise arm fires once per hourly observation, not once per cooldown ---------------- */ + + /// Two consecutive hourly collections of database_size_stats, so the pins below replay the + /// shape the issue describes rather than an invented one: a rise observed at the top of the hour, twelve + /// five-minute cooldowns of re-reads against the same two rows, the next collection an hour later. + private static readonly DateTime HourlyCollection0 = new(2026, 9, 18, 6, 0, 0, DateTimeKind.Utc); + private static readonly DateTime HourlyCollection1 = HourlyCollection0.AddHours(1); + + /// A rise-only file (2% of a 4 TB volume — the level gate cannot see it) stamped with the collection + /// that produced it. 20 GB in the 60-minute window is twice the default 10,240 MB/hr bar. + private static DatabaseFileGrowthInfo RiseOnlyFile(DateTime? observedAt, double growthMb = 20_480, string fileName = "tempdev") + { + var f = GrowingFile(growthMb, windowMinutes: 60); + f.FileName = fileName; + f.ObservedAtUtc = observedAt; + return f; + } + + /// A level-only file: 80% of a small volume and not growing at all. The standing-level shape that + /// re-fires on the cooldown by design. + private static DatabaseFileGrowthInfo LevelOnlyFile(DateTime? observedAt) => new() + { + DatabaseName = "Sales", FileName = "Sales_log", PhysicalName = @"L:\log\Sales_log.ldf", FileTypeDesc = "LOG", + TotalSizeMb = 400_000, GrowthMb = 0, GrowthWindowMinutes = 60, + VolumeMountPoint = @"L:\", VolumeTotalMb = 500_000, VolumeFreeMb = 90_000, ObservedAtUtc = observedAt, + }; + + [Fact] + public async Task FileGrowth_SameHourlyObservationAcrossTwelveCooldowns_FiresOnce_ThenResolvesOnTheNextCollection() + { + /* #3636's shape: the collector landed a 20 GB rise at 06:00 and nothing else until 07:00. Between those + two collections the adapter returned the SAME row (newest = 06:00, baseline = the window's far edge) + on every ~30 s pass, and the pre-#3636 engine fired on every cooldown expiry — up to twelve cards for + one growth event. The cooldown elapsing is not proof a new observation exists. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + var first = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("Database File Growth", first.MetricName); + + /* Eleven more passes, each past the cooldown, none a new observation — the twelve-card loop. */ + for (var pass = 1; pass <= 11; pass++) + { + h.Now = HourlyCollection0.AddMinutes(pass * 5).AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + } + + Assert.Single(h.Deliverer.Outcomes); + /* And the read still happened on every pass — the guard is on the FIRE, never on the fetch, so the + recovery arm keeps seeing fresh evidence. */ + Assert.Equal(12, h.Adapter.FileGrowthFetches); + + /* 07:00: the next collection. The file did not grow in the new window; the read returns it under the + bar, and the recovery is announced exactly as before #3636. */ + h.Adapter.Files.Clear(); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection1, growthMb: 0)); + h.Now = HourlyCollection1.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + var resolution = Assert.Single(h.Resolutions, r => r.MetricName == "Database File Growth"); + Assert.Contains("no file is growing past the threshold", resolution.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task FileGrowth_NewerObservationWithARise_FiresAgain_CarryingItsOwnNumbers() + { + /* A file that keeps growing across successive hourly collections is still a standing condition: each + collection is a NEW observation with a new rise, and it re-fires — the guard removes repeats of one + observation, not the second card for a second hour of growth. The card carries the new observation's + growth, not the first one's. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0, growthMb: 20_480)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + Assert.Contains("grew 20.0 GB in 60 min", h.Deliverer.Outcomes[0].ShortMessage, StringComparison.Ordinal); + + /* The next collection: another 30 GB in the new window. Same file key, newer stamp. */ + h.Adapter.Files.Clear(); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection1, growthMb: 30_720)); + h.Now = HourlyCollection1.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.Contains("grew 30.0 GB in 60 min", h.Deliverer.Outcomes[1].ShortMessage, StringComparison.Ordinal); + + /* And that observation, re-read past another cooldown, is one card too. */ + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.Empty(h.Resolutions); + } + + [Fact] + public async Task FileGrowth_NewerObservationWithoutARise_DoesNotFire_AndResolves() + { + /* The third arm: a newer observation in which the file did NOT grow past the bar is not news for the + rise gate — it is the falling edge. No second card; the recovery is announced. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Adapter.Files.Clear(); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection1, growthMb: 512)); /* a twentieth of the bar */ + h.Now = HourlyCollection1.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + Assert.Single(h.Resolutions, r => r.MetricName == "Database File Growth"); + } + + [Fact] + public async Task FileGrowth_TheLevelArm_StillRefiresEveryCooldown_OnTheSameObservation() + { + /* The guard is on the RISE gate only. A file at 80% of its volume is at 80% on every pass whether or + not a new collection has landed — a standing level, re-fired on the cooldown by design (#2349), and + #3636 leaves that alone. Same stamp on every read; it fires on every cooldown regardless. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(LevelOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(3, h.Deliverer.Outcomes.Count); + + /* And a rise-only file riding on the same server's card does not silence the level file: the card is + per server, one file with news is enough, and the level file is always news. */ + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(5, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task FileGrowth_NewObservationInsideTheCooldown_FiresOnceTheCooldownElapses() + { + /* The memory is 'last ALERTED observation', not 'last SEEN' (the #3579 lesson): a collection that lands + while the cooldown from the previous card is still running has not been reported, so when the + cooldown elapses and the row is still that observation, it fires. Folding the two into one 'last + seen' stamp would record it as seen on the quiet pass and then never fire it. The hourly collector + makes this rare; a shortened cadence or a manual collection makes it real. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + var quickCollection = HourlyCollection0.AddMinutes(2); + h.Adapter.Files.Clear(); + h.Adapter.Files.Add(RiseOnlyFile(quickCollection, growthMb: 25_600)); + h.Now = quickCollection.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); /* cooldown holds it — rate limiting is still the cooldown's job */ + + h.Now = HourlyCollection0.AddMinutes(5).AddSeconds(50); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task FileGrowth_MutedFire_StampsTheObservation_SoUnmutingDoesNotReplayIt() + { + /* A muted fire is still a fire: delivered flagged Muted, it stamps the cooldown, and since #3636 it + stamps the observation. A mute rule lifted mid-hour must not turn the same 06:00 rise into a fresh + card — the operator muted the server's file growth, not the engine's memory of it. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Muted = true; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + var muted = Assert.Single(h.Deliverer.Outcomes); + Assert.True(muted.Muted); + + h.Muted = false; + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + } + + [Fact] + public async Task FileGrowth_RecoveryForgetsTheObservation_SoANewEpisodeFires() + { + /* The falling edge clears the memory with the server (the #2166 lesson, at file grain): a file that + recovers and later grows again is a new episode and its first rise fires, whatever the stamp. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Adapter.Files.Clear(); + h.Now = HourlyCollection1.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Resolutions, r => r.MetricName == "Database File Growth"); + + var laterCollection = HourlyCollection1.AddHours(3); + h.Adapter.Files.Add(RiseOnlyFile(laterCollection)); + h.Now = laterCollection.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task FileGrowth_AFileThatLeavesTheBreachedSet_LosesItsMemory_WhileAnotherKeepsTheCardActive() + { + /* The memory is per FILE, pruned as files leave the breached set, not per server. tempdev fires at + 06:00; at 07:00 tempdev is quiet and templog has the rise — templog has never fired, so the card goes + (no recovery: the server still has a breaching file). At 08:00 tempdev is back with a fresh stamp + and templog is quiet: tempdev's 06:00 memory went with it when it left, and it fires as a new + episode would anyway — asserted through the card's headline naming the file. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0, fileName: "tempdev")); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection0, growthMb: 0, fileName: "templog")); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + Assert.StartsWith("tempdb.tempdev", h.Deliverer.Outcomes[0].ShortMessage, StringComparison.Ordinal); + + h.Adapter.Files.Clear(); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection1, growthMb: 0, fileName: "tempdev")); + h.Adapter.Files.Add(RiseOnlyFile(HourlyCollection1, fileName: "templog")); + h.Now = HourlyCollection1.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.StartsWith("tempdb.templog", h.Deliverer.Outcomes[1].ShortMessage, StringComparison.Ordinal); + Assert.Empty(h.Resolutions); + + /* Same 07:00 observation re-read past the cooldown: templog was reported; nothing new. */ + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task FileGrowth_StamplessRow_KeepsThePre3636CooldownRepeat() + { + /* The stated fallback for an adapter that supplies no observation stamp (the shipped two always do): + a null never matches a remembered stamp, so every read counts as new and the cooldown alone + rate-limits it — the pre-#3636 behaviour, degraded towards repetition rather than silence, the same + direction IAlertStateStore's no-op fallbacks degrade. Pinned so the fallback is a decision and not + an accident of null comparison. */ + var h = new Harness(); + h.Settings.FileGrowthEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = HourlyCollection0.AddSeconds(40); + h.Adapter.Files.Add(RiseOnlyFile(observedAt: null)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + /* ---------------- persistent version store (#1984) ---------------- */ [Fact] @@ -2349,7 +2949,7 @@ public Task> GetRecentBlockedProcessReportsAsync(st throw new InvalidOperationException("store down"); public Task> GetRecentDeadlocksAsync(string serverKey, int hoursBack, CancellationToken cancellationToken = default) => throw new InvalidOperationException("store down"); - public Task> GetPoisonWaitDeltasAsync(string serverKey, double thresholdMs, CancellationToken cancellationToken = default) => + public Task> GetPoisonWaitAccumulationAsync(string serverKey, int windowMinutes, CancellationToken cancellationToken = default) => throw new InvalidOperationException("store down"); public Task> GetLongRunningQueriesAsync(string serverKey, int thresholdMinutes, int maxResults, bool excludeSpServerDiagnostics, bool excludeWaitFor, bool excludeBackups, bool excludeMiscWaits, bool excludeCdc, IReadOnlyList excludedDatabases, CancellationToken cancellationToken = default) => throw new InvalidOperationException("store down"); @@ -2488,6 +3088,210 @@ the plan the way the firing message did — NOT the internal key. The first vers Assert.Contains("plan 22", resolution.Message, StringComparison.Ordinal); } + /* ---------------- #3579: one observation, one card ---------------- */ + + /// The production series' collection instants (#3579), so the pins below replay the shape that + /// was measured rather than an invented one: the 04:02→04:18 rise, six cooldowns of re-reads, the 04:50 + /// collection with the counter back at zero. + private static readonly DateTime Collection0402 = new(2026, 9, 18, 4, 2, 0, DateTimeKind.Utc); + private static readonly DateTime Collection0418 = new(2026, 9, 18, 4, 18, 0, DateTimeKind.Utc); + private static readonly DateTime Collection0450 = new(2026, 9, 18, 4, 50, 0, DateTimeKind.Utc); + + private static ForcePlanFailureInfo ForcePlanRow(DateTime? observedAt, long delta = 1, long total = 1) => new() + { + DatabaseName = "Sales", QueryId = 11, PlanId = 22, ForcingType = "AUTO", FailureReason = "NONE", + FailureDelta = delta, TotalFailures = total, ObservedAtUtc = observedAt + }; + + [Fact] + public async Task ForcePlanFailure_SameObservationAcrossSixCooldowns_FiresOnce_ThenResolvesOnTheNextCollection() + { + /* #3579's measured shape: one plan's counter went 0 → 1 at the 04:18 collection and back to 0 at + 04:50. Between those two collections the adapter returned the SAME row (newest = 04:18, previous = + 04:02) on every ~30 s pass, and the pre-#3579 engine fired at 04:18:53, 04:24:31, 04:30:04, + 04:35:33, 04:40:49 and 04:46:03 — six cards, every one reading New 1 / Total 1, for a force that + failed once. The cooldown elapsing is not proof a new observation exists. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = Collection0418.AddSeconds(53); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0418)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* The five re-reads that each fired before. Every one is past the cooldown; none is a new observation. */ + foreach (var refire in new[] { "04:24:31", "04:30:04", "04:35:33", "04:40:49", "04:46:03" }) + { + h.Now = DateTime.SpecifyKind(DateTime.Parse("2026-09-18 " + refire, System.Globalization.CultureInfo.InvariantCulture), DateTimeKind.Utc); + await engine.EvaluateServerAsync(Harness.Snapshot()); + } + + Assert.Single(h.Deliverer.Outcomes); + /* And the read still happened on every pass — the guard is on the FIRE, never on the fetch, so the + recovery arm keeps seeing fresh evidence. */ + Assert.Equal(6, h.Adapter.ForcePlanFetches); + + /* 04:50: APC released the forcing and the counter reset. The adapter's '>' filter drops the row, and + the recovery is announced exactly as before #3579. */ + h.Adapter.ForcePlanFailures.Clear(); + h.Now = Collection0450.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Single(h.Deliverer.Outcomes); + var resolution = Assert.Single(h.Resolutions, r => r.MetricName == ForcePlanTokens.MetricName); + Assert.Contains("plan 22 no longer failing to force", resolution.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task ForcePlanFailure_NewerObservationWithARise_FiresAgain_CarryingItsOwnNumbers() + { + /* A plan that keeps failing across successive collections is still a standing condition: each + collection is a NEW observation with a new rise, and it re-fires — the guard removes repeats of + one observation, not the second card for a second failure. The card carries the new observation's + delta and total, not the first one's. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = Collection0402.AddSeconds(30); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0402, delta: 1, total: 1)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* The next collection: the counter rose again (1 → 3). Same plan key, newer stamp. */ + h.Adapter.ForcePlanFailures.Clear(); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0418, delta: 2, total: 3)); + h.Now = Collection0418.AddSeconds(30); + await engine.EvaluateServerAsync(Harness.Snapshot()); + + Assert.Equal(2, h.Deliverer.Outcomes.Count); + var second = h.Deliverer.Outcomes[1]; + Assert.Equal(2, second.NumericCurrentValue); + Assert.Contains("failed to force 2x", second.ShortMessage, StringComparison.Ordinal); + Assert.Contains("Total Failures: 3", second.DetailText, StringComparison.Ordinal); + + /* And that observation, re-read past another cooldown, is one card too. */ + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.Empty(h.Resolutions); + } + + [Fact] + public async Task ForcePlanFailure_NewObservationInsideTheCooldown_FiresOnceTheCooldownElapses() + { + /* The memory is 'last ALERTED observation', not 'last SEEN': a collection that lands while the + cooldown from the previous card is still running has not been reported, so when the cooldown + elapses and the row is still that observation, it fires. Folding the two memories into one + 'last seen' stamp would record it as seen on the quiet pass and then never fire it — the guard + would have been silencing a real second failure, which is worse than the repeat it replaces. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = Collection0402.AddSeconds(30); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0402)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + /* A fast collection: the next observation lands two minutes later, inside the five-minute cooldown. */ + var quickCollection = Collection0402.AddMinutes(2); + h.Adapter.ForcePlanFailures.Clear(); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(quickCollection, delta: 1, total: 2)); + h.Now = quickCollection.AddSeconds(30); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); /* cooldown holds it — rate limiting is still the cooldown's job */ + + /* Cooldown elapsed, same not-yet-reported observation: it fires now. */ + h.Now = Collection0402.AddMinutes(5).AddSeconds(30); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + + /* And only once. */ + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task ForcePlanFailure_MutedFire_StampsTheObservation_SoUnmutingDoesNotReplayIt() + { + /* A muted fire is still a fire: it is delivered flagged Muted, it stamps the cooldown, and since + #3579 it stamps the observation. A mute rule lifted mid-interval must not turn the same 04:18 + rise into a fresh card — the operator muted the plan, not the engine's memory of it. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Muted = true; + h.Now = Collection0418.AddSeconds(53); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0418)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + var muted = Assert.Single(h.Deliverer.Outcomes); + Assert.True(muted.Muted); + + h.Muted = false; + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + } + + [Fact] + public async Task ForcePlanFailure_RecoveryForgetsTheObservation_SoANewEpisodeFires() + { + /* The falling edge clears the memory with the plan (the #2166 lesson, at plan grain): a plan that + recovers and later fails again is a new episode and its first rise fires, whatever the stamp. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = Collection0418.AddSeconds(53); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(Collection0418)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Adapter.ForcePlanFailures.Clear(); + h.Now = Collection0450.AddSeconds(40); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Resolutions, r => r.MetricName == ForcePlanTokens.MetricName); + + /* Hours later, forced again and failing again. */ + var laterCollection = Collection0450.AddHours(3); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(laterCollection, delta: 1, total: 1)); + h.Now = laterCollection.AddSeconds(30); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + + [Fact] + public async Task ForcePlanFailure_StamplessRow_KeepsThePre3579CooldownRepeat() + { + /* The stated fallback for an adapter that supplies no observation stamp (the shipped two always do): + a null never matches a remembered stamp, so every read counts as new and the cooldown alone + rate-limits it — the pre-#3579 behaviour, degraded towards repetition rather than silence, the + same direction IAlertStateStore's no-op fallbacks degrade. Pinned so the fallback is a decision + and not an accident of null comparison. */ + var h = new Harness(); + h.Settings.ForcePlanFailureEnabled = true; + h.Settings.CooldownMinutes = 5; + h.Now = Collection0418.AddSeconds(53); + h.Adapter.ForcePlanFailures.Add(ForcePlanRow(observedAt: null)); + var engine = h.Build(); + + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Single(h.Deliverer.Outcomes); + + h.Now = h.Now.AddMinutes(6); + await engine.EvaluateServerAsync(Harness.Snapshot()); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + } + [Fact] public async Task ForcePlanFailure_ExcludedDatabase_IsNeverAlerted() { diff --git a/Darling/Darling.Tests/AlertHistoryRowSeverityTests.cs b/Darling/Darling.Tests/AlertHistoryRowSeverityTests.cs new file mode 100644 index 000000000..6ef3e1cb6 --- /dev/null +++ b/Darling/Darling.Tests/AlertHistoryRowSeverityTests.cs @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Linq; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3539 A8e: an alert-history row is styled and reported at the severity the alert actually FIRED at, +/// not the colour its name implies. +/// +/// The defect. is by NAME — "Poison Wait" is +/// red — and it was the only severity the grids had. Poison Wait has been graded Warning/Critical at its +/// fire site since #2711 (PostgreSQL) and #3539 A4 (SQL Server), so a Warning-graded fire rendered red in +/// Lite's grid and the Viewer's alike; the same gap ran the other way for every metric the engine grades +/// ABOVE its name (a CRITICAL low-disk fire, a SUSPECT database) — amber rows for critical pages. The tier +/// was on at fire time and the JSON projection dropped it, so +/// the row had nothing else to offer. +/// +/// The fix, and what these pins hold. carries the tier +/// as a trailing nullable member (both SKUs write through it, so no store changed on either); +/// reads it and falls back to the by-name classifier ONLY for rows +/// that carry none. The pins: the member round-trips by name and reads back; a row written before it +/// existed deserializes exactly as it did (the additive-only contract); the fallback fires only where no +/// tier exists, and says the right thing there; both grids' row classes reach the shared decision and no +/// grid still calls the by-name predicates directly. +/// +public sealed class AlertHistoryRowSeverityTests +{ + /* ─────────────────────────── the member on the wire ─────────────────────────── */ + + private static string WithSeverity(AlertSeverityLevel? level) + { + var context = new AlertContext { SeverityOverride = level }; + context.Details.Add(new AlertDetailItem { Heading = "THREADPOOL", Fields = { ("Accumulated wait", "61 s") } }); + return AlertContextSerializer.Serialize(context); + } + + [Theory] + [InlineData(AlertSeverityLevel.Warning, "\"Severity\":\"Warning\"")] + [InlineData(AlertSeverityLevel.Critical, "\"Severity\":\"Critical\"")] + public void TheTierIsPersistedByName_AndReadsBack(AlertSeverityLevel level, string fragment) + { + var json = WithSeverity(level); + + /* By NAME, not ordinal: the column outlives any build, and a stored "1" would change meaning the + day a member is inserted ahead of Critical. */ + Assert.Contains(fragment, json, StringComparison.Ordinal); + Assert.DoesNotContain("\"Severity\":" + (int)level, json, StringComparison.Ordinal); + + Assert.Equal(level, AlertContextSerializer.TryReadSeverity(json)); + + /* The full rehydration agrees with the cheap read, and the Details it always carried are intact. */ + Assert.True(AlertContextSerializer.TryDeserialize(json, out var restored)); + Assert.Equal(level, restored.SeverityOverride); + Assert.Single(restored.Details); + } + + /// + /// Additive only: a row written BEFORE the member existed — the exact JSON the pre-#3539 serializer + /// emitted, hand-written here so it cannot drift with the serializer — rehydrates its Details and + /// Incidents as it always did and reads back NO tier. Null is the honest answer for that row, and it is + /// what sends the grids to the by-name fallback. + /// + [Fact] + public void ARowWrittenBeforeTheMemberExisted_DeserializesUnchanged_AndCarriesNoTier() + { + const string legacy = + "{\"Details\":[{\"Heading\":\"THREADPOOL\",\"Fields\":[{\"Label\":\"Avg wait\",\"Value\":\"600 ms\"}],\"Body\":null,\"IsCodeBlock\":false,\"Remediation\":null}]," + + "\"Incidents\":[{\"DedupKey\":\"abc\",\"InvolvedObjects\":[\"x\"],\"OccurrenceCount\":2,\"WaitRange\":null,\"TotalOccurrences\":null,\"IncidentStartedUtc\":null,\"Database\":null,\"LastEventUtc\":null}]}"; + + Assert.Null(AlertContextSerializer.TryReadSeverity(legacy)); + + Assert.True(AlertContextSerializer.TryDeserialize(legacy, out var context)); + Assert.Null(context.SeverityOverride); + var detail = Assert.Single(context.Details); + Assert.Equal("THREADPOOL", detail.Heading); + Assert.Equal(("Avg wait", "600 ms"), Assert.Single(detail.Fields)); + var incident = Assert.Single(context.Incidents!); + Assert.Equal("abc", incident.DedupKey); + Assert.Equal(2, incident.OccurrenceCount); + } + + /// A fire with no override persists a null member and reads back none — the same state a + /// legacy row is in, which is why one fallback serves both. + [Fact] + public void AFireWithNoOverride_CarriesNoTier() + { + var json = WithSeverity(null); + Assert.Null(AlertContextSerializer.TryReadSeverity(json)); + Assert.True(AlertContextSerializer.TryDeserialize(json, out var restored)); + Assert.Null(restored.SeverityOverride); + } + + /// The cheap read refuses everything that is not the writer's exact spelling — garbage, a bare + /// ordinal (the coupling the string form exists to avoid), a case variant, a non-object root — and a + /// resolution row's null context. It must never throw: the grids call it once per row. + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("not json")] + [InlineData("[]")] + [InlineData("{\"Details\":[]}")] + [InlineData("{\"Details\":[],\"Severity\":null}")] + [InlineData("{\"Details\":[],\"Severity\":1}")] + [InlineData("{\"Details\":[],\"Severity\":\"1\"}")] + [InlineData("{\"Details\":[],\"Severity\":\"critical\"}")] + [InlineData("{\"Details\":[],\"Severity\":\"Fatal\"}")] + public void TheCheapRead_AnswersNull_ForAnythingThatIsNotAPersistedTier(string? json) + { + Assert.Null(AlertContextSerializer.TryReadSeverity(json)); + Assert.Null(AlertHistoryRowSeverity.FiredAt(json)); + } + + /* ─────────────────────────── the row's severity ─────────────────────────── */ + + /// + /// The headline case: a Poison Wait row that fired WARNING is a warning row, whatever the name says; + /// one that fired CRITICAL is critical; one that carries no tier is critical BY NAME — every SQL Server + /// Poison Wait row written before #3539 A4 was a presence-flat critical fire, so the fallback is the + /// faithful replay for exactly the rows that reach it. + /// + [Fact] + public void APoisonWaitRow_TakesTheTierItFiredAt_AndTheNameOnlyWhenItHasNone() + { + var warning = WithSeverity(AlertSeverityLevel.Warning); + Assert.True(AlertHistoryRowSeverity.IsWarning("Poison Wait", warning)); + Assert.False(AlertHistoryRowSeverity.IsCritical("Poison Wait", warning)); + Assert.Equal(("warning", AlertHistoryRowSeverity.SourceFired), AlertHistoryRowSeverity.Describe("Poison Wait", warning)); + + var critical = WithSeverity(AlertSeverityLevel.Critical); + Assert.True(AlertHistoryRowSeverity.IsCritical("Poison Wait", critical)); + Assert.False(AlertHistoryRowSeverity.IsWarning("Poison Wait", critical)); + Assert.Equal(("critical", AlertHistoryRowSeverity.SourceFired), AlertHistoryRowSeverity.Describe("Poison Wait", critical)); + + /* No tier on the row: the by-name arm, and it says so. */ + Assert.True(AlertHistoryRowSeverity.IsCritical("Poison Wait", null)); + Assert.True(AlertMetricClassifier.IsCritical("Poison Wait")); // the premise the fallback rests on + Assert.Equal(("critical", AlertHistoryRowSeverity.SourceMetricName), AlertHistoryRowSeverity.Describe("Poison Wait", null)); + } + + /// The other direction of the same gap: metrics the engine grades ABOVE their name's colour. + /// A CRITICAL-graded low-disk fire (#1136) and a SUSPECT database (Database State's critical arm) were + /// amber rows; with the tier on the row they are red. Their override-less rows keep the name's amber, + /// which is what the channels rendered for them too. + [Theory] + [InlineData("Volume Free Space")] + [InlineData("Database State")] + public void AMetricGradedAboveItsName_RendersTheGradeItFiredAt(string metric) + { + Assert.True(AlertMetricClassifier.IsWarning(metric)); // the name alone says amber + + var critical = WithSeverity(AlertSeverityLevel.Critical); + Assert.True(AlertHistoryRowSeverity.IsCritical(metric, critical)); + Assert.False(AlertHistoryRowSeverity.IsWarning(metric, critical)); + + Assert.False(AlertHistoryRowSeverity.IsCritical(metric, null)); + Assert.True(AlertHistoryRowSeverity.IsWarning(metric, null)); + } + + /// The presence-flat metrics fire with no override, so their rows carry no tier and the name + /// decides exactly as before — this change moves nothing for them. (Grading them is the engine's half of + /// A8e, not the grid's.) + [Theory] + [InlineData("Deadlocks Detected", true)] + [InlineData("High CPU", false)] + [InlineData("tempdb Space", false)] + [InlineData("Blocking Detected", false)] + public void APresenceFlatMetric_KeepsItsByNameColour(string metric, bool criticalByName) + { + Assert.Equal(criticalByName, AlertHistoryRowSeverity.IsCritical(metric, null)); + Assert.Equal(!criticalByName, AlertHistoryRowSeverity.IsWarning(metric, null)); + Assert.Equal(criticalByName, AlertMetricClassifier.IsCritical(metric)); + Assert.Equal( + (criticalByName ? "critical" : "warning", AlertHistoryRowSeverity.SourceMetricName), + AlertHistoryRowSeverity.Describe(metric, null)); + } + + /// Resolution rows are the name's business and never consult a tier — they are persisted with + /// a null context, and even a stray tier on one must not turn "Poison Waits Cleared" red. The two + /// deliberate INFO reports fire with no override and stay unhighlighted, as + /// intends. + [Fact] + public void ResolutionAndInformationalRows_AreTheNamesBusiness() + { + foreach (var json in new[] { null, WithSeverity(AlertSeverityLevel.Critical) }) + { + Assert.False(AlertHistoryRowSeverity.IsCritical("Poison Waits Cleared", json)); + Assert.False(AlertHistoryRowSeverity.IsWarning("Poison Waits Cleared", json)); + Assert.Equal(("resolution", AlertHistoryRowSeverity.SourceMetricName), AlertHistoryRowSeverity.Describe("Poison Waits Cleared", json)); + } + + Assert.False(AlertHistoryRowSeverity.IsCritical("Collector Cost Digest", null)); + Assert.False(AlertHistoryRowSeverity.IsWarning("Collector Cost Digest", null)); + Assert.Equal(("info", AlertHistoryRowSeverity.SourceMetricName), AlertHistoryRowSeverity.Describe("Collector Cost Digest", null)); + Assert.Equal(("info", AlertHistoryRowSeverity.SourceMetricName), AlertHistoryRowSeverity.Describe("Fleet Sweep Rollup", null)); + } + + /* ─────────────────────────── the Viewer's row ─────────────────────────── */ + + private static ViewerAlertRow ViewerRow(string metric, string? contextJson) => new() + { + AlertTime = new DateTime(2026, 9, 18, 12, 0, 0), + MetricName = metric, + CurrentValue = 61_000, + ThresholdValue = 60_000, + AlertSent = true, + NotificationType = "webhook", + Muted = false, + ContextJson = contextJson, + }; + + /// The Viewer's grid row reaches the shared decision: a Warning-graded Poison Wait row is + /// amber, not red; a row with no tier keeps the name's red. Lite's AlertHistoryRow is pinned the + /// same way in Lite.Tests (AlertHistoryRowSeverityLiteTests). + [Fact] + public void TheViewerGridRow_RendersTheTierTheAlertFiredAt() + { + var graded = ViewerRow("Poison Wait", WithSeverity(AlertSeverityLevel.Warning)); + Assert.True(graded.IsWarning); + Assert.False(graded.IsCritical); + Assert.False(graded.IsResolved); + + var legacy = ViewerRow("Poison Wait", null); + Assert.True(legacy.IsCritical); + Assert.False(legacy.IsWarning); + + var cleared = ViewerRow("Poison Waits Cleared", null); + Assert.True(cleared.IsResolved); + Assert.False(cleared.IsCritical); + Assert.False(cleared.IsWarning); + } + + /* ─────────────────────────── one decision, every grid ─────────────────────────── */ + + /// + /// Both SKUs' grid rows reach , and NO shipping file under Lite/ or + /// Darling/ calls the by-name IsCritical / IsWarning predicates directly any more — a + /// grid that did would be back to colouring a Warning-graded row red. The deprecated Dashboard is + /// outside the census on purpose: its own engine still fires Poison Wait presence-flat CRITICAL, so its + /// by-name red is faithful to its own rows, and it is a frozen twin. + /// + [Fact] + public void BothGridRows_ReachTheSharedDecision_AndNoGridStillClassifiesByNameAlone() + { + var rows = new[] + { + Path.Combine("Lite", "Services", "LocalDataService.AlertHistory.cs"), + Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.AlertHistory.cs"), + }; + foreach (var relative in rows) + { + var code = CSharpSourceWalker.StripCommentsAndStrings(RepoFile.ReadRepoFile(relative.Split(Path.DirectorySeparatorChar))); + Assert.Contains("AlertHistoryRowSeverity.IsCritical(MetricName, ContextJson)", code, StringComparison.Ordinal); + Assert.Contains("AlertHistoryRowSeverity.IsWarning(MetricName, ContextJson)", code, StringComparison.Ordinal); + } + + var byNameCallers = new[] { "Lite", "Darling" } + .SelectMany(root => Directory.EnumerateFiles(Path.Combine(RepoFile.Root, root), "*.cs", SearchOption.AllDirectories)) + .Where(f => + { + var segments = f.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return !segments.Contains("bin") && !segments.Contains("obj") + && !segments.Contains("Darling.Tests") && !segments.Contains("Lite.Tests"); + }) + .Where(f => + { + var code = CSharpSourceWalker.StripCommentsAndStrings(File.ReadAllText(f)); + return code.Contains("AlertMetricClassifier.IsCritical(", StringComparison.Ordinal) + || code.Contains("AlertMetricClassifier.IsWarning(", StringComparison.Ordinal); + }) + .Select(f => Path.GetRelativePath(RepoFile.Root, f)) + .OrderBy(f => f, StringComparer.Ordinal) + .ToArray(); + + Assert.Empty(byNameCallers); + } +} diff --git a/Darling/Darling.Tests/AlertMasterSwitchSurfaceTests.cs b/Darling/Darling.Tests/AlertMasterSwitchSurfaceTests.cs index 3b8075980..d30f1d430 100644 --- a/Darling/Darling.Tests/AlertMasterSwitchSurfaceTests.cs +++ b/Darling/Darling.Tests/AlertMasterSwitchSurfaceTests.cs @@ -47,7 +47,9 @@ public sealed class AlertMasterSwitchSurfaceTests /* ---------------- the delivery-call census ---------------- */ /// - /// A call that can put an alert on a channel: the shared deliverer seam (DeliverAsync), the + /// A call that can put an alert on a channel: the shared deliverer seam (DeliverAsync, and its + /// #3580 reporting twin DeliverAndReportAsync — the same send, answering what the channels did, + /// which the self-alert funnel now calls so the two daily documents can stamp delivered-today), the /// analysis notify seam (NotifyAsync / SendFindingAlertAsync), Lite's direct send seam /// (TrySendAlertEmailAsync), the shared send core (TrySendAsync), and the deliberate /// channel-probe statics (SendTest*). Dot-qualified on purpose: a DECLARATION has no receiver, @@ -55,7 +57,7 @@ public sealed class AlertMasterSwitchSurfaceTests /// comment-and-string-stripped source, so prose mentioning a seam is not a site. /// private static readonly Regex s_deliveryCall = new( - @"\??\.\s*(?:DeliverAsync|NotifyAsync|TrySendAlertEmailAsync|TrySendAsync|SendFindingAlertAsync|SendTestPagerDutyAsync|SendTestTeamsAsync|SendTestSlackAsync|SendTestGenericAsync)\s*\(", + @"\??\.\s*(?:DeliverAsync|DeliverAndReportAsync|NotifyAsync|TrySendAlertEmailAsync|TrySendAsync|SendFindingAlertAsync|SendTestPagerDutyAsync|SendTestTeamsAsync|SendTestSlackAsync|SendTestGenericAsync)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant); /// How a censused site pays for its place on a delivery path. @@ -282,8 +284,10 @@ public void EverySelfAlertFamily_ConsultsTheMasterSwitch() for (var i = 0; i < lines.Length; i++) { - /* A firing call, not the funnel's own declaration: FireAsync is called bare (same class). */ - if (!Regex.IsMatch(lines[i], @"(? — the funnel reports + what the channels did), so the exclusion allows a type-argument list on the Task. */ + if (!Regex.IsMatch(lines[i], @"(?]*>)?\s+FireAsync\s*\(")) { continue; } diff --git a/Darling/Darling.Tests/AlertMetricClassifierTests.cs b/Darling/Darling.Tests/AlertMetricClassifierTests.cs index 71c562296..c9386d4e4 100644 --- a/Darling/Darling.Tests/AlertMetricClassifierTests.cs +++ b/Darling/Darling.Tests/AlertMetricClassifierTests.cs @@ -18,6 +18,12 @@ namespace Darling.Tests; /// the old duplicated inline copies missed, and the last four are the same drift caught one layer down in /// Darling's self-alert recoveries (#991) — plus the critical (Deadlock/Poison) and warning buckets, over /// the metric names the alert engines actually emit. +/// +/// #3539 A8e: the critical/warning buckets are the FALLBACK the two SKU grids use for a row that +/// carries no persisted tier; a row that does is styled by the tier it fired at, through +/// AlertHistoryRowSeverity (pinned in AlertHistoryRowSeverityTests). The by-name pins here +/// therefore hold for the rows that predate the member, which is what "Poison Wait" being critical by name +/// means now — every such SQL Server row was a presence-flat critical fire. /// public class AlertMetricClassifierTests { diff --git a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs index c67f45d54..da4223c1e 100644 --- a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs +++ b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs @@ -884,7 +884,7 @@ real failure rather than a matcher that never matches anything. */ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScopes = { (Path.Combine("PerformanceMonitor.Alerting", "AlertEngine.cs"), 14, 6), - (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 8, 11), + (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 9, 12), }; /// @@ -945,9 +945,14 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop /// delivery end. And AlertEngine.cs carries a FOURTEENTH since #3495: the maintenance-annotation /// probe on the High CPU fire path — counted because its swallowed failure silently costs the card the /// one line that closes the triage, and an operator chasing a mystery backup deserves to see that the - /// probe went blind rather than that no maintenance ran. + /// probe went blind rather than that no maintenance ran. And DarlingSelfAlertEvaluator.cs a + /// NINTH since #3580: the daily documents' delivered-today stamp read, ONE site serving both the digest + /// and the rollup (so one literal name; the warning beside it names the document) — counted because a + /// swallowed stamp read is the gate falling back to process memory, which is the pre-#3580 + /// re-announce-per-restart posture returning for that tick, and a population of those under store + /// contention is exactly what this census exists to make visible. Its sibling WRITE is exempt. /// - private const int CountedSites = 32; + private const int CountedSites = 33; /// /// Log-message fragments that identify a catch block DELIBERATELY not counted, each paired with the @@ -980,6 +985,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop ["Failed to check failed jobs"] = "the fetcher reads the monitored server's msdb; the block's only store op is a write both stores swallow", ["CONVERTS the fault into the unreadable count"] = "a parse arm, not a read: the fleet-sweep rollup's store read is counted above it, and a document that does not parse becomes the rollup's own reportable unreadable count - the fault is evidence, not a swallow", ["Could not resolve Agent job names"] = "reads the monitored server's msdb through the host resolver, not the store - the Recently-failed-job precedent one seam over; the card degrades to the unresolved form whose raw marker keeps the gap visible, and the page still delivers", + ["delivery stamp could not be written"] = "a write (#3580): the daily document was already delivered and process memory already gates it; the dropped stamp costs one re-announcement at the next restart and never a delivery - the stamp READ beside it is the read, and it is counted", }; /// @@ -1070,8 +1076,10 @@ a person rather than netting out silently. */ rollup's own unreadable count rather than a read failure. 23rd since #3497: the Agent-job resolver's catch, an msdb read on the monitored server degrading to the unresolved form. 24th since #3514: the web-dashboard TLS certificate self-alert's catch, whose evidence is the in-memory - WebTlsCertificateState report the web host publishes - there is no store read to swallow. */ - Assert.Equal(24, totalExempt); + WebTlsCertificateState report the web host publishes - there is no store read to swallow. 25th + since #3580: the daily documents' delivery-stamp WRITE, a write whose loss costs one + re-announcement at the next restart and never a delivery. */ + Assert.Equal(25, totalExempt); /* Every exemption in the table is actually used. An exemption for a message that no longer exists is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch diff --git a/Darling/Darling.Tests/AlertStoredValueTests.cs b/Darling/Darling.Tests/AlertStoredValueTests.cs index d43b0754e..b75683655 100644 --- a/Darling/Darling.Tests/AlertStoredValueTests.cs +++ b/Darling/Darling.Tests/AlertStoredValueTests.cs @@ -91,6 +91,7 @@ private sealed class Settings : IAlertEngineSettings public int DiskCriticalFreePercent { get; set; } = 3; public int DiskCriticalFreeGb { get; set; } = 2; public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int SelfDiskFreeWarnGb { get; set; } = 50; public int CollectionStaleMinutes { get; set; } = 30; public int CollectionFailureThreshold { get; set; } = 10; public int PvsThresholdPercent { get; set; } = 40; @@ -117,6 +118,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private sealed class NullHistory : IAlertHistoryStore diff --git a/Darling/Darling.Tests/AnalysisCoverageLivePostgresTests.cs b/Darling/Darling.Tests/AnalysisCoverageLivePostgresTests.cs new file mode 100644 index 000000000..beaf997a4 --- /dev/null +++ b/Darling/Darling.Tests/AnalysisCoverageLivePostgresTests.cs @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Analysis; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Analysis; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3538 A2 on the real store (gated, DARLING_TEST_PG): every rate and fraction fact divides by the time +/// the collector actually observed, a window it only partly observed says so in every analysis payload, +/// and a fully collected window is unchanged. Lite's FactCollectorTests / +/// AnalysisCoverageTests are the twin; this is the same scenario through Darling's collector, +/// service and tools, because the coverage witness is a QUERY and a query is proven on the engine that +/// runs it. +/// +/// The scenario is the review's failure case: a CXPACKET storm at a true 25% of observed time and +/// ten blocking events, with the collector down for three of the window's four hours. Divided by the +/// nominal window that read as 6% and 2.5/hr — under every bar — with no caveat. The series is planted +/// as a collector would have written it: a baseline reading at the window start whose delta is +/// unknowable, then a delta every fifteen minutes for the hour the collector was up, and nothing after. +/// +[Collection("live-postgres")] +public sealed class AnalysisCoverageLivePostgresTests +{ + private const string ServerName = "darling-analysis-coverage-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task APartlyCollectedWindow_RatesPerObservedTime_AndEveryPayloadSaysSo() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live coverage test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await RegisterServerAsync(connection, ct); + + /* Whole-minute bounds so the PG microsecond comparisons are exact; the window ends a minute + ago so "now" inside the tools is safely past every planted row. */ + var windowEnd = TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1); + var windowStart = windowEnd.AddHours(-4); + + /* The data-span gate measures lifetime history: one old row carries the server past it. */ + await PlantWaitAsync(connection, windowEnd.AddHours(-30), "OLD_WAIT", 1_000L, ct); + + /* The first hour: a baseline reading, then four deltas of 225,000 ms = 900,000 ms of + CXPACKET over ONE observed hour. 0.25 of observed time; 0.0625 of the nominal window. */ + await PlantWaitAsync(connection, windowStart, "CXPACKET", 0L, ct); + for (var i = 1; i <= 4; i++) + await PlantWaitAsync(connection, windowStart.AddMinutes(15 * i), "CXPACKET", 225_000L, ct); + + /* Ten blocking events in that hour: 10/hr observed, 2.5/hr nominal. */ + for (var i = 0; i < 10; i++) + await PlantBlockingAsync(connection, windowStart.AddMinutes(5 + i * 5), 70 + i, ct); + + var collector = new PgFactCollector(postgres); + var context = new AnalysisContext + { + ServerId = ServerId, + ServerName = ServerName, + TimeRangeStart = windowStart, + TimeRangeEnd = windowEnd, + ServerUtcOffset = TimeSpan.Zero + }; + var facts = await collector.CollectFactsAsync(context); + + /* ── the stamp: one hour of four, the three-hour tail the largest hole. */ + var coverage = context.Coverage!; + Assert.True(coverage.IsPartial); + Assert.Equal(0.25, coverage.Fraction, precision: 6); + Assert.Equal(3_600_000, coverage.ObservedMs, precision: 3); + Assert.Equal(10_800_000, coverage.LargestGapMs, precision: 3); + Assert.Equal(5, coverage.SampleCount); + + /* ── the facts: per observed time, four times what the nominal division claimed. */ + var cx = Assert.Single(facts, f => f.Key == "CXPACKET"); + Assert.Equal(0.25, cx.Value, precision: 6); + Assert.Equal(0.25, cx.Metadata["coverage_fraction"], precision: 6); + Assert.Equal(14_400_000, cx.Metadata["period_duration_ms"]); + Assert.Equal(cx.Value, cx.Metadata["wait_time_ms"] / (cx.Metadata["period_duration_ms"] * cx.Metadata["coverage_fraction"]), precision: 6); + + var blocking = Assert.Single(facts, f => f.Key == "BLOCKING_EVENTS"); + Assert.Equal(10.0, blocking.Value, precision: 6); + Assert.Equal(1.0, blocking.Metadata["observed_hours"], precision: 6); + Assert.Equal(4.0, blocking.Metadata["period_hours"], precision: 6); + + var gap = Assert.Single(facts, f => f.Key == WindowCoverage.FactKey); + Assert.Equal(WindowCoverage.FactSource, gap.Source); + Assert.Equal(0.25, gap.Value, precision: 6); + Assert.Equal(10_800_000, gap.Metadata["largest_gap_ms"], precision: 3); + + /* ── the tools. The window inside them ends at "now", a minute past windowEnd, so the + figures are a hair under the collector-level ones; the claims are the same. */ + var service = new DarlingAnalysisService(postgres); + + var analysis = await DarlingMcpTools.AnalyzeServer(service, postgres, ServerName, 4); + using (var doc = JsonDocument.Parse(analysis)) + { + var root = doc.RootElement; + Assert.Equal("findings", root.GetProperty("status").GetString()); + Assert.Contains("PARTIAL COVERAGE", root.GetProperty("caveat").GetString()!, StringComparison.Ordinal); + var cov = root.GetProperty("coverage"); + Assert.True(cov.GetProperty("partial").GetBoolean()); + Assert.InRange(cov.GetProperty("observed_fraction").GetDouble(), 0.24, 0.26); + Assert.InRange(cov.GetProperty("largest_gap_hours").GetDouble(), 2.99, 3.05); + + var cxFinding = root.GetProperty("findings").EnumerateArray() + .Single(f => f.GetProperty("root_fact").GetProperty("key").GetString() == "CXPACKET"); + Assert.InRange(cxFinding.GetProperty("root_fact").GetProperty("value").GetDouble(), 0.24, 0.27); + } + Assert.True(service.LastWindowCoverage is { IsPartial: true }); + + var factsPayload = await DarlingMcpTools.GetAnalysisFacts(service, postgres, ServerName, 4); + using (var doc = JsonDocument.Parse(factsPayload)) + { + var root = doc.RootElement; + Assert.Contains("PARTIAL COVERAGE", root.GetProperty("caveat").GetString()!, StringComparison.Ordinal); + Assert.True(root.GetProperty("coverage").GetProperty("partial").GetBoolean()); + Assert.Contains(root.GetProperty("facts").EnumerateArray(), + f => f.GetProperty("key").GetString() == WindowCoverage.FactKey); + } + + /* compare_analysis: the baseline window (28h back) was never observed, the comparison + window partly — both caveats, and the gap fact reported through the coverage blocks + rather than as a compared key. */ + var compared = await DarlingMcpTools.CompareAnalysis(service, postgres, ServerName, 4, 28); + using (var doc = JsonDocument.Parse(compared)) + { + var root = doc.RootElement; + var caveat = root.GetProperty("caveat").GetString()!; + Assert.Contains("The BASELINE window produced no facts at all", caveat, StringComparison.Ordinal); + Assert.Contains("The COMPARISON window was only partly collected", caveat, StringComparison.Ordinal); + Assert.True(root.GetProperty("baseline").GetProperty("coverage").GetProperty("unobserved").GetBoolean()); + Assert.True(root.GetProperty("comparison").GetProperty("coverage").GetProperty("partial").GetBoolean()); + Assert.DoesNotContain(root.GetProperty("facts").EnumerateArray(), + f => f.GetProperty("key").GetString() == WindowCoverage.FactKey); + + /* #3538 A3 composes with the caveat: every verdict row and family carries coverage_caveat, + the storm is a comparison-only row banded by presence (worse: 0.25 saturates CXPACKET's + ladder), the rules are stated, and the summary counts families beside rows. */ + Assert.True(root.GetProperty("summary").GetProperty("coverage_caveat").GetBoolean()); + Assert.All(root.GetProperty("facts").EnumerateArray(), f => Assert.True(f.GetProperty("coverage_caveat").GetBoolean())); + Assert.All(root.GetProperty("families").EnumerateArray(), f => Assert.True(f.GetProperty("coverage_caveat").GetBoolean())); + var cxRow = Assert.Single(root.GetProperty("facts").EnumerateArray(), f => f.GetProperty("key").GetString() == "CXPACKET"); + Assert.Equal("comparison_only", cxRow.GetProperty("presence").GetString()); + Assert.Equal("presence", cxRow.GetProperty("band_source").GetString()); + Assert.Equal("worse", cxRow.GetProperty("status").GetString()); + Assert.Equal("parallelism", cxRow.GetProperty("family").GetString()); + /* BLOCKING_EVENTS at 10/hr (base 0.5) is the other comparison-only key that registers; the + two are two families (parallelism, lock_contention), so families_worse counts causes. */ + var blockingRow = Assert.Single(root.GetProperty("facts").EnumerateArray(), f => f.GetProperty("key").GetString() == "BLOCKING_EVENTS"); + Assert.Equal("worse", blockingRow.GetProperty("status").GetString()); + Assert.Equal("lock_contention", blockingRow.GetProperty("family").GetString()); + Assert.True(root.GetProperty("summary").GetProperty("new_issues").GetInt32() >= 2); + Assert.True(root.GetProperty("summary").GetProperty("families_worse").GetInt32() >= 2); + Assert.Contains("N=1 vs N=1", root.GetProperty("reading").GetString()!, StringComparison.Ordinal); + Assert.Contains("robust-sigma", root.GetProperty("band_rules").GetProperty("baseline").GetString()!, StringComparison.Ordinal); + } + + /* ── zero coverage: a 2h window anchored inside the dead stretch composes with the #3524 + envelope — unavailable, never an all-clear. */ + var anchor = DateTime.SpecifyKind(windowEnd, DateTimeKind.Utc).ToString("o"); + var dead = await DarlingMcpTools.AnalyzeServer(service, postgres, ServerName, 2, anchor); + using (var doc = JsonDocument.Parse(dead)) + { + Assert.Equal("unavailable", doc.RootElement.GetProperty("status").GetString()); + } + Assert.Contains("NOT an all-clear", dead, StringComparison.Ordinal); + Assert.DoesNotContain("within normal ranges", dead, StringComparison.Ordinal); + + /* ── the control: fill the remaining three hours and the same window is fully covered, the + storm reads the same 0.25 (it was the same storm), blocking drops to its true 2.5/hr over + four observed hours, and the gap fact is gone. */ + for (var i = 5; i <= 16; i++) + await PlantWaitAsync(connection, windowStart.AddMinutes(15 * i), "CXPACKET", 225_000L, ct); + + var fullContext = new AnalysisContext + { + ServerId = ServerId, + ServerName = ServerName, + TimeRangeStart = windowStart, + TimeRangeEnd = windowEnd, + ServerUtcOffset = TimeSpan.Zero + }; + var fullFacts = await collector.CollectFactsAsync(fullContext); + + Assert.Equal(1.0, fullContext.Coverage!.Fraction, precision: 6); + Assert.False(fullContext.Coverage.IsPartial); + Assert.DoesNotContain(fullFacts, f => f.Key == WindowCoverage.FactKey); + Assert.Equal(0.25, Assert.Single(fullFacts, f => f.Key == "CXPACKET").Value, precision: 6); + Assert.Equal(2.5, Assert.Single(fullFacts, f => f.Key == "BLOCKING_EVENTS").Value, precision: 6); + + var fullAnalysis = await DarlingMcpTools.AnalyzeServer(service, postgres, ServerName, 4); + using (var doc = JsonDocument.Parse(fullAnalysis)) + { + Assert.Equal(JsonValueKind.Null, doc.RootElement.GetProperty("caveat").ValueKind); + Assert.False(doc.RootElement.GetProperty("coverage").GetProperty("partial").GetBoolean()); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static DateTime TruncateToMinutes(DateTime value) => + DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerMinute)), DateTimeKind.Unspecified); + + private static async Task RegisterServerAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) +VALUES ($1, $2, $3, TRUE, 16, $4, $4) +ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE, sql_major_version = 16;", connection); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified)); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task PlantWaitAsync( + NpgsqlConnection connection, DateTime at, string waitType, long deltaWaitMs, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO wait_stats + (collection_id, collection_time, server_id, server_name, wait_type, + delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms) +VALUES ($1, $2, $3, $4, $5, $6, $7, 0)", connection); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(at); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(waitType); + command.Parameters.AddWithValue(deltaWaitMs > 0 ? 10L : 0L); + command.Parameters.AddWithValue(deltaWaitMs); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task PlantBlockingAsync(NpgsqlConnection connection, DateTime at, int blockedSpid, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO blocked_process_reports + (blocked_report_id, collection_time, server_id, server_name, event_time, + wait_time_ms, blocking_spid, blocked_spid, blocking_status, database_name) +VALUES ($1, $2, $3, $4, $2, 12000, 60, $5, 'suspended', 'AppDb')", connection); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(at); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(blockedSpid); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM wait_stats WHERE server_id = {ServerId}; " + + $"DELETE FROM blocked_process_reports WHERE server_id = {ServerId}; " + + $"DELETE FROM analysis_findings WHERE server_id = {ServerId}; " + + $"DELETE FROM analysis_muted WHERE server_id = {ServerId}; " + + $"DELETE FROM servers WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/AnalyzeWindowEmptyLivePostgresTests.cs b/Darling/Darling.Tests/AnalyzeWindowEmptyLivePostgresTests.cs new file mode 100644 index 000000000..61c62de71 --- /dev/null +++ b/Darling/Darling.Tests/AnalyzeWindowEmptyLivePostgresTests.cs @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Analysis; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Gated (DARLING_TEST_PG) proof of #3524: analyze_server must not answer "all metrics are +/// within normal ranges" when the analysis window collected NOTHING. The pipeline's data-span gate +/// measures LIFETIME history, so a server whose collection died (broken credential, unreachable +/// target) still passes it — and the zero-facts pass used to return a bare [] that the tool +/// rendered as a true-negative all-clear. Both sides of the distinction are asserted, on the REAL +/// 24h gate rather than a zeroed one, because the bug lives precisely in the gap between "enough +/// history" and "an empty window". +/// +[Collection("live-postgres")] +public sealed class AnalyzeWindowEmptyLivePostgresTests +{ + private const string ServerName = "darling-analyze-window-empty-e2e"; + private const string StaleWait = "WE3524_STALE_WAIT"; + private const string BenignWait = "WE3524_BENIGN_WAIT"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task ADeadCollectorWindow_IsUnavailable_AndAWindowWithFactsKeepsTheAllClear() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live window-empty analysis test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await RegisterServerAsync(connection, ct); + var service = new DarlingAnalysisService(postgres); + + /* ── the dead-collector shape: 25 hours of history (the real 24h gate PASSES) whose + newest row is five hours old, so the tool's default 4h window holds nothing. */ + await PlantWaitAsync(connection, Naive(DateTime.UtcNow.AddHours(-30)), StaleWait, 60_000L, ct); + await PlantWaitAsync(connection, Naive(DateTime.UtcNow.AddHours(-5)), StaleWait, 60_000L, ct); + + var deadWindow = await DarlingMcpTools.AnalyzeServer(service, postgres, ServerName); + using (var doc = JsonDocument.Parse(deadWindow)) + { + Assert.Equal("unavailable", doc.RootElement.GetProperty("status").GetString()); + + /* The #2506 persistence disclosure survives the new envelope — an anchored + empty-window run still owes the caller that context, and this unanchored one + reports the ordinary answer. */ + Assert.True(doc.RootElement.GetProperty("hints").GetProperty("persisted").GetBoolean()); + } + Assert.Contains("get_collection_health", deadWindow, StringComparison.Ordinal); + Assert.Contains("NOT an all-clear", deadWindow, StringComparison.Ordinal); + Assert.DoesNotContain("within normal ranges", deadWindow, StringComparison.Ordinal); + + /* The service says WHICH kind of nothing this was: the window, not the lifetime span. */ + Assert.NotNull(service.WindowEmptyMessage); + Assert.Null(service.InsufficientDataMessage); + + /* ── the control, and the reason the fix is a distinction rather than a rewording: the + SAME server with a benign wait collected THROUGHOUT the window has facts to score, finds + nothing wrong, and keeps the genuine true-negative all-clear. + + A realistic series — a reading every fifteen minutes from the window's start to now — + rather than the single row this control originally planted, because since #3538 A2 the + engine divides by the time the collector actually observed, and a lone reading with + nothing before it observes no time at all (its delta was the calculator's first + sighting). A single in-window row is now, correctly, the dead-collector shape; the + all-clear is earned by a window the collector was up for. */ + var now = DateTime.UtcNow; + for (var minutesAgo = 240; minutesAgo >= 0; minutesAgo -= 15) + await PlantWaitAsync(connection, Naive(now.AddMinutes(-minutesAgo)), BenignWait, 100L, ct); + + var healthyWindow = await DarlingMcpTools.AnalyzeServer(service, postgres, ServerName); + using (var doc = JsonDocument.Parse(healthyWindow)) + { + Assert.Equal("empty", doc.RootElement.GetProperty("status").GetString()); + + /* #3538 A2: the all-clear now says how much of the window it speaks for. */ + Assert.InRange( + doc.RootElement.GetProperty("hints").GetProperty("coverage").GetProperty("observed_fraction").GetDouble(), + 0.99, 1.0); + } + Assert.Contains("All metrics are within normal ranges", healthyWindow, StringComparison.Ordinal); + Assert.DoesNotContain("PARTIAL", healthyWindow, StringComparison.Ordinal); + Assert.Null(service.WindowEmptyMessage); + Assert.NotNull(service.LastWindowCoverage); + Assert.False(service.LastWindowCoverage!.IsPartial); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// Naive-UTC, the Kind every timestamp column in this store is bound with. + private static DateTime Naive(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Unspecified); + + private static async Task RegisterServerAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) +VALUES ($1, $2, $3, TRUE, 15, $4, $4) +ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE, sql_major_version = 15;", connection); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(Naive(DateTime.UtcNow)); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task PlantWaitAsync( + NpgsqlConnection connection, DateTime at, string waitType, long waitMs, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO wait_stats + (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) +VALUES ($1, $2, $3, $4, $5, $6, $7)", connection); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(at); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(waitType); + command.Parameters.AddWithValue(50L); + command.Parameters.AddWithValue(waitMs); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM wait_stats WHERE server_id = {ServerId}; " + + $"DELETE FROM analysis_findings WHERE server_id = {ServerId}; " + + $"DELETE FROM analysis_muted WHERE server_id = {ServerId}; " + + $"DELETE FROM servers WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs b/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs new file mode 100644 index 000000000..5f7d49e77 --- /dev/null +++ b/Darling/Darling.Tests/CaptureDownChunkOrderTests.cs @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3597: the capture-down self-alert read asks "what did this collector's LATEST run log?" once per +/// collector as a chunk-orderable LIMIT 1, not as a window function over the server's whole +/// collection_log history. +/// +/// This is the read #3496 named and deliberately left — ROW_NUMBER() OVER (PARTITION BY +/// collector_name ORDER BY log_id DESC) cannot early-stop by reordering alone. collection_log is a +/// hypertable partitioned on collection_time with no index on log_id, so the window's only legal +/// plan decompressed EVERY chunk in the server's retention horizon, Merge-Appended them and numbered ~100 K +/// rows to keep two: 9,573 buffers over 61 chunks on a rig with 60 days of one server's log, every alert +/// pass, every 30 seconds, per server — the heaviest read on the pass by two to three orders of magnitude, +/// and one of the four sites the issue saw die at the 10 s deadline while the interval-hourly refresh +/// starved the store. Per collector, ORDER BY collection_time DESC LIMIT 1 lets ChunkAppend walk the +/// chunks newest-first and stop at the first row: 14 buffers, the newest chunk only, 120 of 122 chunk scans +/// never executed. The property is horizon-independent, so a longer retention cannot regress it. +/// +/// The regression is QUIET, exactly as #3496's was: a revert to the window returns the same two rows on +/// any store small enough for a test and only shows up as deadline breaches once a store's retention has +/// filled. So the shape is pinned at the source, and the gated arm asks the planner. +/// +/* Live-fixture tests share one Postgres store; the collection serializes them so cross-test row churn + cannot race another class's assertions. */ +[Collection("live-postgres")] +public sealed class CaptureDownChunkOrderTests +{ + /// Distinctive fake id — a real server_id is a storage-name hash, never this. + private const int TestServerId = -735971; + private const string TestServerName = "capture-down-chunk-order-e2e"; + + [Fact] + public void EachCollector_IsAskedOnce_OrderedByThePartitionColumn_LimitOne() + { + var sql = DarlingSelfAlertEvaluator.MissingCaptureSessionsSql; + + /* One arm per capture collector, UNION ALL between them, each stopping at its newest row. */ + Assert.Equal(2, Regex.Matches(sql, @"ORDER BY cl\.collection_time DESC\s+LIMIT 1").Count); + Assert.Single(Regex.Matches(sql, @"\bUNION ALL\b")); + Assert.Contains("cl.collector_name = 'deadlocks'", sql, StringComparison.Ordinal); + Assert.Contains("cl.collector_name = 'blocked_process_report'", sql, StringComparison.Ordinal); + + /* The verdict is still on the LATEST run's status, applied after each arm has picked its row. */ + Assert.Contains("WHERE x.status = 'SESSION_MISSING'", sql, StringComparison.Ordinal); + } + + /// + /// The negative half: no window function, and no ordering by log_id in any spelling — the two + /// shapes that force every chunk to execute. Matched as shapes rather than the literals that shipped, so a + /// re-spelling cannot slip past the pin the way the original slipped past review. + /// + [Fact] + public void NoWindowFunction_AndNoOrderingByLogId() + { + var sql = DarlingSelfAlertEvaluator.MissingCaptureSessionsSql; + + Assert.DoesNotMatch(new Regex(@"\bOVER\s*\(", RegexOptions.IgnoreCase), sql); + Assert.DoesNotMatch(new Regex(@"ROW_NUMBER", RegexOptions.IgnoreCase), sql); + Assert.DoesNotMatch(new Regex(@"log_id", RegexOptions.IgnoreCase), sql); + } + + /// + /// The evidence no string pin can give: that the planner stops at the newest chunk. Builds the store the + /// way the service does (ladder, then collection_log's hypertable conversion where TimescaleDB is + /// present), seeds one server's capture-collector rows across eight days — eight 1-day chunks — and + /// EXPLAINs the shipped statement with its real bound parameter: no WindowAgg, and at most one + /// chunk executed per arm, every other chunk scan reported never executed. Then the read answers + /// through the same path, and it is the NEWEST run that decides: a SESSION_MISSING three days ago + /// followed by a success is not a missing session; a success three days ago followed by + /// SESSION_MISSING is. + /// + [Fact] + public async Task TheShippedRead_ExecutesOnlyTheNewestChunk_AndTheNewestRunDecides_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live capture-down access-path test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + /* #1922: probe on its own connection. The service's own runtime conversion for collection_log, which sits + outside the collector catalog and so outside ConvertToHypertablesAsync. */ + var timescaleEnabled = await LiveTimescaleProbe.TryEnableAsync(connectionString!, ct); + if (timescaleEnabled) + { + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + } + + var bodySucceeded = false; + try + { + await DeleteTestRowsAsync(connection, ct); + + /* Eight days of one-minute blocked_process_report and five-minute deadlocks rows, plus a filler + collector so the newest chunk holds rows the arms must skip past. All Kind-Unspecified: naive-UTC + storage, see DarlingObservability.LogCollectionAsync. */ + var utcNow = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + await SeedHistoryAsync(connection, utcNow, ct); + + /* The two verdict rows. deadlocks: SESSION_MISSING three days ago, then a SUCCESS a minute ago — NOT + missing. blocked_process_report: SUCCESS all along, then SESSION_MISSING a minute ago — missing. */ + await InsertAsync(connection, 9_100_000_001, "deadlocks", utcNow.AddDays(-3).AddSeconds(7), "SESSION_MISSING", ct); + await InsertAsync(connection, 9_100_000_002, "deadlocks", utcNow.AddMinutes(-1), "SUCCESS", ct); + await InsertAsync(connection, 9_100_000_003, "blocked_process_report", utcNow.AddMinutes(-1), "SESSION_MISSING", ct); + + using (var analyze = new NpgsqlCommand("ANALYZE collect.collection_log", connection)) + { + await analyze.ExecuteNonQueryAsync(ct); + } + + var plan = await ExplainShippedReadAsync(connection, ct); + + Assert.DoesNotContain("WindowAgg", plan, StringComparison.Ordinal); + + if (timescaleEnabled) + { + /* Every chunk scan the plan carries, split into executed and never-executed. ChunkAppend orders + the chunks newest-first for ORDER BY collection_time DESC, so each arm's LIMIT 1 is satisfied + by the newest chunk and the rest never start. */ + var chunkScans = plan.Split('\n').Where(l => Regex.IsMatch(l, @"Scan .* on _hyper_\d+_\d+_chunk")).ToList(); + Assert.True(chunkScans.Count >= 2 * 8, + "expected the plan to carry at least eight chunk scans per arm (eight seeded days):\n" + plan); + var executed = chunkScans.Where(l => !l.Contains("never executed", StringComparison.Ordinal)).ToList(); + Assert.True(executed.Count <= 2, + "more than one chunk executed per arm — the read is walking history again:\n" + plan); + } + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var missing = await DarlingSelfAlertEvaluator.ReadMissingCaptureSessionsAsync(postgres, TestServerId, ct); + Assert.Equal(new[] { "Blocking" }, missing); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteTestRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// One multi-row INSERT per collector over the eight days, so the seed is three statements rather than + /// fifteen thousand. log_id is derived from the row's instant so id order and time order agree, as + /// CollectionIdGenerator's do. + /// + private static async Task SeedHistoryAsync(NpgsqlConnection connection, DateTime utcNow, CancellationToken ct) + { + var start = utcNow.AddDays(-8); + foreach (var (collector, stepMinutes) in new[] { ("blocked_process_report", 1), ("deadlocks", 5), ("wait_stats", 1) }) + { + using var insert = new NpgsqlCommand( + "INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, rows_collected) " + + "SELECT 9_000_000_000 + (EXTRACT(EPOCH FROM t)::bigint * 10) + $5, $1, $2, $3, t, 20, 'SUCCESS', 0 " + + "FROM generate_series($4::timestamp, $4::timestamp + interval '8 days' - interval '2 minutes', ($6::text || ' minutes')::interval) AS t", connection); + insert.Parameters.AddWithValue(TestServerId); + insert.Parameters.AddWithValue(TestServerName); + insert.Parameters.AddWithValue(collector); + insert.Parameters.AddWithValue(start); + insert.Parameters.AddWithValue((long)stepMinutes); + insert.Parameters.AddWithValue(stepMinutes.ToString(System.Globalization.CultureInfo.InvariantCulture)); + await insert.ExecuteNonQueryAsync(ct); + } + } + + private static async Task InsertAsync(NpgsqlConnection connection, long logId, string collector, DateTime when, string status, CancellationToken ct) + { + using var insert = new NpgsqlCommand( + "INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, rows_collected) " + + "VALUES ($1, $2, $3, $4, $5, 20, $6, 0)", connection); + insert.Parameters.AddWithValue(logId); + insert.Parameters.AddWithValue(TestServerId); + insert.Parameters.AddWithValue(TestServerName); + insert.Parameters.AddWithValue(collector); + insert.Parameters.AddWithValue(when); + insert.Parameters.AddWithValue(status); + await insert.ExecuteNonQueryAsync(ct); + } + + private static async Task ExplainShippedReadAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var explain = new NpgsqlCommand( + "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) " + DarlingSelfAlertEvaluator.MissingCaptureSessionsSql, connection); + explain.Parameters.AddWithValue(TestServerId); + var plan = new StringBuilder(); + using var reader = await explain.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + plan.AppendLine(reader.GetString(0)); + } + + return plan.ToString(); + } + + private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand("DELETE FROM collect.collection_log WHERE server_id = $1", connection); + cleanup.Parameters.AddWithValue(TestServerId); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/CollectionSignalsChunkOrderTests.cs b/Darling/Darling.Tests/CollectionSignalsChunkOrderTests.cs index c67bfc8c7..a8c363bc9 100644 --- a/Darling/Darling.Tests/CollectionSignalsChunkOrderTests.cs +++ b/Darling/Darling.Tests/CollectionSignalsChunkOrderTests.cs @@ -32,9 +32,10 @@ namespace Darling.Tests; /// same rows on any store small enough for a test, passes every behavioral assertion, and only shows up /// months later as tail-latency deadline breaches on the store whose retention has filled — exactly how /// the defect presented the first time. So the ORDER the statement asks for is pinned at the source, -/// scoped to the one statement (the capture-down read's ROW_NUMBER ... ORDER BY cl.log_id DESC in -/// the same file is a different shape — a window function cannot early-stop by ordering swap alone — and -/// is deliberately not swept). +/// scoped to the one statement. The capture-down read in the same file was the different shape this pin +/// originally left alone — ROW_NUMBER ... ORDER BY cl.log_id DESC, which no ordering swap could +/// early-stop; #3597 reshaped it into one chunk-orderable LIMIT 1 per collector, and +/// pins that one. /// public sealed class CollectionSignalsChunkOrderTests { diff --git a/Darling/Darling.Tests/CollectionSweepCommandTimeoutTests.cs b/Darling/Darling.Tests/CollectionSweepCommandTimeoutTests.cs index 20caca6b9..b4c2b0d74 100644 --- a/Darling/Darling.Tests/CollectionSweepCommandTimeoutTests.cs +++ b/Darling/Darling.Tests/CollectionSweepCommandTimeoutTests.cs @@ -97,7 +97,7 @@ public sealed class CollectionSweepCommandTimeoutTests /// list missed, not a member of this regime. ReadStoreSizeBytesAsync runs on the disk-check /// cadence. RunTestHypotheticalIndexAsync / RunExecuteActualPlanAsync and /// DarlingCommandExecutor are the command plane, with a 5-minute claim lease and no heartbeat. - /// DarlingDeltaCalculator's four seeds and StoreConfigProvider's seven seeding sites run + /// DarlingDeltaCalculator's seeds (ten since #3540 A4) and StoreConfigProvider's seven seeding sites run /// ONCE at startup. StoreConfigProvider.ReadConfigVersionAsync is the 15 s reload beacon and is /// the closest call of all — it runs on the sweep's own tick — but it runs on the SERIAL loop thread /// ahead of every launch, so its blast radius is the whole fleet and its floor is a single-row lookup diff --git a/Darling/Darling.Tests/CollectorCostDigestTests.cs b/Darling/Darling.Tests/CollectorCostDigestTests.cs index 1e883b9ff..631489787 100644 --- a/Darling/Darling.Tests/CollectorCostDigestTests.cs +++ b/Darling/Darling.Tests/CollectorCostDigestTests.cs @@ -265,6 +265,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private sealed class RecordingHistoryStore : IAlertHistoryStore diff --git a/Darling/Darling.Tests/CollectorDatabaseScopeRungTests.cs b/Darling/Darling.Tests/CollectorDatabaseScopeRungTests.cs index 9099ad3f3..4f54ab322 100644 --- a/Darling/Darling.Tests/CollectorDatabaseScopeRungTests.cs +++ b/Darling/Darling.Tests/CollectorDatabaseScopeRungTests.cs @@ -8,7 +8,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Reflection; using Npgsql; @@ -36,18 +35,20 @@ namespace Darling.Tests; /// because the composed predicate is scoped-in AND NOT excluded. New databases stay OUT of a /// non-empty scope until named — the allow-list-not-deny-list argument the issue was won on. /// -/// This file carries the "I am the top rung" claims that moved off -/// (V124) when this rung landed, the same handoff that -/// file received from (V123) — a fully-migrated store must map -/// to EXACTLY this version, or the viewer's connect-time gate refuses a store that is actually -/// current. +/// The "I am the top rung" claims have moved ON to +/// (V126), the same handoff this file received from +/// (V124) and that file received from (V123). What stays here +/// is everything true of this rung wherever it sits in the ladder; what left is every claim that was +/// really about being NEWEST — keeping a copy of those would assert this rung is still the top, +/// which is how the NEXT rung's build goes red. /// public sealed class CollectorDatabaseScopeRungTests { private const int RungVersion = 125; private const int PreviousVersion = 124; - /// This rung's sentinel ordinal in the viewer probe — the newest, so the last argument. + /// This rung's sentinel ordinal in the viewer probe. No longer the last argument — V126 + /// appended its own — so this is a position within the signature rather than its end. private const int ProbeOrdinal = 100; private const string ScopeColumn = "databases"; @@ -55,7 +56,7 @@ public sealed class CollectorDatabaseScopeRungTests /* ---- the rung ------------------------------------------------------------------------------------ */ [Fact] - public void TheRungIsRegisteredAtTheTopOfADenseLadder() + public void TheRungIsRegisteredInADenseLadder() { var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); @@ -65,7 +66,11 @@ public void TheRungIsRegisteredAtTheTopOfADenseLadder() Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); - Assert.Equal(RungVersion, StorageVersion.SchemaVersion); + + /* Not `RungVersion == SchemaVersion` any more: that asserted this rung is the newest, which + stopped being true when V126 landed. The invariant that outlives the handoff is that the + LADDER's top and the declared version agree, which the two lines above already say. */ + Assert.True(RungVersion < StorageVersion.SchemaVersion); Assert.Equal(versions.Distinct().OrderBy(v => v), versions); } @@ -113,15 +118,15 @@ this rung out of MigrationDataMovingRungCensusPins' register. */ /* ---- the probe (three sites, top arm) ------------------------------------------------------------- */ /// - /// The viewer probe's three sites carry this rung's sentinel, and the map treats it as the TOP arm. + /// The viewer probe's three sites carry this rung's sentinel, and its arm still answers. /// /// The probe asks the question, the caller reads the answer, the map has the parameter — three /// sites, and a sentinel present at only some of them shifts every LATER ordinal onto the wrong column. - /// Miss all three and a fully-migrated store probes one rung short, so the connect-time gate refuses a - /// store that is in fact current — permanently, because no later upgrade changes the answer. + /// The top-arm claims (last argument, textual newest-first ordering) moved to + /// with the V126 handoff. /// [Fact] - public void TheProbeMapsAFullyMigratedStoreToThisTopRung() + public void TheProbeCarriesThisRungsSentinel_AndAFullyMigratedStoreMapsToTheLaddersTop() { Assert.Contains($"column_name = '{ScopeColumn}'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); Assert.Contains("table_name = 'config_collector_schedules'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); @@ -136,8 +141,10 @@ public void TheProbeMapsAFullyMigratedStoreToThisTopRung() .GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!; var arity = method.GetParameters().Length; - /* The top rung's sentinel IS the last argument. */ - Assert.Equal(ProbeOrdinal, arity - 1); + /* The ordinal has to be a position that exists, and one that is no longer the last: `arity - 1` + asserted this rung is the NEWEST sentinel, which stopped being true the moment V126 appended + its own. Strictly-less is the form every other non-top rung's test here uses. */ + Assert.True(ProbeOrdinal < arity - 1); /* Every sentinel true = a fully-migrated store, which must map to exactly this version. Built by reflection so the arity tracks the signature. */ @@ -154,19 +161,6 @@ test of that rung. */ var behind = (object[])atThisRung.Clone(); behind[ProbeOrdinal] = false; Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!); - - /* And in the source, the arm sits ABOVE V124's — newest-first is the whole contract of that method — - and returns this build's version rather than a literal that could drift from it. This is the - textual half of the top-arm claim, inherited from FleetSweepCadenceKnobRungTests the way that - file inherited it from FleetSweepStateRungTests. */ - var v125 = viewer.IndexOf("if (hasCollectorScheduleDatabases)", StringComparison.Ordinal); - var v124 = viewer.IndexOf("if (hasFleetSweepCadenceKnobs)", StringComparison.Ordinal); - Assert.True(v125 >= 0, "the viewer has no V125 sentinel arm — a fully-migrated store would map to 124"); - Assert.True(v124 >= 0, "the V124 arm is gone, so this pin is comparing against nothing"); - Assert.True(v125 < v124, "the V125 arm sits below V124's, so a current store maps one rung low"); - Assert.Contains( - "return " + StorageVersion.SchemaVersion.ToString(CultureInfo.InvariantCulture) + ";", - viewer[v125..], StringComparison.Ordinal); } /* ---- every schedule-row surface handles the column ------------------------------------------------ */ diff --git a/Darling/Darling.Tests/CollectorEngineCapabilityDerivationTests.cs b/Darling/Darling.Tests/CollectorEngineCapabilityDerivationTests.cs index ebfb1ab1c..8bac92cd2 100644 --- a/Darling/Darling.Tests/CollectorEngineCapabilityDerivationTests.cs +++ b/Darling/Darling.Tests/CollectorEngineCapabilityDerivationTests.cs @@ -477,12 +477,21 @@ public void TheShippedCatalogSplitsCleanlyOnTheKindAxis() Assert.All(sqlServer, c => Assert.True(CollectorEngineCapability.IsCollectedOnEngineKind(c, MonitoredEngineKind.SqlServer))); Assert.All(postgres, c => Assert.False(CollectorEngineCapability.IsCollectedOnEngineKind(c, MonitoredEngineKind.SqlServer))); - /* Aurora is a strict superset of the surfaces the PostgreSQL collectors read, so every one of them - applies there. This is the half that would break if a new collector were written against something - Aurora removes rather than adds. */ - Assert.All(postgres, c => Assert.True( - CollectorEngineCapability.IsCollectedOnEngineKind(c, MonitoredEngineKind.AuroraPostgres), - $"{c.Name} is reported as a permanent gap on Aurora PostgreSQL")); + /* Aurora is a strict superset of the surfaces the PostgreSQL collectors read - with ONE exception + since #3604, and it is the case this half was written to catch: pg_wait_sampling is written against + something Aurora REMOVES (the ability to preload the module; Aurora permits a fixed list of + libraries and pg_wait_sampling is not on it), so it is a permanent gap there by design, and + CoveredInsteadBy sends an Aurora caller to pg_wait_stats, which reads the engine's own counters. + Named here rather than filtered generically, so a SECOND collector gated off Aurora has to argue + its case in this comment rather than slip through. */ + var auroraGaps = postgres + .Where(c => !CollectorEngineCapability.IsCollectedOnEngineKind(c, MonitoredEngineKind.AuroraPostgres)) + .Select(c => c.Name) + .ToArray(); + Assert.Equal(new[] { "pg_wait_sampling" }, auroraGaps); + Assert.Contains("pg_wait_stats", + CollectorEngineCapability.NotCollectedMessage("aurora-01", 0, MonitoredEngineKind.AuroraPostgres, "pg_wait_sampling"), + StringComparison.Ordinal); /* Stock PostgreSQL is where the Aurora-only surfaces become a real gap (#2532), so the two tokens genuinely differ — counted from the catalog rather than listed, and asserted as a PROPER subset so diff --git a/Darling/Darling.Tests/CollectorStateContractTests.cs b/Darling/Darling.Tests/CollectorStateContractTests.cs index 52ddbf57f..a69b13c3a 100644 --- a/Darling/Darling.Tests/CollectorStateContractTests.cs +++ b/Darling/Darling.Tests/CollectorStateContractTests.cs @@ -110,13 +110,16 @@ public void TheCollectorsDeclaringStateArePinned() collector is a two-host concern rather than a definition-local one. Pinned on the catalog surface both hosts iterate. - ONE of them now: default_trace_events' last-seen trace FILE (#1962). pg_index_bloat's - per-database rotation cursor (#3153) was the other and #3234 retired it — the statistics - estimate covers every index in one statement, so there is no position to resume from. It did - not need host CODE, and neither does the survivor: the wiring below is generic. Enumerated in - the same file that pins that wiring, so a collector newly declaring state lands here first. */ + TWO of them now. default_trace_events' last-seen trace FILE (#1962); pg_index_bloat's + per-database rotation cursor (#3153) was the other until #3234 retired it — the statistics + estimate covers every index in one statement, so there is no position to resume from. Then + pg_wait_sampling (#3604): the arm that ran (its instrument token, which the reads disclose) and, + on the service-sampler arm, the cumulative tally the next cycle adds to — state a MAX() over the + table cannot recover, since the table holds only the tally's last written value per key. Neither + needed host CODE: the wiring below is generic. Enumerated in the same file that pins that wiring, + so a collector newly declaring state lands here first. */ Assert.Equal( - new[] { "default_trace_events" }, + new[] { "default_trace_events", "pg_wait_sampling" }, CollectorCatalog.All .Where(c => c.StateKeys.Count > 0) .Select(c => c.Name) diff --git a/Darling/Darling.Tests/CommandPlaneCommandTimeoutTests.cs b/Darling/Darling.Tests/CommandPlaneCommandTimeoutTests.cs index 9c52b8c0e..4287689f2 100644 --- a/Darling/Darling.Tests/CommandPlaneCommandTimeoutTests.cs +++ b/Darling/Darling.Tests/CommandPlaneCommandTimeoutTests.cs @@ -87,7 +87,7 @@ public sealed class CommandPlaneCommandTimeoutTests /// site that pin's file-scoped list missed, not a member of this regime. ReadStoreSizeBytesAsync /// runs on the 5-minute disk-check cadence. StoreConfigProvider's other twelve sites seed and /// reconcile ONCE per process start, unlike the beacon below which the sweep re-runs every 15 s for the - /// life of the process. DarlingDeltaCalculator's four are startup seeds. + /// life of the process. DarlingDeltaCalculator's seeds (ten since #3540 A4) are startup seeds. /// DarlingCommandExecutor's test_connect, snapshot_now, analyze_now, /// purge_now, fetch_plan and fetch_active_queries branches touch the store through /// members other groups own or not at all. diff --git a/Darling/Darling.Tests/CommentFilterAdoptionTests.cs b/Darling/Darling.Tests/CommentFilterAdoptionTests.cs index 415280e0a..151b55062 100644 --- a/Darling/Darling.Tests/CommentFilterAdoptionTests.cs +++ b/Darling/Darling.Tests/CommentFilterAdoptionTests.cs @@ -61,7 +61,7 @@ namespace Darling.Tests; /// .github/darling-paths-filter.yml, which names Darling/**, the shared libraries Darling /// compiles against, and the gate's own inputs — so no entry added to the build job's filter can stand up /// its throwaway cluster. CrossAppGuardCiGateTests does see the -/// two Lite.Tests keys below — #3067 widened its anchor past the app directory, since +/// three Lite.Tests keys below — #3067 widened its anchor past the app directory, since /// Lite.Tests is a sibling of Lite rather than a directory inside it — and exempts them on /// this same reasoning, under the same bound. /// @@ -80,7 +80,7 @@ public sealed class CommentFilterAdoptionTests /// gives: the way a wildcard grows is that nobody has to /// name a new entry. /// - /// Four kinds live here and they are not the same kind. Five collect a doc-comment run, + /// Four kinds live here and they are not the same kind. Six collect a doc-comment run, /// where the prefix is what defines the run. One reads a stated, measured bound off a named file. /// One filters SQL, which the C# walk cannot help with. One demonstrates the shape on an /// arranged fixture, which is this file. @@ -135,6 +135,17 @@ public sealed class CommentFilterAdoptionTests + "before any figure is compared - a truncated walk fails there instead of silently pinning half " + "a comment.", + ["Lite.Tests/FactScorerTests.cs"] = + "COLLECTS a doc run. GetWaitThresholds_EveryEntryCarriesItsMeasurementLineage (#3538 A5) gathers the " + + "contiguous // run above each entry of FactScorer.GetWaitThresholds and asks it for a percentile, " + + "a max, 'measured' or 'unmeasured' - the lineage lives ONLY in those comments, so asking for the " + + "walker would leave nothing to read. Stated bound: the collector recognises // lines and entry " + + "lines and skips everything else, so lineage written in a /* */ block whose continuation lines " + + "carry no prefix is invisible to it and the entry reads as UNDOCUMENTED - a spurious red, the " + + "loud direction. Entries are matched by a quoted-key regex, so a block-comment continuation line " + + "that happened to spell one would register as a phantom entry needing lineage of its own, " + + "which is again loud; nothing in the table today is a block comment.", + ["Lite.Tests/LiteSidebarDotRendersTheCardStatusTests.cs"] = "STATED BOUND, and asking for the walker would BREAK it. Its doc comment records the measurement: " + "ServerConnection.cs carries exactly one block comment, the licence header, so dropping " diff --git a/Darling/Darling.Tests/CompressionStuckConfirmReadTests.cs b/Darling/Darling.Tests/CompressionStuckConfirmReadTests.cs new file mode 100644 index 000000000..bf91691b5 --- /dev/null +++ b/Darling/Darling.Tests/CompressionStuckConfirmReadTests.cs @@ -0,0 +1,615 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3575: the compression-stuck check's -infinity arm is a NON-ATOMIC read of TimescaleDB's own view, +/// and the fix is to read it twice. +/// +/// The defect these pin against. already +/// guarded its dead-job arm with !isRunning, and a production store paged through the guard anyway — +/// the alert's stamp 53 ms inside a 63 ms scheduled run that succeeded. The 2.28.1 view definition explains +/// it: job_status is CASE WHEN pgs.state = 'active' THEN 'Running' … END over a +/// LEFT JOIN pg_stat_activity on application_name, and next_start is the +/// bgw_job_stat row. The scheduler commits -infinity before the worker exists; the worker is +/// gone before its mark_end is visible to a snapshot taken a moment earlier. A tight poll of +/// across a 10-second-cadence policy on a PG18 + +/// TimescaleDB 2.28.1 rig caught -infinity + Scheduled at BOTH edges of every one of seven runs. +/// The predicate stays pure and single-shot; +/// re-reads after and reports only what +/// persists. These tests script the two reads through the internal seam, so an edge and a dead row are +/// each a pair of result sets and nothing here sleeps. +/// +/// What is NOT pinned here, deliberately. The stuck-Running arm's six-hour bound and the +/// evaluator's re-arm-once/escalate machine are unchanged by #3575 and keep their own pins +/// (TimescaleSupportTests, DarlingSelfAlertTests). The only claims this file makes are about the +/// second read: when it is taken, what it ratifies, what it clears, and what a failed one does. +/// +public sealed class CompressionStuckConfirmReadTests +{ + private static readonly DateTime s_now = new(2026, 9, 18, 8, 47, 0, DateTimeKind.Utc); + + /* The three row shapes the view can hand the predicate for one job, named for what they are. */ + + /// The dead-job shape and the run-instant edge: identical on one read — that is the defect. + private static CompressionJobStatRow NegInfinityScheduled(long jobId, string hypertable = "file_io_stats") => + new(jobId, NextStartIsNegativeInfinity: true, JobStatus: "Scheduled", + LastRunStartedAtUtc: s_now.AddHours(-1), ScheduleInterval: TimeSpan.FromHours(1), HypertableName: hypertable); + + /// A healthy job between runs: finite next_start, not running. + private static CompressionJobStatRow Healthy(long jobId, string hypertable = "file_io_stats") => + new(jobId, NextStartIsNegativeInfinity: false, JobStatus: "Scheduled", + LastRunStartedAtUtc: s_now.AddMinutes(-1), ScheduleInterval: TimeSpan.FromHours(1), HypertableName: hypertable); + + /// The mid-run marker: -infinity WITH Running. Belongs to the elapsed arm, never the dead-job arm. + private static CompressionJobStatRow MidRun(long jobId, string hypertable = "file_io_stats") => + new(jobId, NextStartIsNegativeInfinity: true, JobStatus: "Running", + LastRunStartedAtUtc: s_now.AddMilliseconds(-40), ScheduleInterval: TimeSpan.FromHours(1), HypertableName: hypertable); + + /// A hung run: Running since eight hours ago against the six-hour floor. + private static CompressionJobStatRow HungRun(long jobId, string hypertable = "query_stats") => + new(jobId, NextStartIsNegativeInfinity: true, JobStatus: "Running", + LastRunStartedAtUtc: s_now.AddHours(-8), ScheduleInterval: TimeSpan.FromHours(1), HypertableName: hypertable); + + /// + /// A scripted read: hands back each result set in turn, records how many times it was asked, and throws + /// the scripted exception in place of a result set when one is planted. + /// + private sealed class ScriptedReads + { + private readonly Queue _script = new(); + public int Calls { get; private set; } + + public ScriptedReads Then(params CompressionJobStatRow[] rows) + { + _script.Enqueue((IReadOnlyList)rows); + return this; + } + + public ScriptedReads ThenThrow(Exception ex) + { + _script.Enqueue(ex); + return this; + } + + public Task> Read(CancellationToken ct) + { + Calls++; + Assert.True(_script.Count > 0, $"the read was asked a {Calls}th time with nothing scripted for it"); + var next = _script.Dequeue(); + return next is Exception ex + ? Task.FromException>(ex) + : Task.FromResult((IReadOnlyList)next); + } + } + + /// A recording delay: never sleeps, remembers every span it was asked to wait. + private sealed class RecordedDelay + { + public List Waits { get; } = new(); + + public Task Wait(TimeSpan span, CancellationToken ct) + { + Waits.Add(span); + return Task.CompletedTask; + } + } + + /* ---------------- the arm projection ---------------- */ + + [Fact] + public void ClassifyCompressionJob_NamesTheArm_AndIsCompressionJobStuck_IsItsProjection() + { + /* The boolean the existing pins hold is the classifier's projection, so the two cannot disagree — the + reason the classifier is the implementation and not a sibling copy of the same branches. */ + foreach (var (row, expected) in new (CompressionJobStatRow Row, StuckCompressionJobArm Arm)[] + { + (NegInfinityScheduled(1), StuckCompressionJobArm.NextStartNegativeInfinity), + (Healthy(2), StuckCompressionJobArm.None), + (MidRun(3), StuckCompressionJobArm.None), + (HungRun(4), StuckCompressionJobArm.RunningPastBound), + }) + { + var arm = TimescaleSupport.ClassifyCompressionJob( + row.NextStartIsNegativeInfinity, row.JobStatus, row.LastRunStartedAtUtc, row.ScheduleInterval, s_now, out var reason); + var stuck = TimescaleSupport.IsCompressionJobStuck( + row.NextStartIsNegativeInfinity, row.JobStatus, row.LastRunStartedAtUtc, row.ScheduleInterval, s_now, out var boolReason); + + Assert.Equal(expected, arm); + Assert.Equal(arm != StuckCompressionJobArm.None, stuck); + Assert.Equal(reason, boolReason); + } + } + + /* ---------------- the confirm-read: when the second read is taken ---------------- */ + + [Fact] + public async Task HealthyPass_ReadsOnce_AndNeverWaits() + { + /* The common hourly pass: nothing on the racing arm, so the pass costs exactly what it did before + #3575 — one read, no delay. A confirm taken unconditionally would hold the serial sweep loop five + seconds every hour for nothing. */ + var reads = new ScriptedReads().Then(Healthy(1), Healthy(2), MidRun(3)); + var delay = new RecordedDelay(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, delay.Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + Assert.Empty(result); + Assert.Equal(1, reads.Calls); + Assert.Empty(delay.Waits); + } + + [Fact] + public async Task HungRunOnly_ReportsFromTheFirstRead_AndNeverWaits() + { + /* The stuck-Running arm is judged on hours of elapsed time; a second look five seconds later could not + change it, so it neither triggers the confirm nor waits on one. */ + var reads = new ScriptedReads().Then(HungRun(4), Healthy(1)); + var delay = new RecordedDelay(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, delay.Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + var job = Assert.Single(result); + Assert.Equal(4L, job.JobId); + Assert.Contains("Running", job.Reason, StringComparison.Ordinal); + Assert.Equal(1, reads.Calls); + Assert.Empty(delay.Waits); + } + + [Fact] + public async Task NegInfinityTrip_WaitsTheConfirmDelay_ThenReadsAgain_Once() + { + /* One confirm per pass, not per job: two jobs on the racing arm still cost one delay and one second + read. The span waited is the published constant, so the budgeting argument on it is the one the + code actually honours. */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(1), NegInfinityScheduled(2)) + .Then(NegInfinityScheduled(1), NegInfinityScheduled(2)); + var delay = new RecordedDelay(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, delay.Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + Assert.Equal(2, result.Count); + Assert.Equal(2, reads.Calls); + Assert.Equal(new[] { TimescaleSupport.StuckCompressionConfirmDelay }, delay.Waits); + } + + /* ---------------- the confirm-read: what the second read decides ---------------- */ + + [Fact] + public async Task TransientEdge_ClearsOnConfirm_ReportsNothing_AndSaysSoAtInformation() + { + /* THE production shape: the first read lands on the run instant and sees -infinity + Scheduled; five + seconds later the run is long over and the job reads healthy. Nothing is reported, so nothing is + re-armed and nothing is paged — and the near miss is written down where a person reading the log + after this alert family fires would look for it. */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(1011, "file_io_stats")) + .Then(Healthy(1011, "file_io_stats")); + var delay = new RecordedDelay(); + var log = new CapturingTestLogger(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, delay.Wait, s_now, log, TestContext.Current.CancellationToken); + + Assert.Empty(result); + Assert.Equal(2, reads.Calls); + Assert.Contains("Information:", log.Joined, StringComparison.Ordinal); + Assert.Contains("1011", log.Joined, StringComparison.Ordinal); + Assert.Contains("file_io_stats", log.Joined, StringComparison.Ordinal); + Assert.Contains("run-instant edge", log.Joined, StringComparison.Ordinal); + Assert.Contains("#3575", log.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("Warning:", log.Joined, StringComparison.Ordinal); + } + + [Fact] + public async Task PersistentNegInfinity_IsConfirmed_AndReported() + { + /* A row the scheduler has abandoned — or a crashed run sitting out its five-minute backoff, reproduced + on the rig by SIGKILLing a worker — reads the same on both passes. Detection is unchanged in kind; + it is five seconds later in time. */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(7, "wait_stats")) + .Then(NegInfinityScheduled(7, "wait_stats")); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + var job = Assert.Single(result); + Assert.Equal(7L, job.JobId); + Assert.Equal("wait_stats", job.HypertableName); + Assert.Contains("-infinity", job.Reason, StringComparison.Ordinal); + } + + [Fact] + public async Task EdgeThatBecameMidRunOnConfirm_Clears() + { + /* The start edge seen twice in one run, at different stages: first -infinity + Scheduled (the worker + not yet Running), then -infinity + Running (mid-run). The second read's arm is None, so the trip + clears — mid-run belongs to the elapsed arm and always did. */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(1)) + .Then(MidRun(1)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + Assert.Empty(result); + } + + [Fact] + public async Task MixedPass_KeepsTheHungRun_ConfirmsOneTrip_ClearsTheOther() + { + /* Every row class in one pass, so the merge is pinned as a table rather than one row at a time: the + hung run from the first read, the confirmed trip from the second, the transient trip dropped, and + the always-healthy job never mentioned. */ + var reads = new ScriptedReads() + .Then(HungRun(4), NegInfinityScheduled(1), NegInfinityScheduled(2), Healthy(3)) + .Then(HungRun(4), NegInfinityScheduled(1), Healthy(2), Healthy(3)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + Assert.Equal(new[] { 4L, 1L }, result.Select(r => r.JobId).ToArray()); + } + + [Fact] + public async Task JobThatOnlyTripsOnTheConfirm_IsNotReported() + { + /* The confirm ratifies the first pass; it does not widen it. A job that went -infinity between the two + reads has been seen once, which is the count this issue proved insufficient, and it gets its own two + reads next hour. */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(1), Healthy(2)) + .Then(NegInfinityScheduled(1), NegInfinityScheduled(2)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken); + + Assert.Equal(new[] { 1L }, result.Select(r => r.JobId).ToArray()); + } + + /* ---------------- the confirm-read: failure isolation ---------------- */ + + [Fact] + public async Task ConfirmReadFails_DropsTheTrips_KeepsTheHungRun_WarnsOnce_DoesNotThrow() + { + /* A confirm that fails confirms nothing. The -infinity trips are not reported on the strength of the + single read this issue proved insufficient; the hung run, judged from the first read, still is. The + failure is a Warning naming the count and the consequence — the views were readable seconds ago, so + this is a hiccup on the one read that decides whether to page — and it never reaches the sweep. */ + var reads = new ScriptedReads() + .Then(HungRun(4), NegInfinityScheduled(1), NegInfinityScheduled(2)) + .ThenThrow(new InvalidOperationException("connection reset by peer")); + var log = new CapturingTestLogger(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, log, TestContext.Current.CancellationToken); + + var job = Assert.Single(result); + Assert.Equal(4L, job.JobId); + Assert.Contains("Warning:", log.Joined, StringComparison.Ordinal); + Assert.Contains("2 job(s)", log.Joined, StringComparison.Ordinal); + Assert.Contains("confirm read", log.Joined, StringComparison.Ordinal); + Assert.Contains("next hour", log.Joined, StringComparison.Ordinal); + Assert.Contains("connection reset by peer", log.Joined, StringComparison.Ordinal); + Assert.Contains("#3575", log.Joined, StringComparison.Ordinal); + } + + [Fact] + public async Task FirstReadFails_ReturnsEmpty_LogsDebug_NeverWaits_DoesNotThrow() + { + /* The pre-#3575 posture, unchanged: a failed first read is "no signal this check" at Debug — the views + may simply be absent on a plain-PG store — and there is nothing to confirm, so no delay is taken. */ + var reads = new ScriptedReads().ThenThrow(new InvalidOperationException("relation does not exist")); + var delay = new RecordedDelay(); + var log = new CapturingTestLogger(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, delay.Wait, s_now, log, TestContext.Current.CancellationToken); + + Assert.Empty(result); + Assert.Empty(delay.Waits); + Assert.StartsWith("Debug:", log.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("Warning:", log.Joined, StringComparison.Ordinal); + } + + [Fact] + public async Task Cancellation_DuringTheConfirmDelay_Propagates() + { + /* Shutdown during the five-second wait is cancellation, not a read failure: it propagates to the worker's + own quiet catch rather than being swallowed into "confirm failed" and logged as a store fault. */ + var reads = new ScriptedReads().Then(NegInfinityScheduled(1)); + using var cts = new CancellationTokenSource(); + + Task CancelInsteadOfWaiting(TimeSpan span, CancellationToken ct) + { + cts.Cancel(); + return Task.FromCanceled(cts.Token); + } + + await Assert.ThrowsAnyAsync(() => + TimescaleSupport.ReadStuckCompressionJobsAsync(reads.Read, CancelInsteadOfWaiting, s_now, null, cts.Token)); + } + + /* ---------------- the pure merge, pinned directly ---------------- */ + + [Fact] + public void ConfirmStuckCompressionJobs_NullConfirm_DropsEveryNegInfinityTrip_KeepsRunningPastBound() + { + var first = TimescaleSupport.ClassifyStuckCompressionJobs( + new[] { HungRun(4), NegInfinityScheduled(1), NegInfinityScheduled(2) }, s_now); + Assert.Equal(3, first.Count); + + var merged = TimescaleSupport.ConfirmStuckCompressionJobs(first, confirm: null, s_now, logger: null); + + Assert.Equal(new[] { 4L }, merged.Select(m => m.JobId).ToArray()); + } + + [Fact] + public void ConfirmStuckCompressionJobs_CarriesTheConfirmPassRow() + { + /* The reported job is built from the CONFIRM pass's row — the later read is the one that stood. Today the + two reasons are the same string; the pin is on which row is carried, using the hypertable name the two + passes would only ever disagree on in a test. */ + var first = TimescaleSupport.ClassifyStuckCompressionJobs(new[] { NegInfinityScheduled(1, "first") }, s_now); + var merged = TimescaleSupport.ConfirmStuckCompressionJobs( + first, new[] { NegInfinityScheduled(1, "confirm") }, s_now, logger: null); + + var job = Assert.Single(merged); + Assert.Equal("confirm", job.HypertableName); + } + + /* ---------------- the version read: when it is taken and what it decides (#3591) ---------------- */ + + /// A scripted version read: hands back the planted version (or throws), counting calls. + private sealed class ScriptedVersion + { + private readonly Version? _version; + private readonly Exception? _throw; + public int Calls { get; private set; } + + public ScriptedVersion(Version? version) => _version = version; + public ScriptedVersion(Exception ex) => _throw = ex; + + public Task Read(CancellationToken ct) + { + Calls++; + return _throw is null ? Task.FromResult(_version) : Task.FromException(_throw); + } + } + + [Fact] + public async Task HealthyPass_NeverReadsTheVersion() + { + /* The version decides the -infinity arm's sentence and nothing else, so a pass with nothing on that arm + does not pay for it — the common hourly pass is still one read. */ + var reads = new ScriptedReads().Then(Healthy(1), HungRun(4)); + var version = new ScriptedVersion(new Version(2, 28, 1)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken, version.Read); + + Assert.Single(result); + Assert.Equal(0, version.Calls); + Assert.False(result[0].SchedulerRetries); + } + + [Fact] + public async Task PersistentNegInfinity_On_2_28_1_IsConfirmed_WithTheCrashBackoffSentence_AndSchedulerRetries() + { + /* The fleet's shape: a SIGKILLed worker's row on 2.28.1, reproduced on the rig. Still reported — the + arm is the arm — but the sentence is the true one for this version, and SchedulerRetries tells the + evaluator not to re-arm it (which, measured, resets the backoff rather than shortening it). */ + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(7, "wait_stats")) + .Then(NegInfinityScheduled(7, "wait_stats")); + var version = new ScriptedVersion(new Version(2, 28, 1)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken, version.Read); + + var job = Assert.Single(result); + Assert.Equal(7L, job.JobId); + Assert.Equal(1, version.Calls); /* once per pass, like the confirm */ + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityCrashBackoffReason, job.Reason); + Assert.True(job.SchedulerRetries); + } + + [Fact] + public async Task PersistentNegInfinity_BelowTheFix_KeepsTheOldSentence_AndTheReArm() + { + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(7)) + .Then(NegInfinityScheduled(7)); + var version = new ScriptedVersion(new Version(2, 26, 3)); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, logger: null, TestContext.Current.CancellationToken, version.Read); + + var job = Assert.Single(result); + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityPermanentReason, job.Reason); + Assert.False(job.SchedulerRetries); + } + + [Fact] + public async Task VersionReadFails_OrIsAbsent_IsTheOldSentence_AndNeverFailsThePass() + { + /* Words, not verdicts: a version read that throws is swallowed at Debug and the pass proceeds exactly as + if the version were unknown — the conservative sentence, the #1581 re-arm. The pre-#3591 seam (no + reader at all) is the same case. */ + foreach (var reader in new Func>?[] + { + new ScriptedVersion(new InvalidOperationException("permission denied for table pg_extension")).Read, + new ScriptedVersion((Version?)null).Read, + null, + }) + { + var reads = new ScriptedReads() + .Then(NegInfinityScheduled(7)) + .Then(NegInfinityScheduled(7)); + var log = new CapturingTestLogger(); + + var result = await TimescaleSupport.ReadStuckCompressionJobsAsync( + reads.Read, new RecordedDelay().Wait, s_now, log, TestContext.Current.CancellationToken, reader); + + var job = Assert.Single(result); + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityPermanentReason, job.Reason); + Assert.False(job.SchedulerRetries); + Assert.DoesNotContain("Warning:", log.Joined, StringComparison.Ordinal); + } + } + + [Fact] + public void ConfirmStuckCompressionJobs_TheVersionPhrasesTheConfirmRow_HungRunUntouched() + { + /* The merge applies the version to the CONFIRM pass — the row that is carried — and the hung run, judged + from the first pass, never sees it. */ + var first = TimescaleSupport.ClassifyStuckCompressionJobs(new[] { HungRun(4), NegInfinityScheduled(1) }, s_now); + var merged = TimescaleSupport.ConfirmStuckCompressionJobs( + first, new[] { HungRun(4), NegInfinityScheduled(1) }, s_now, logger: null, new Version(2, 28, 1)); + + Assert.Equal(2, merged.Count); + Assert.Contains("Running", merged[0].Reason, StringComparison.Ordinal); + Assert.False(merged[0].SchedulerRetries); + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityCrashBackoffReason, merged[1].Reason); + Assert.True(merged[1].SchedulerRetries); + } + + /* ---------------- the delay constant, against what it has to clear and what it costs ---------------- */ + + [Fact] + public void ConfirmDelay_ClearsTheMeasuredEdges_AndIsSmallAgainstEveryCadenceItSitsInside() + { + var delay = TimescaleSupport.StuckCompressionConfirmDelay; + + /* Quoted measurements, not derived: the rig's whole run was ~7.5 ms end to end and its start edge + ~3 ms; the production store's hourly no-op runs were 40–100 ms; a Windows backend start is realistically + tens of milliseconds. The delay must outlast all of them by an order of magnitude at least. */ + Assert.True(delay >= TimeSpan.FromMilliseconds(100 * 10), + $"the confirm delay ({delay}) must clear a 100 ms run-instant edge ten times over"); + + /* ...and it must stay far below the shortest PERSISTENT -infinity state there is, TimescaleDB's + MIN_WAIT_AFTER_CRASH_MS of five minutes, or a real crash could slip between the two reads. */ + Assert.True(delay < TimeSpan.FromMinutes(1), + $"the confirm delay ({delay}) must be far below the five-minute crash backoff a crashed row sits at -infinity for"); + + /* ...and it must be negligible against the cadences it sits inside — the hourly check and the six-hour + stuck-Running floor — so a genuinely dead job is detected on the same pass it always was. */ + Assert.True(delay.TotalSeconds * 100 < TimeSpan.FromHours(1).TotalSeconds, + $"the confirm delay ({delay}) must be under 1 % of the hourly check cadence"); + Assert.True(delay * 100 < TimescaleSupport.StuckRunningBound(TimeSpan.FromHours(1)), + $"the confirm delay ({delay}) must be under 1 % of the stuck-Running bound"); + } + + /* ---------------- de-alignment: the check's wall-clock phase ---------------- */ + + [Fact] + public void NextCompressionCheckUtc_SnapsToThePhase_NeverToAMinuteBoundary() + { + var interval = TimeSpan.FromHours(1); + + /* The production fire: 08:47:00.053, 53 ms into file_io_stats' :47:00 run. Scheduled from this fire the + old way, the next sample would have been 09:47:00.053 + the loop's latency — on the boundary again. + Snapped, it is 09:47:30 exactly. */ + var fired = new DateTime(2026, 9, 18, 8, 47, 0, 53, DateTimeKind.Utc); + var next = TimescaleSupport.NextCompressionCheckUtc(fired, interval); + Assert.Equal(new DateTime(2026, 9, 18, 9, 47, 30, DateTimeKind.Utc), next); + Assert.Equal(DateTimeKind.Utc, next.Kind); + + /* From any second of the minute, the result sits on the phase, and the phase is not zero — the whole + point is to be OFF the :00 instant the compression policies fire on. */ + Assert.NotEqual(0, TimescaleSupport.CompressionCheckPhaseSeconds); + for (var second = 0; second < 60; second++) + { + var now = new DateTime(2026, 9, 18, 8, 47, second, 500, DateTimeKind.Utc); + var due = TimescaleSupport.NextCompressionCheckUtc(now, interval); + Assert.Equal(TimescaleSupport.CompressionCheckPhaseSeconds, due.Second); + Assert.Equal(0, due.Millisecond); + /* And the cadence stays hourly to within half a minute either way — never two hours, never zero. */ + var spacing = due - now; + Assert.InRange(spacing, interval - TimeSpan.FromSeconds(30), interval + TimeSpan.FromSeconds(30)); + } + } + + [Fact] + public void NextCompressionCheckUtc_LocksThePhase_AcrossSimulatedFires_UnderLoopLatency() + { + /* The steady state the field will see: each fire happens at the first 15-second sweep pass at or after + the due time, so the fire is 0–15 s late; the next due must still snap back to :30, so the phase does + not creep the way "UtcNow + 1 h" did. Simulated over a day with every latency the loop can produce. */ + var interval = TimeSpan.FromHours(1); + var due = TimescaleSupport.NextCompressionCheckUtc(new DateTime(2026, 9, 18, 1, 46, 13, DateTimeKind.Utc), interval); + + for (var hour = 0; hour < 24; hour++) + { + var latency = TimeSpan.FromSeconds(hour % 16); /* 0..15 s, the loop's whole range */ + var fired = due + latency; + due = TimescaleSupport.NextCompressionCheckUtc(fired, interval); + + Assert.Equal(TimescaleSupport.CompressionCheckPhaseSeconds, due.Second); + Assert.Equal(46, due.Minute); /* the minute never moves while the loop keeps under the half-minute */ + } + } + + [Fact] + public void NextCompressionCheckUtc_ALatePass_SnapsBackToThePhase_OrMovesAWholeMinute_NeverTowardTheBoundary() + { + /* A sweep pass delayed past the half-minute (the #2327 store-metrics worst case can hold the loop for + minutes) fires late. Re-anchored from the fire the old way, the next due would carry that lateness + forward and creep toward a boundary. Snapped, there are only two outcomes, and neither is nearer :00. + + Late inside its own minute (:46:42): the next due snaps BACK to :46:30 — twelve seconds short of a + full hour, still on the phase. */ + var lateInMinute = new DateTime(2026, 9, 18, 9, 46, 42, DateTimeKind.Utc); + Assert.Equal( + new DateTime(2026, 9, 18, 10, 46, 30, DateTimeKind.Utc), + TimescaleSupport.NextCompressionCheckUtc(lateInMinute, TimeSpan.FromHours(1))); + + /* Late into the NEXT minute (:47:05, a pass more than 35 s behind its :46:30 due): the check moves one + whole minute later, to :47:30, and stays on the phase there. */ + var lateIntoNextMinute = new DateTime(2026, 9, 18, 9, 47, 5, DateTimeKind.Utc); + Assert.Equal( + new DateTime(2026, 9, 18, 10, 47, 30, DateTimeKind.Utc), + TimescaleSupport.NextCompressionCheckUtc(lateIntoNextMinute, TimeSpan.FromHours(1))); + } + + [Fact] + public void ThePhase_IsHalfTheCompressionGridStep_AndClearOfEveryPolicyStart() + { + /* The compression policies start at :MM:00 for every MM in CompressionPhaseMinutes (the fixed-schedule + initial_start carries whole minutes and nothing smaller). Half the one-minute grid step is the point + furthest from every start instant in both directions. Derived from the grid's step, not restated. */ + Assert.Equal(30, TimescaleSupport.CompressionCheckPhaseSeconds); + Assert.Equal(TimeSpan.FromMinutes(1) / 2, TimeSpan.FromSeconds(TimescaleSupport.CompressionCheckPhaseSeconds)); + + /* And the AddCompressionPolicySql initial_start really is whole minutes: any policy this product owns + puts its job on :MM:00, so the :30 phase is 30 s from every one of them. */ + foreach (var table in TimescaleSupport.CompressionPhaseOrder) + { + Assert.True(TimescaleSupport.TryCompressionPhaseMinutesFor(table, out var minute)); + Assert.Contains( + $"INTERVAL '{minute} minutes'", + TimescaleSupport.AddCompressionPolicySql(table), StringComparison.Ordinal); + Assert.DoesNotContain("seconds", TimescaleSupport.AddCompressionPolicySql(table), StringComparison.Ordinal); + } + } +} diff --git a/Darling/Darling.Tests/CpuRankingLiveTests.cs b/Darling/Darling.Tests/CpuRankingLiveTests.cs new file mode 100644 index 000000000..170f23265 --- /dev/null +++ b/Darling/Darling.Tests/CpuRankingLiveTests.cs @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Darling.Tests; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace PerformanceMonitor.Darling.Tests; + +/// +/// #3523: get_top_queries_by_cpu / get_top_procedures_by_cpu ranked by summed ELAPSED time. +/// +/// The defect. On a wait-bound server, elapsed and CPU disagree wildly — a query that sleeps on +/// locks for minutes ranks above one that burns a core — so the real CPU consumers could be absent from the +/// page entirely, while attributed_cpu_ratio (page CPU / measured process CPU) read as "hidden or +/// evicted CPU" when it actually meant "wrong sort key". Every CPU investigation starts at this tool. +/// +/// Why a LIVE test on top of the SQL-text pins. The queries reads have TWO ordering sites — the +/// ranking CTE's ORDER BY ... LIMIT top + 5 decides which groups SURVIVE at all, and the outer +/// post-WAITFOR-trim sort decides the returned order. A text pin restates each clause; only real rows through +/// real Postgres prove the cut. The seed makes the CPU king the WORST group by elapsed time with more +/// competing groups than the over-fetch admits, so under the old key it was not merely mis-sorted — it was +/// cut before the outer sort could see it. +/// +[Collection("live-postgres")] +public sealed class CpuRankingLiveTests +{ + private const string ServerName = "darling-cpu-ranking-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private const string Db = "waitbound"; + + [Fact] + public async Task ByCpuReads_RankAndCutByWorkerTime_WhenCpuAndElapsedDisagree_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live CPU-ranking test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await CleanupAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var succeeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var now = DarlingMcpTestData.Naive(DateTime.UtcNow); + + /* The wait-bound shape: one CPU king whose elapsed time is the SMALLEST on the box, and six + lock-sleepers whose elapsed dwarfs it while their CPU is noise. Seven groups against + top: 1 (over-fetch 6) means an elapsed-keyed CTE cuts the king before the outer sort. */ + await PlantQueryAsync(connection, ct, now.AddMinutes(-9), "0xCPUKING", + "SELECT CpuBurner FROM Numbers", cpuUs: 900_000L, elapsedUs: 1_000L); + for (var i = 1; i <= 6; i++) + { + await PlantQueryAsync(connection, ct, now.AddMinutes(-8), $"0xSLEEPER{i}", + $"SELECT Blocked{i} FROM Locked WHERE Id = {i}", cpuUs: 1_000L + i, elapsedUs: 800_000L + i * 10_000L); + } + + /* ---- the cut: top 1 must be the CPU king, in BOTH groupings (the rollup const has its own + copies of both ordering sites). All rows are ad-hoc, so the rollup grouping degenerates to + per-hash and exercises purely its ordering keys. */ + foreach (var rollUp in new[] { false, true }) + { + var top1 = await DarlingDataReader.GetTopQueriesByCpuAsync( + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 1, databaseName: null, + rollUpByHostObject: rollUp, cancellationToken: ct); + var king = Assert.Single(top1); + Assert.Equal("0xCPUKING", king.QueryHash); + } + + /* ---- the order: the full page comes back in descending CPU order, king first. */ + var page = await DarlingDataReader.GetTopQueriesByCpuAsync( + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 20, databaseName: null, + rollUpByHostObject: false, cancellationToken: ct); + Assert.Equal(7, page.Count); + Assert.Equal("0xCPUKING", page[0].QueryHash); + Assert.Equal(page.Select(r => r.TotalCpuUs).OrderByDescending(v => v), page.Select(r => r.TotalCpuUs)); + + /* ---- procedures: same disagreement, same promise (single ordering site, no over-fetch). */ + await PlantProcedureAsync(connection, ct, now.AddMinutes(-7), "usp_CpuHog", cpuUs: 900_000L, elapsedUs: 1_000L); + await PlantProcedureAsync(connection, ct, now.AddMinutes(-6), "usp_WaitBound", cpuUs: 5_000L, elapsedUs: 900_000L); + + var topProc = await DarlingDataReader.GetTopProceduresByCpuAsync( + postgres, ServerId, now.AddHours(-1), now.AddMinutes(5), top: 1, databaseName: null, cancellationToken: ct); + Assert.Equal("usp_CpuHog", Assert.Single(topProc).ObjectName); + + succeeded = true; + } + finally + { + /* #1902: teardown on its OWN connection, never the body's — see HostObjectRollupLiveTests. */ + await LiveStoreCleanup.RunAsync(connectionString!, succeeded, async (cleanup, cleanupCt) => + await CleanupAsync(cleanup, cleanupCt)); + } + } + + private static async Task PlantQueryAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, + string queryHash, string queryText, long cpuUs, long elapsedUs) + { + var sqlHandle = "0xSQLH" + Convert.ToHexString( + System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(queryText)))[..12]; + var digest = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(queryText)); + + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, + query_hash, query_plan_hash, sql_handle, plan_handle, query_text, + query_text_digest, delta_execution_count, delta_worker_time, + delta_elapsed_time, delta_logical_reads, min_dop, max_dop) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)", + CollectionIdGenerator.Next(), at, ServerId, ServerName, Db, + queryHash, "0xPLANHASH", sqlHandle, "0xPLANH", queryText, + digest, 10L, cpuUs, elapsedUs, 100L, 1, 1); + } + + private static async Task PlantProcedureAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, + string objectName, long cpuUs, long elapsedUs) => + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO procedure_stats (collection_id, collection_time, server_id, server_name, database_name, + schema_name, object_name, object_type, delta_execution_count, + delta_worker_time, delta_elapsed_time) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", + CollectionIdGenerator.Next(), at, ServerId, ServerName, Db, + "dbo", objectName, "SQL_STORED_PROCEDURE", 10L, cpuUs, elapsedUs); + + private static async Task CleanupAsync(NpgsqlConnection connection, CancellationToken ct) => + await DarlingMcpTestData.ExecAsync(connection, ct, + $"DELETE FROM query_stats WHERE server_id = {ServerId}; DELETE FROM procedure_stats WHERE server_id = {ServerId}; DELETE FROM servers WHERE server_id = {ServerId}"); +} diff --git a/Darling/Darling.Tests/CustomAlertResolveGrantLiveTests.cs b/Darling/Darling.Tests/CustomAlertResolveGrantLiveTests.cs index 7a92afa91..f06034a24 100644 --- a/Darling/Darling.Tests/CustomAlertResolveGrantLiveTests.cs +++ b/Darling/Darling.Tests/CustomAlertResolveGrantLiveTests.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Npgsql; using PerformanceMonitor.Alerting; +using PerformanceMonitor.Notifications; using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Storage; using Xunit; @@ -41,6 +42,15 @@ public sealed class CustomAlertResolveGrantLiveTests private sealed class NoopDeliverer : IAlertDeliverer { public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) => Task.CompletedTask; + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private static string RequireLivePostgres() diff --git a/Darling/Darling.Tests/CustomAlertSeverityEscalationLiveTests.cs b/Darling/Darling.Tests/CustomAlertSeverityEscalationLiveTests.cs index f1289030e..d5b3e8ee5 100644 --- a/Darling/Darling.Tests/CustomAlertSeverityEscalationLiveTests.cs +++ b/Darling/Darling.Tests/CustomAlertSeverityEscalationLiveTests.cs @@ -45,6 +45,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private static string RequireLivePostgres() diff --git a/Darling/Darling.Tests/CustomAlertTeardownLiveTests.cs b/Darling/Darling.Tests/CustomAlertTeardownLiveTests.cs index 7edeaae99..5d38eff04 100644 --- a/Darling/Darling.Tests/CustomAlertTeardownLiveTests.cs +++ b/Darling/Darling.Tests/CustomAlertTeardownLiveTests.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Npgsql; using PerformanceMonitor.Alerting; +using PerformanceMonitor.Notifications; using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Storage; using Xunit; @@ -35,6 +36,15 @@ public sealed class CustomAlertTeardownLiveTests private sealed class NoopDeliverer : IAlertDeliverer { public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) => Task.CompletedTask; + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private static string RequireLivePostgres() diff --git a/Darling/Darling.Tests/DailyDeadlockWindowCensusTests.cs b/Darling/Darling.Tests/DailyDeadlockWindowCensusTests.cs new file mode 100644 index 000000000..228ebdd33 --- /dev/null +++ b/Darling/Darling.Tests/DailyDeadlockWindowCensusTests.cs @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3525: the shared daily classifier bands deadlocks as a RATE over +/// — so every PRODUCTION signals bundle has to declare the window its counts cover, or the band silently +/// takes the unrateable arm and a genuinely storming day maxes out at Warning. This is the +/// DeadlockRateBandRungTests.EveryProductionMetricBundleDeclaresTheWindowAndTheTiers census, one +/// signals type over: same walker, same brace-matching, same whole-repo scope. +/// +public sealed class DailyDeadlockWindowCensusTests +{ + /// + /// Every production bundle that assigns Deadlocks also assigns + /// Window — and, since #3539 A2, CollectionRuns and PeakBlockWaitMs. + /// + /// Keyed on Deadlocks AND HasData assigned in the same initializer — the + /// rung census's own lesson about member names shared across types, met the way it met it: two other + /// production types (InactionFigures, the PostgreSQL DatabaseRow) carry a + /// Deadlocks member, and neither has a HasData, while a signals bundle without + /// HasData is unbuildable in practice (it would band NoData unconditionally). A type-name + /// scan is not an option for the reason the rung census states: three of the four sites are + /// target-typed => new() { ... }. Prefixed members like TotalDeadlocks are excluded + /// by the leading character class. Tests are excluded because a fixture deliberately omitting the + /// window IS the unrateable-arm pin; deprecated/ because the old Dashboard froze its own + /// banding. + /// + [Fact] + public void EveryProductionDailySignalsBundle_DeclaresTheWindow() + { + var offenders = new List(); + var found = 0; + + foreach (var file in ProductionCSharpFiles()) + { + var code = CSharpSourceWalker.StripCommentsAndStrings(System.IO.File.ReadAllText(file)); + + foreach (var initializer in SignalsBundleInitializers(code)) + { + found++; + + /* #3539 A2 widened the census to the two members that landed beside the window: the + collection-run denominator (without it the error share cannot form and every erroring + window is Warning on presence) and the peak block (without it the day's blocking band has + no wait arm and a 60-second block cannot redden a cell). */ + if (!AssignsMember(initializer, "Window") + || !AssignsMember(initializer, "CollectionRuns") + || !AssignsMember(initializer, "PeakBlockWaitMs")) + { + offenders.Add(System.IO.Path.GetFileName(file)); + } + } + } + + /* The scan has to have FOUND the bundles, or "no offenders" is vacuous. Four production sites: + DarlingHealthReader's ToSignals (the calendar/MCP day), the viewer's DailySummaryRow.ToSignals, + Lite's DailySummaryRow.ToSignals, and the fleet sweep's span read. Pinned as an exact count — a + FIFTH bundle is a new surface that has to be looked at, and a floor would let it in silently, + while a count below four means the regex stopped matching, not that the code got better. */ + Assert.Equal(4, found); + + Assert.True( + offenders.Count == 0, + "these production DailyHealthSignals bundles omit a denominator or the peak block (Window, " + + "CollectionRuns or PeakBlockWaitMs), so a rate or share cannot form and the band falls to its " + + "presence arm: " + + string.Join(", ", offenders.Distinct().OrderBy(f => f, StringComparer.Ordinal))); + } + + /// + /// The three CALENDAR-DAY projections declare the day's window through the ONE clamp helper — stated + /// against the source because the census above can only see that A window was assigned, and a calendar + /// day whose denominator drifted would band the same count differently across the three surfaces that + /// answer the same question. The helper (not a bare 24h constant) is the pin because the still-forming + /// day must clamp to its elapsed portion (#3525 review: an active storm banded over unelapsed hours + /// reads Healthy mid-crisis), and a projection that hand-rolls FromDays(1) reintroduces that dilution. + /// + [Fact] + public void EveryCalendarDayProjection_BandsThroughTheDayWindowClamp() + { + var surfaces = new (string What, string Text)[] + { + ("service daily read", RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingHealthReader.cs")), + ("viewer calendar row", RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.DailySummary.cs")), + ("Lite calendar row", RepoFile.ReadRepoFile( + "Lite", "Services", "LocalDataService.DailySummary.cs")), + }; + + foreach (var (what, text) in surfaces) + { + Assert.False(string.IsNullOrWhiteSpace(text), $"{what}: read nothing, so this pin would assert nothing"); + Assert.Contains( + "Window = DailyHealthBandCalculator.CalendarDayWindow(SummaryDate, ReferenceUtc)", + text, StringComparison.Ordinal); + Assert.DoesNotContain("Window = TimeSpan.FromDays(1)", text, StringComparison.Ordinal); + } + } + + /// + /// The fleet sweep's window is its OWN span, never a calendar day — the whole #3525 point for the + /// sweep was that a 15-minute span and a 24-hour day must not band the same count the same way. + /// + [Fact] + public void TheSweepWindow_IsTheSpan_NotACalendarDay() + { + var sweep = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "FleetSweepEngine.cs"); + + Assert.Contains("Window = spanEndUtc - spanStartUtc", sweep, StringComparison.Ordinal); + Assert.DoesNotContain("Window = TimeSpan.FromDays(1)", sweep, StringComparison.Ordinal); + Assert.DoesNotContain("CalendarDayWindow", sweep, StringComparison.Ordinal); + } + + /* ─────────────────────── helpers (the rung census's own, re-keyed) ─────────────────────── */ + + private static readonly Regex AssignsDeadlocks = + new(@"(?)", RegexOptions.Compiled); + + private static IEnumerable ProductionCSharpFiles() + { + foreach (var file in System.IO.Directory.EnumerateFiles( + RepoFile.Root, "*.cs", System.IO.SearchOption.AllDirectories)) + { + var segments = file.Split(System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar); + if (segments.Contains("bin") || segments.Contains("obj") + || segments.Contains("deprecated") + || segments.Any(s => s.EndsWith("Tests", StringComparison.Ordinal))) + { + continue; + } + + yield return file; + } + } + + private static IEnumerable SignalsBundleInitializers(string code) + { + foreach (var initializer in BraceMatchedInitializers(code)) + { + if (AssignsMember(initializer, "HasData")) + { + yield return initializer; + } + } + } + + private static IEnumerable BraceMatchedInitializers(string code) + { + foreach (var match in AssignsDeadlocks.Matches(code).Cast()) + { + var depth = 0; + var open = -1; + + for (var i = match.Index; i >= 0; i--) + { + if (code[i] == '}') + { + depth++; + } + else if (code[i] == '{' && depth-- == 0) + { + open = i; + break; + } + } + + if (open < 0) + { + continue; + } + + depth = 0; + for (var i = open; i < code.Length; i++) + { + if (code[i] == '{') + { + depth++; + } + else if (code[i] == '}' && --depth == 0) + { + yield return code[open..(i + 1)]; + break; + } + } + } + } + + private static bool AssignsMember(string initializer, string member) => + Regex.IsMatch(initializer, $@"(?)"); +} diff --git a/Darling/Darling.Tests/DailyHealthBandTests.cs b/Darling/Darling.Tests/DailyHealthBandTests.cs index 26dc263d9..77a9d11a8 100644 --- a/Darling/Darling.Tests/DailyHealthBandTests.cs +++ b/Darling/Darling.Tests/DailyHealthBandTests.cs @@ -20,18 +20,25 @@ namespace Darling.Tests; /// public class DailyHealthBandTests { + private static readonly TimeSpan Day = TimeSpan.FromHours(24); + private static readonly TimeSpan Hour = TimeSpan.FromHours(1); + private static DailyHealthSignals Signals( bool hasData = true, long deadlocks = 0, long collectionErrors = 0, long highCpu = 0, - long blocking = 0, long memPressure = 0, long memCritical = 0, long alerts = 0) => new() + long blocking = 0, long memPressure = 0, long memCritical = 0, long alerts = 0, + TimeSpan window = default, long collectionRuns = 0, long peakBlockMs = 0) => new() { HasData = hasData, Deadlocks = deadlocks, CollectionErrors = collectionErrors, + CollectionRuns = collectionRuns, HighCpuEvents = highCpu, BlockingEvents = blocking, + PeakBlockWaitMs = peakBlockMs, MemoryPressureEvents = memPressure, MemoryCriticalEvents = memCritical, AlertCount = alerts, + Window = window, }; [Fact] @@ -47,58 +54,334 @@ public void Collected_AndNothingElevated_IsHealthy() Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(Signals())); } + /// + /// Each remaining Critical trigger alone, over a finished 24-hour day (#3539 A2 made every count + /// trigger window-aware, so the window is declared): severe memory pressure on presence; sustained + /// CPU at the day-scale bar of 30 hot samples; a blocking RATE at the 20/hr tier (480 over the day); + /// a single 60-second block whatever the count. Collection errors are no longer a Critical trigger at + /// any count — see the collection-error pins below. + /// [Theory] - [InlineData("deadlock", 1, 0, 0, 0, 0, 0)] - [InlineData("collection-error", 0, 1, 0, 0, 0, 0)] - [InlineData("memory-critical", 0, 0, 0, 0, 1, 0)] - [InlineData("sustained-cpu", 0, 0, 6, 0, 0, 0)] - [InlineData("heavy-blocking", 0, 0, 0, 11, 0, 0)] - public void CriticalTriggers_EachAloneIsCritical(string _, long deadlocks, long collErrors, long highCpu, long blocking, long memCritical, long alerts) - { - var s = Signals(deadlocks: deadlocks, collectionErrors: collErrors, highCpu: highCpu, blocking: blocking, memCritical: memCritical, alerts: alerts); + [InlineData("memory-critical", 0, 0, 0L, 1)] + [InlineData("sustained-cpu", 30, 0, 0L, 0)] + [InlineData("blocking-rate", 0, 480, 0L, 0)] + [InlineData("sixty-second-block", 0, 1, 60_000L, 0)] + public void CriticalTriggers_EachAloneIsCritical(string _, long highCpu, long blocking, long peakBlockMs, long memCritical) + { + var s = Signals(highCpu: highCpu, blocking: blocking, peakBlockMs: peakBlockMs, memCritical: memCritical, window: Day); Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(s)); } + /* ── the deadlock RATE trigger (#3525 — #3368's twin, routed through the card band's tiers) ── */ + + [Fact] + public void ACriticalDeadlockRate_AloneIsCritical() + { + // 480 over 24h = 20.0/hr, the Critical tier — the same pair the Overview card's dot bands on. + Assert.Equal( + DailyHealthBand.Critical, + DailyHealthBandCalculator.Classify(Signals(deadlocks: 480, window: Day))); + } + + /// + /// The defect #3525 was filed on: one deadlock in a 24-hour day banded the WHOLE day Critical — the + /// count trigger #3368 removed from the card, still shipping in this classifier. Measured on the same + /// 43-server fleet, count > 0 read 87.9% of 24-hour windows Critical, so ~7 of 8 calendar cells + /// painted red from deadlocks alone. At 0.04/hr the day is Healthy; the deadlock stays countable — the + /// tooltip lists it and the drill is offered — the BAND just stops claiming a crisis. + /// + [Fact] + public void ASingleDeadlockInADay_IsNoLongerCritical() + { + var s = Signals(deadlocks: 1, window: Day); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(s)); + + Assert.Contains(DayDrillTarget.Deadlocks, DailyHealthBandCalculator.AvailableDrills(s)); + Assert.Contains("1 deadlock", DailyHealthBandCalculator.Describe(s)); + } + + /// + /// The two-window proof, on the DAY classifier — the same per-hour rate bands the day identically over + /// a 1-hour window (a fleet-sweep span at the cadence ceiling's scale) and a 24-hour one (a calendar + /// day). Counts are integer-rate-times-whole-hours so the asserted rate is exactly the one the band + /// sees (the DeadlockRateBandTests discipline). + /// [Theory] - [InlineData("moderate-cpu", 3, 0, 0, 0)] - [InlineData("some-blocking", 0, 5, 0, 0)] + [InlineData(4, DailyHealthBand.Healthy)] + [InlineData(5, DailyHealthBand.Warning)] + [InlineData(19, DailyHealthBand.Warning)] + [InlineData(20, DailyHealthBand.Critical)] + [InlineData(50, DailyHealthBand.Critical)] + public void TheSameDeadlockRate_BandsTheDayTheSame_OverAnHourAndADay(long ratePerHour, DailyHealthBand expected) + { + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(deadlocks: ratePerHour, window: Hour))); + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(deadlocks: ratePerHour * 24, window: Day))); + } + + /// + /// And the discriminating converse: the same COUNT over the two windows bands differently, which a + /// count trigger cannot do at all — the pin that goes red on any revert to counting. + /// + [Fact] + public void TheSameDeadlockCount_OverTwoWindows_BandsDifferently() + { + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(deadlocks: 30, window: Hour))); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(Signals(deadlocks: 30, window: Day))); + } + + /// + /// A sub-hour window — a fleet sweep at any cadence under an hour — is not rate-banded (#3368's own + /// arm): a non-zero count reads Warning (deadlocks demonstrably happened; no rate supports Critical, + /// and 1 deadlock in 15 minutes is 4/hr arithmetically but the hour was not observed), and a zero + /// count stays out of the deadlock trigger entirely rather than claiming anything. An undeclared + /// window — default(TimeSpan), a producer that declared nothing — takes the same arm, so no + /// path can rate-multiply or restore count-is-Critical by omission. + /// + [Fact] + public void ASubHourOrUndeclaredWindow_FallsToWarning_NeverCritical() + { + foreach (var window in new[] { default, TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(59) }) + { + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(deadlocks: 1, window: window))); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(deadlocks: 10_000, window: window))); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(Signals(deadlocks: 0, window: window))); + } + } + + /// + /// The day bands on the tiers it is handed — the store-backed pair (#3368, V120) travels through + /// , so a Darling store with raised tiers moves the + /// calendar, get_daily_summary and the sweep together with the card. + /// + [Fact] + public void TheDeadlockRateTiers_AreOverridable() + { + var raised = new DailyHealthThresholds { DeadlockRates = new DeadlockRateThresholds(100.0, 500.0) }; + var s = Signals(deadlocks: 480, window: Day); // 20/hr: Critical on the shipped pair + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(s)); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(s, raised)); + } + + /// Each Warning trigger alone over a finished day: six hot samples (the day-scale Warning bar), + /// blocking at the 5/hr tier (120 over the day), non-severe memory pressure, an alert. + [Theory] + [InlineData("moderate-cpu", 6, 0, 0, 0)] + [InlineData("blocking-rate", 0, 120, 0, 0)] [InlineData("memory-pressure", 0, 0, 1, 0)] [InlineData("alert", 0, 0, 0, 1)] public void WarningTriggers_EachAloneIsWarning(string _, long highCpu, long blocking, long memPressure, long alerts) { - var s = Signals(highCpu: highCpu, blocking: blocking, memPressure: memPressure, alerts: alerts); + var s = Signals(highCpu: highCpu, blocking: blocking, memPressure: memPressure, alerts: alerts, window: Day); Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(s)); } [Fact] public void CriticalBeatsWarning_WhenBothPresent() { - var s = Signals(deadlocks: 1, highCpu: 3, alerts: 4); + // A critical deadlock rate (480/24h = 20/hr) plus moderate CPU + alerts (warning) still bands Critical. + var s = Signals(deadlocks: 480, highCpu: 3, alerts: 4, window: Day); Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(s)); } + /* ── the high-CPU trigger (#3539 A2): a bar that scales with the window ── */ + + /// + /// Over an HOUR the pre-#3539 constants hold exactly: one hot sample is Warning, six is Critical — the + /// excursion-scale minimum dominates below 4.8 hours, so every fleet-sweep span at the default cadence + /// bands as it always did. + /// [Theory] + [InlineData(0, DailyHealthBand.Healthy)] + [InlineData(1, DailyHealthBand.Warning)] [InlineData(5, DailyHealthBand.Warning)] [InlineData(6, DailyHealthBand.Critical)] - public void HighCpu_CriticalThreshold_IsSixSamples(long highCpu, DailyHealthBand expected) + public void HighCpu_OverAnHour_BandsOnTheExcursionScaleMinimum(long highCpu, DailyHealthBand expected) { + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(highCpu: highCpu, window: Hour))); + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(highCpu: highCpu, window: TimeSpan.FromMinutes(15)))); + /* An undeclared window yields the same minimum — a producer that declares nothing bands as it did. */ Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(highCpu: highCpu))); } + /// + /// Over a DAY the rate decides: six hot samples — which reddened 5.9% of the measured server-days — is + /// now the Warning bar, and Critical needs thirty, the sustained-heat count 1.4% of server-days reach. + /// The measured routine excursion is 1–3 samples (95.8% of 758 excursions), so two of them in a day + /// (six) is Warning and one (up to five) is Healthy, where 19.2% of days used to turn amber on a single + /// 80%+ sample. + /// [Theory] - [InlineData(10, DailyHealthBand.Warning)] - [InlineData(11, DailyHealthBand.Critical)] - public void Blocking_CriticalThreshold_IsElevenEvents(long blocking, DailyHealthBand expected) + [InlineData(0, DailyHealthBand.Healthy)] + [InlineData(5, DailyHealthBand.Healthy)] + [InlineData(6, DailyHealthBand.Warning)] + [InlineData(29, DailyHealthBand.Warning)] + [InlineData(30, DailyHealthBand.Critical)] + [InlineData(155, DailyHealthBand.Critical)] // the worst measured server-day + public void HighCpu_OverADay_BandsOnTheSustainedHeatRate(long highCpu, DailyHealthBand expected) => + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(highCpu: highCpu, window: Day))); + + /// The bar is the GREATER of the minimum and the rate: 6 until 4.8 hours, then 1.25 × hours. + /// Stated at the seam so the shape — not just its two endpoints — is pinned. + [Fact] + public void HighCpu_TheBar_IsTheGreaterOfMinimumAndRate() { - Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(blocking: blocking))); + var t = DailyHealthThresholds.Default; + Assert.Equal(6.0, t.HighCpuCriticalSamplesFor(TimeSpan.Zero)); + Assert.Equal(6.0, t.HighCpuCriticalSamplesFor(TimeSpan.FromMinutes(15))); + Assert.Equal(6.0, t.HighCpuCriticalSamplesFor(Hour)); + Assert.Equal(6.0, t.HighCpuCriticalSamplesFor(TimeSpan.FromHours(4.8))); + Assert.Equal(15.0, t.HighCpuCriticalSamplesFor(TimeSpan.FromHours(12))); + Assert.Equal(30.0, t.HighCpuCriticalSamplesFor(Day)); + + Assert.Equal(1.0, t.HighCpuWarningSamplesFor(Hour)); + Assert.Equal(6.0, t.HighCpuWarningSamplesFor(Day)); + + /* The two constants each carry their measured lineage: 1.25/hr is 30 per day; the minimum is six, + the 97.8th percentile of excursion length. */ + Assert.Equal(30.0, t.HighCpuCriticalSamplesPerHour * 24); + Assert.Equal(6, t.HighCpuCriticalSamplesMinimum); + } + + /// Two routine 3-sample excursions in a day used to redden the cell (#3282's finding); they + /// are Warning now, and the same six in one hour — a sustained excursion for that span — stays Critical. + [Fact] + public void TwoRoutineExcursions_InADay_AreNoLongerCritical() + { + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(highCpu: 6, window: Day))); + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(highCpu: 6, window: Hour))); + } + + /* ── the blocking trigger (#3539 A2/A3): the card band's own rate and wait arms ── */ + + /// + /// The same per-hour rate bands the day identically over an hour and a day, through the card band's + /// tiers — 5/hr Warning, 20/hr Critical. Counts are rate × whole hours so the asserted rate is exactly + /// the one the band sees. + /// + [Theory] + [InlineData(4, DailyHealthBand.Healthy)] + [InlineData(5, DailyHealthBand.Warning)] + [InlineData(19, DailyHealthBand.Warning)] + [InlineData(20, DailyHealthBand.Critical)] + public void TheSameBlockingRate_BandsTheDayTheSame_OverAnHourAndADay(long ratePerHour, DailyHealthBand expected) + { + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(blocking: ratePerHour, window: Hour))); + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(blocking: ratePerHour * 24, window: Day))); + } + + /// + /// The constant this replaced: eleven blocking events reddened 5.1% of measured server-days. Eleven in a + /// day is 0.46/hr and Healthy by count; eleven in an hour is inside the measured trough and Warning; + /// the same count over two windows bands differently, which the count trigger could not do. + /// + [Fact] + public void ElevenBlockingEventsInADay_IsNoLongerCritical() + { + var day = Signals(blocking: 11, window: Day); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(day)); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(blocking: 11, window: Hour))); + + /* Still countable: the drill is offered and the tooltip lists it with its rate. */ + Assert.Contains(DayDrillTarget.Blocking, DailyHealthBandCalculator.AvailableDrills(day)); + Assert.Contains("11 blocking events (0.5/hr)", DailyHealthBandCalculator.Describe(day)); + } + + /// The day's peak block is the wait arm: a 60-second block is a Critical day and a 10-second + /// one a Warning day whatever the count or the window — the card band's rate-independent arms. + [Fact] + public void ThePeakBlock_IsTheDaysWaitArm() + { + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(blocking: 1, peakBlockMs: 60_000, window: Day))); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(blocking: 1, peakBlockMs: 10_000, window: Day))); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(Signals(blocking: 1, peakBlockMs: 9_999, window: Day))); + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(blocking: 1, peakBlockMs: 60_000, window: TimeSpan.FromMinutes(15)))); + } + + /// A sub-hour or undeclared window is not rate-banded on blocking either: a non-zero count reads + /// Warning, never Critical by count, and a zero count stays out of the trigger. + [Fact] + public void ASubHourOrUndeclaredWindow_BandsBlockingWarning_NeverCriticalByCount() + { + foreach (var window in new[] { default, TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(59) }) + { + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(blocking: 1, window: window))); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(blocking: 10_000, window: window))); + Assert.Equal(DailyHealthBand.Healthy, DailyHealthBandCalculator.Classify(Signals(blocking: 0, window: window))); + } + } + + /* ── the collection-error trigger (#3539 A2): a share of runs, Warning ceiling ── */ + + /// + /// The rule this replaced was CollectionErrors > 0 → Critical; one transient ERROR row painted + /// a day red (#1805 was one). The errors now band as a share of the window's runs against the + /// collector-health classifier's own 20% bar, at ITS tier: past the bar the day is Warning, below it + /// the errors are disclosed but do not band, and nothing here can make a day Critical. + /// + [Theory] + [InlineData(1, 30_000, DailyHealthBand.Healthy)] // one transient error in a day's ~30k runs + [InlineData(6_000, 30_000, DailyHealthBand.Healthy)] // exactly 20% — the bar is strict, as the classifier's is + [InlineData(6_001, 30_000, DailyHealthBand.Warning)] + [InlineData(30_000, 30_000, DailyHealthBand.Warning)] // every run erroring is still Warning here, never Critical + [InlineData(0, 0, DailyHealthBand.Healthy)] + public void CollectionErrors_BandOnTheirShareOfRuns_WarningAtMost(long errors, long runs, DailyHealthBand expected) => + Assert.Equal(expected, DailyHealthBandCalculator.Classify(Signals(collectionErrors: errors, collectionRuns: runs, window: Day))); + + /// With no denominator declared a non-zero error count fails away from Healthy into Warning — + /// never Critical — and the tooltip prints the count without a share it could not compute. + [Fact] + public void CollectionErrors_WithNoDeclaredRuns_AreWarning_NeverCritical() + { + var s = Signals(collectionErrors: 1, window: Day); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(s)); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(collectionErrors: 10_000, window: Day))); + Assert.Contains("1 collection error", DailyHealthBandCalculator.Describe(s)); + Assert.DoesNotContain("% of", DailyHealthBandCalculator.Describe(s)); + } + + [Fact] + public void CollectionErrorLine_CarriesTheShare_WhenRunsAreDeclared() + { + var s = Signals(collectionErrors: 12, collectionRuns: 9_800, window: Day); + Assert.Contains("12 collection errors (0.1% of 9,800 runs)", DailyHealthBandCalculator.Describe(s)); + Assert.Contains("12 collection errors (0.1% of 9,800 runs)", DailyHealthBandCalculator.BuildReasons(s)); + Assert.Equal(HealthSeverity.Healthy, DailyHealthBandCalculator.CollectionErrorSeverity(12, 9_800)); + Assert.Equal(20.0, CollectorHealthClassifier.WarningFailureRatePercent); + } + + /// The 15-minute sweep span and the 24-hour day agree on identical behaviour: the same per-hour + /// blocking rate, the same error share, the same excursion-scale CPU count below the seam. + [Fact] + public void TheSweepSpan_AndTheDay_AgreeOnIdenticalBehaviour() + { + var quarterHour = TimeSpan.FromMinutes(15); + + /* Blocking: sub-hour is the unrateable arm, so the comparison that CAN be made is the hour vs the + day at one rate — pinned above — and the wait arm, which is identical at every span. */ + Assert.Equal( + DailyHealthBandCalculator.Classify(Signals(blocking: 3, peakBlockMs: 60_000, window: quarterHour)), + DailyHealthBandCalculator.Classify(Signals(blocking: 288, peakBlockMs: 60_000, window: Day))); + + /* Collection errors: a share is a share. */ + Assert.Equal( + DailyHealthBandCalculator.Classify(Signals(collectionErrors: 30, collectionRuns: 100, window: quarterHour)), + DailyHealthBandCalculator.Classify(Signals(collectionErrors: 3_000, collectionRuns: 10_000, window: Day))); + Assert.Equal( + DailyHealthBand.Warning, + DailyHealthBandCalculator.Classify(Signals(collectionErrors: 30, collectionRuns: 100, window: quarterHour))); + + /* CPU: the minimum decides at both 15 minutes and an hour — the sweep floor and the default cadence. */ + Assert.Equal( + DailyHealthBandCalculator.Classify(Signals(highCpu: 6, window: quarterHour)), + DailyHealthBandCalculator.Classify(Signals(highCpu: 6, window: Hour))); } [Fact] public void Thresholds_AreOverridable() { - var strict = new DailyHealthThresholds { HighCpuCriticalSamples = 3 }; - Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(highCpu: 3), strict)); - Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(highCpu: 3))); + var strict = new DailyHealthThresholds { HighCpuCriticalSamplesMinimum = 3 }; + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(Signals(highCpu: 3, window: Hour), strict)); + Assert.Equal(DailyHealthBand.Warning, DailyHealthBandCalculator.Classify(Signals(highCpu: 3, window: Hour))); } [Theory] @@ -175,6 +458,33 @@ public void BuildReasons_BlockingLine_OmitsPeak_WhenZero() Assert.DoesNotContain(reasons, r => r.Contains("peak block", StringComparison.Ordinal)); } + /// + /// The deadlock line carries the per-hour rate the band evaluated (#3525) — the card reason's own + /// disclosure rule: "120 deadlocks" against an amber cell cannot say which tier was crossed, because + /// 120 in an hour and 120 in a day are the same string. The count stays (the countable fact); the rate + /// is added (the banded one). Shared by the tooltip and the day-detail reasons, so both surfaces say it. + /// + [Fact] + public void DeadlockLine_CarriesTheRate_WhenTheWindowIsRateable() + { + var day = Signals(deadlocks: 120, window: Day); // 5.0/hr — the Warning tier exactly + Assert.Contains("120 deadlocks (5.0/hr)", DailyHealthBandCalculator.Describe(day)); + Assert.Contains("120 deadlocks (5.0/hr)", DailyHealthBandCalculator.BuildReasons(day)); + + // A single deadlock still reads singular, rate beside it. + Assert.Contains("1 deadlock (0.0/hr)", DailyHealthBandCalculator.Describe(Signals(deadlocks: 1, window: Day))); + } + + [Fact] + public void DeadlockLine_PrintsTheCountAlone_OnAnUnrateableWindow() + { + // No declared window: no rate is computable, and printing one would claim a measurement nobody + // took — the count alone is exactly what the band had to go on. + var described = DailyHealthBandCalculator.Describe(Signals(deadlocks: 2)); + Assert.Contains("2 deadlocks", described); + Assert.DoesNotContain("/hr", described); + } + [Fact] public void AvailableDrills_NoData_OffersNothing() { @@ -227,4 +537,155 @@ public void BuildKeyMetricsLine_NoWait_ShowsNone_And_LargeWaitInMinutes() Assert.Contains("Top wait: none", line); Assert.Contains("Total wait: 20.0 min", line); // 1200s / 60 } + + /* ─────────── #3525 review: the still-forming day clamps to its elapsed portion ─────────── */ + + [Fact] + public void CalendarDayWindow_FinishedDay_IsTwentyFourHours() + { + var day = new DateTime(2026, 7, 8); + Assert.Equal(TimeSpan.FromDays(1), DailyHealthBandCalculator.CalendarDayWindow(day, new DateTime(2026, 7, 9))); + Assert.Equal(TimeSpan.FromDays(1), DailyHealthBandCalculator.CalendarDayWindow(day, new DateTime(2026, 9, 1, 12, 0, 0))); + } + + [Fact] + public void CalendarDayWindow_TodayClampsToElapsed_AndFutureIsZero() + { + var day = new DateTime(2026, 7, 8); + Assert.Equal(TimeSpan.FromHours(1), DailyHealthBandCalculator.CalendarDayWindow(day, day.AddHours(1))); + Assert.Equal(TimeSpan.FromMinutes(30), DailyHealthBandCalculator.CalendarDayWindow(day, day.AddMinutes(30))); + Assert.Equal(TimeSpan.Zero, DailyHealthBandCalculator.CalendarDayWindow(day, day.AddDays(-1))); + } + + [Fact] + public void TodayCell_ActiveStorm_IsNotDilutedByUnelapsedHours() + { + /* The review's own numbers: 60 deadlocks in the first hour of the still-forming day. Banded over + a full 24h the rate reads 2.5/hr (below the 5/hr Warning tier) and the crisis paints Healthy; + over the elapsed hour it is 60/hr — past the 20/hr Critical tier. The clamp is what keeps an + in-progress storm red on the calendar. The same 60 over a genuinely FINISHED day is honestly + 2.5/hr, and stays sub-Warning by design. */ + var day = new DateTime(2026, 7, 8); + var stormWindow = DailyHealthBandCalculator.CalendarDayWindow(day, day.AddHours(1)); + Assert.Equal( + DailyHealthBand.Critical, + DailyHealthBandCalculator.Classify(Signals(deadlocks: 60, window: stormWindow))); + + var finishedWindow = DailyHealthBandCalculator.CalendarDayWindow(day, day.AddDays(2)); + Assert.Equal( + DailyHealthBand.Healthy, + DailyHealthBandCalculator.Classify(Signals(deadlocks: 60, window: finishedWindow))); + } + + [Fact] + public void TodayCell_MinutesOld_FallsToTheUnrateableArm() + { + /* Sub-hour elapsed lands in DeadlockSeverity's unrateable arm (#3368's Warning-not-rate rule): + minutes into the day a single deadlock reads Warning, never a fabricated multiplied rate. */ + var window = DailyHealthBandCalculator.CalendarDayWindow( + new DateTime(2026, 7, 8), new DateTime(2026, 7, 8, 0, 10, 0)); + Assert.Equal( + DailyHealthBand.Warning, + DailyHealthBandCalculator.Classify(Signals(deadlocks: 1, window: window))); + } +} + + +/// +/// #3541 A9: the one decision both SKUs' daily-summary readers make about a returned day — is it a +/// measurement, or the shape retention left behind — pinned identically here and in the twin project. +/// +/// The defect: the daily aggregate's spine is a UNION over nine sources aging out at different +/// horizons, each COALESCEd to zero, so a day between the shortest horizon (the signals' 30 days) and the +/// longest (the alert log's 90) kept its spine row while every signal the band reads was gone — and zeros +/// band Healthy. The state is decided from two inputs, the day and the horizon; the run count only decides +/// between the two INSIDE-retention states. The horizon test comes first, because runs > 0 alone was +/// half the fix and called the whole second month collected. +/// +public class DailySummaryRetentionTests +{ + private static readonly DateTime Horizon = new(2026, 8, 19); + + [Theory] + [InlineData("2026-08-18", 1_440, 0, DailySummaryDataState.Purged)] /* the day before: a surviving run record does not rescue it */ + [InlineData("2026-08-18", 0, 0, DailySummaryDataState.Purged)] /* nor does the absence of one change the verdict */ + [InlineData("2026-07-01", 5, 0, DailySummaryDataState.Purged)] + [InlineData("2026-08-18", 1_440, 7, DailySummaryDataState.PastHorizon)] /* signal rows still there: the purge has not reached it */ + [InlineData("2026-08-18", 0, 1, DailySummaryDataState.PastHorizon)] /* even one source present withholds "purged" */ + [InlineData("2026-08-19", 1, 0, DailySummaryDataState.Collected)] /* the horizon day itself is held */ + [InlineData("2026-09-01", 1_440, 7, DailySummaryDataState.Collected)] + [InlineData("2026-09-01", 1_440, 0, DailySummaryDataState.Collected)] /* inside retention a quiet day needs no signal rows to be collected */ + [InlineData("2026-09-01", 0, 3, DailySummaryDataState.NoRunRecord)] /* inside retention, signals but no run recorded — a disclosure, the band stands */ + public void TheState_IsDecidedByTheHorizonAndPresenceFirst_ThenByTheRunCount(string day, long runs, int present, DailySummaryDataState expected) + { + var date = DateTime.ParseExact(day, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); + Assert.Equal(expected, DailySummaryRetention.StateFor(date, runs, present, Horizon)); + /* A time-of-day on either side changes nothing: the decision is on DATES. */ + Assert.Equal(expected, DailySummaryRetention.StateFor(date.AddHours(23), runs, present, Horizon.AddHours(5))); + } + + /// The horizon is the cutoff instant's DATE — see + /// for why that is exact on the TimescaleDB path and at most a partial day generous on the DELETE path. + [Fact] + public void TheHorizon_IsTheCutoffsDate() + { + var now = new DateTime(2026, 9, 18, 14, 30, 0, DateTimeKind.Utc); + Assert.Equal(new DateTime(2026, 8, 19), DailySummaryRetention.HorizonFor(now, 30)); + Assert.Equal(new DateTime(2026, 9, 17), DailySummaryRetention.HorizonFor(now, 1)); + Assert.Equal(DateTimeKind.Utc, DailySummaryRetention.HorizonFor(now, 30).Kind); + Assert.Throws(() => DailySummaryRetention.HorizonFor(now, 0)); + } + + [Fact] + public void TheVocabulary_IsOneWordPerState_AndTheNoteSaysWhatTheZerosAre() + { + Assert.Equal("collected", DailySummaryRetention.Label(DailySummaryDataState.Collected)); + Assert.Equal("purged", DailySummaryRetention.Label(DailySummaryDataState.Purged)); + Assert.Equal("past_horizon", DailySummaryRetention.Label(DailySummaryDataState.PastHorizon)); + Assert.Equal("no_run_record", DailySummaryRetention.Label(DailySummaryDataState.NoRunRecord)); + + Assert.Null(DailySummaryRetention.Note(DailySummaryDataState.Collected, Horizon)); + var purged = DailySummaryRetention.Note(DailySummaryDataState.Purged, Horizon)!; + Assert.StartsWith("PURGED", purged, StringComparison.Ordinal); + Assert.Contains("2026-08-19", purged, StringComparison.Ordinal); + Assert.Contains("absences, not measurements", purged, StringComparison.Ordinal); + var past = DailySummaryRetention.Note(DailySummaryDataState.PastHorizon, Horizon, 3)!; + Assert.StartsWith("PAST HORIZON", past, StringComparison.Ordinal); + Assert.Contains("3 of 7 signal sources", past, StringComparison.Ordinal); + Assert.Equal(7, DailySummaryRetention.SignalSourceCount); + var noRun = DailySummaryRetention.Note(DailySummaryDataState.NoRunRecord, Horizon)!; + Assert.StartsWith("NO RUN RECORD", noRun, StringComparison.Ordinal); + Assert.Contains("the band stands", noRun, StringComparison.Ordinal); + } + + /// The band's own contract, end to end: a signals projection with HasData folded from a + /// past-horizon state is No Data even under a Critical count — the calendar's grey, never green or red. + /// The two inside-retention states keep the band, because inside retention a zero is a measurement. + [Fact] + public void APastHorizonDay_BandsNoData_WhateverItsCounts_AndAnInsideRetentionDayKeepsItsBand() + { + foreach (var state in new[] { DailySummaryDataState.Collected, DailySummaryDataState.NoRunRecord }) + { + var signals = new DailyHealthSignals + { + HasData = state is not (DailySummaryDataState.Purged or DailySummaryDataState.PastHorizon), + Deadlocks = 480, + CollectionRuns = 1_440, + Window = TimeSpan.FromDays(1), + }; + Assert.Equal(DailyHealthBand.Critical, DailyHealthBandCalculator.Classify(signals)); + } + + foreach (var state in new[] { DailySummaryDataState.Purged, DailySummaryDataState.PastHorizon }) + { + var signals = new DailyHealthSignals + { + HasData = state is not (DailySummaryDataState.Purged or DailySummaryDataState.PastHorizon), + Deadlocks = 480, + CollectionRuns = 1_440, + Window = TimeSpan.FromDays(1), + }; + Assert.Equal(DailyHealthBand.NoData, DailyHealthBandCalculator.Classify(signals)); + } + } } diff --git a/Darling/Darling.Tests/DarlingAlertReadAdapterTests.cs b/Darling/Darling.Tests/DarlingAlertReadAdapterTests.cs index 14826770b..354bbcbc9 100644 --- a/Darling/Darling.Tests/DarlingAlertReadAdapterTests.cs +++ b/Darling/Darling.Tests/DarlingAlertReadAdapterTests.cs @@ -15,6 +15,7 @@ using PerformanceMonitor.Alerting; using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Notifications; using Xunit; namespace Darling.Tests; @@ -123,12 +124,11 @@ public void FeedQueries_PreserveLitesWindowsCapsAndFilters() Assert.Contains("ORDER BY deadlock_time DESC", DarlingAlertReadAdapter.DeadlocksSql); Assert.Contains("LIMIT 50", DarlingAlertReadAdapter.DeadlocksSql); - /* Poison waits: Lite's exact wait-type list, 3-row window, parameterized 10-minute floor. */ + /* Poison waits (#3539 A4): the same wait-type list, a parameterized window floor, and an + ACCUMULATION per wait type — see PoisonWaitsSql_IsAWindowAccumulation_NotTheNewestDeltas. */ Assert.Contains("'THREADPOOL', 'RESOURCE_SEMAPHORE', 'RESOURCE_SEMAPHORE_QUERY_COMPILE'", DarlingAlertReadAdapter.PoisonWaitsSql); - Assert.Contains("delta_waiting_tasks > 0", DarlingAlertReadAdapter.PoisonWaitsSql); Assert.Contains("collection_time >= $2", DarlingAlertReadAdapter.PoisonWaitsSql); - Assert.Contains("LIMIT 3", DarlingAlertReadAdapter.PoisonWaitsSql); /* Long-running queries: latest snapshot only, parameterized staleness floor ($4 — never now()), user sessions, parameterized cap, filter splice point. */ @@ -148,6 +148,38 @@ public void FeedQueries_PreserveLitesWindowsCapsAndFilters() Assert.Contains("LIMIT 5", DarlingAlertReadAdapter.AnomalousJobsSql); } + /// + /// #3539 A4: the poison read SUMs every row in the window per wait type. Three things the retired text + /// had must be ABSENT — the delta_waiting_tasks > 0 filter (a task waiting across the interval + /// boundary accrues time with zero completed tasks, and the measured fleet holds such rows), the + /// LIMIT 3 (a limit on a sum is an undercount) and any threshold — and the shape must be the sums, + /// the row count and the newest collection_time, grouped by wait type. The Lite twin's DuckDB text is + /// pinned to the same clauses in Lite.Tests so the two SKUs cannot drift. + /// + [Fact] + public void PoisonWaitsSql_IsAWindowAccumulation_NotTheNewestDeltas() + { + var sql = DarlingAlertReadAdapter.PoisonWaitsSql; + + Assert.DoesNotContain("delta_waiting_tasks > 0", sql); + Assert.DoesNotContain("LIMIT", sql); + Assert.DoesNotContain("avg_ms_per_wait", sql); + + Assert.Contains("SUM(delta_wait_time_ms)::bigint AS accumulated_wait_ms", sql); + Assert.Contains("SUM(delta_waiting_tasks)::bigint AS accumulated_waits", sql); + Assert.Contains("COUNT(*)::bigint AS observed_intervals", sql); + Assert.Contains("MAX(collection_time) AS newest_collection_time", sql); + Assert.Contains("GROUP BY wait_type", sql); + /* No interval handling: the V128 lane owns sample_interval_seconds and rebases onto this text. */ + Assert.DoesNotContain("sample_interval", sql); + + /* The read-side wait-type list IS the evaluator's, spelled once each and equal. */ + foreach (var waitType in PoisonWaitEvaluator.SqlServerWaitTypes) + { + Assert.Contains($"'{waitType}'", sql, StringComparison.Ordinal); + } + } + [Fact] public void Adapter_ImplementsTheSharedReadSeam() { @@ -213,10 +245,30 @@ await InsertAsync(connection, 1L, collectionTime, TestServerId, TestServerName, utcNow.AddMinutes(-4), "process1", "UPDATE Users SET Reputation = 1", DeadlockGraphXml); - /* --- poison waits: 100000ms over 50 tasks -> 2000ms avg --- */ + /* --- poison waits (#3539 A4): four THREADPOOL rows inside the window that the retired read + judged wrongly or not at all — a 703-task 8 ms storm row, a time-with-no-completed-task + row (the old tasks > 0 filter dropped it), a (0, 0) calculator marker, and one more storm + row; plus one RESOURCE_SEMAPHORE row, and a THREADPOOL row OUTSIDE the window that must + not be summed. Expected THREADPOOL: 5,779 + 304 + 0 + 594,000 = 600,083 ms across + 703 + 0 + 0 + 29,700 waits over 4 observed intervals. --- */ + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 1L, collectionTime.AddMinutes(-3), TestServerId, TestServerName, "THREADPOOL", 703L, 5779L); + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 2L, collectionTime.AddMinutes(-2), TestServerId, TestServerName, "THREADPOOL", 0L, 304L); + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 3L, collectionTime.AddMinutes(-1), TestServerId, TestServerName, "THREADPOOL", 0L, 0L); + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 4L, collectionTime, TestServerId, TestServerName, "THREADPOOL", 29700L, 594000L); + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 4L, collectionTime, TestServerId, TestServerName, "RESOURCE_SEMAPHORE", 8L, 3154L); await InsertAsync(connection, "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", - 1L, collectionTime, TestServerId, TestServerName, "THREADPOOL", 50L, 100000L); + 0L, collectionTime.AddMinutes(-30), TestServerId, TestServerName, "THREADPOOL", 1000L, 999999L); /* --- long-running queries: one 10-minute query + one in an excluded database --- */ await InsertAsync(connection, @@ -284,14 +336,31 @@ await InsertAsync(connection, Assert.Equal("UPDATE Users SET Reputation = 1", deadlock.VictimSqlText); Assert.Equal("SPID 55 (victim) vs SPID 60", deadlock.ProcessSummary); - /* --- poison waits: fetch-then-threshold, exactly like Lite's loop --- */ - var poison = await adapter.GetPoisonWaitDeltasAsync(TestServerKey, thresholdMs: 500, ct); - var worst = Assert.Single(poison); - Assert.Equal("THREADPOOL", worst.WaitType); - Assert.Equal(100000L, worst.DeltaMs); - Assert.Equal(50L, worst.DeltaTasks); - Assert.Equal(2000d, worst.AvgMsPerWait, precision: 3); - Assert.Empty(await adapter.GetPoisonWaitDeltasAsync(TestServerKey, thresholdMs: 5000, ct)); + /* --- poison waits: the window sum per type, no threshold, worst first; the out-of-window row + is excluded and the sub-bar RESOURCE_SEMAPHORE row comes back too (observed-and-quiet is + an answer the engine needs) --- */ + var poison = await adapter.GetPoisonWaitAccumulationAsync(TestServerKey, PoisonWaitEvaluator.WindowMinutes, ct); + Assert.Equal(2, poison.Count); + Assert.Equal("THREADPOOL", poison[0].WaitType); + Assert.Equal(600_083L, poison[0].AccumulatedWaitMs); + Assert.Equal(30_403L, poison[0].AccumulatedWaits); + Assert.Equal(4L, poison[0].ObservedIntervals); + /* PostgreSQL's timestamp is microsecond-precision and .NET's DateTime carries 100 ns ticks, so + the seeded instant round-trips truncated to the microsecond (CI: 15:39:48.9353066 stored as + .9353060). Compare at the store's precision; the point of the pin is that the NEWEST in-window + row's clock came back, not the out-of-window one's — asserted separately below. */ + Assert.Equal(collectionTime.Ticks / 10, poison[0].NewestCollectionTime.Ticks / 10); + Assert.True(poison[0].NewestCollectionTime > collectionTime.AddMinutes(-2), + "the newest collection_time must be the in-window row's, not the -30 minute row's"); + Assert.Equal("RESOURCE_SEMAPHORE", poison[1].WaitType); + Assert.Equal(3_154L, poison[1].AccumulatedWaitMs); + Assert.Equal(1L, poison[1].ObservedIntervals); + /* And the evaluator reads that window as the storm it is: Warning, where the retired shape saw + a 20 ms average on the biggest row and slept. */ + var graded = PoisonWaitEvaluator.EvaluateSqlServer(poison); + var storm = Assert.Single(graded); + Assert.Equal("THREADPOOL", storm.WaitType); + Assert.Equal(AlertSeverityLevel.Warning, storm.Severity); /* --- long-running queries: threshold + excluded-database drop --- */ var lrq = await adapter.GetLongRunningQueriesAsync( diff --git a/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs b/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs index c6a275606..2afc9f4f2 100644 --- a/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs +++ b/Darling/Darling.Tests/DarlingAlertTuningKnobsTests.cs @@ -38,8 +38,10 @@ public void SelfAlertKnobs_DefaultsAreTheConstantsTheyReplaced_AndReadsClampLike var config = new DarlingConfig(); var settings = new DarlingAlertSettings(config); - /* Defaults mirror the V55 DDL — the compile-time constants these knobs replaced. */ + /* Defaults mirror the V55 DDL — the compile-time constants these knobs replaced — and the V126 + GB floor mirrors its own shipped constant (#3528). */ Assert.Equal(10, settings.SelfDiskFreeWarnPercent); + Assert.Equal(50, settings.SelfDiskFreeWarnGb); Assert.Equal(30, settings.CollectionStaleMinutes); Assert.Equal(10, settings.CollectionFailureThreshold); Assert.Equal(3, settings.DiskCriticalFreePercent); @@ -51,6 +53,7 @@ public void SelfAlertKnobs_DefaultsAreTheConstantsTheyReplaced_AndReadsClampLike 0 failure threshold on the fast path would fire on any single failure, and the analysis cooldown keeps the shared engine's documented [30, 10080]. */ config.Alerts.SelfDiskFreeWarnPercent = 150; + config.Alerts.SelfDiskFreeWarnGb = -1; config.Alerts.CollectionStaleMinutes = 0; config.Alerts.CollectionFailureThreshold = 0; config.Alerts.DiskCriticalFreePercent = -5; @@ -58,6 +61,8 @@ public void SelfAlertKnobs_DefaultsAreTheConstantsTheyReplaced_AndReadsClampLike config.Alerts.AnalysisNotifyCooldownMinutes = 99999; Assert.Equal(100, settings.SelfDiskFreeWarnPercent); + /* #3528: floored at 0 like the sibling GB knobs — 0 is meaningful (it removes the floor). */ + Assert.Equal(0, settings.SelfDiskFreeWarnGb); Assert.Equal(5, settings.CollectionStaleMinutes); Assert.Equal(1, settings.CollectionFailureThreshold); Assert.Equal(0, settings.DiskCriticalFreePercent); diff --git a/Darling/Darling.Tests/DarlingAlertingTests.cs b/Darling/Darling.Tests/DarlingAlertingTests.cs index 342b349be..21d3e889d 100644 --- a/Darling/Darling.Tests/DarlingAlertingTests.cs +++ b/Darling/Darling.Tests/DarlingAlertingTests.cs @@ -24,7 +24,7 @@ namespace Darling.Tests; /// /// Pins Darling's Phase-5 slice-D pieces. Ungated: mirrors -/// Lite's App defaults member-for-member (cpu 80/Total, blocking 1, deadlock 1, poison 500, +/// Lite's App defaults member-for-member (cpu 80/Total, blocking 1, deadlock 1, poison 500 [retired as a threshold by #3539, still a default], /// LRQ 30 + the hardcoded 5/all-filters read shape, tempdb 80, low disk 10%/5GB, multiplier 3, /// lookback 60, cooldown 5, email cooldown 15) with Lite's load-time clamps, and the V3 /// "alerting-stores" migration creates the three Lite-twin tables. Gated on DARLING_TEST_PG: @@ -199,6 +199,15 @@ public async Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellat Outcomes.Add(outcome); await _inner.DeliverAsync(outcome, cancellationToken); } + + /* #3580: REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store Parity). This wrapper + records and forwards, so it forwards the REPORT too — the inner deliverer here is the real + DarlingAlertDeliverer, and swallowing its answer would make the wrapper lie about it. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + Outcomes.Add(outcome); + return await _inner.DeliverAndReportAsync(outcome, cancellationToken); + } } [Fact] @@ -232,9 +241,12 @@ await InsertAsync(connection, 1L, collectionTime, TestServerId, TestServerName, utcNow.AddMinutes(-4), "process1", "UPDATE Users SET Reputation = 1", DeadlockGraphXml); + /* #3539 A4: 36,000 THREADPOOL waits averaging 20 ms — 720 s of worker starvation inside the + ten-minute window, over the shared Warning bar (600 s). The storm shape the retired + avg-ms-per-wait read could not fire on; a 20 ms average was invisible to it. */ await InsertAsync(connection, "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", - 1L, collectionTime, TestServerId, TestServerName, "THREADPOOL", 50L, 100000L); + 1L, collectionTime, TestServerId, TestServerName, "THREADPOOL", 36000L, 720000L); AlertEngine BuildEngine(RecordingDeliverer deliverer, MuteRuleService muteRuleService) { diff --git a/Darling/Darling.Tests/DarlingAnomalyBaselineTests.cs b/Darling/Darling.Tests/DarlingAnomalyBaselineTests.cs index d4878b17f..988687e28 100644 --- a/Darling/Darling.Tests/DarlingAnomalyBaselineTests.cs +++ b/Darling/Darling.Tests/DarlingAnomalyBaselineTests.cs @@ -242,6 +242,170 @@ QUALIFY NOT (delta_cntr_value = 0 AND COALESCE(LAG(delta_cntr_value) OVER /* The pre-window row filter is unchanged from Lite. */ Assert.Contains("counter_name = 'Batch Requests/sec'", TimescaleSupport.CreatePerfmonBaselineSql, StringComparison.Ordinal); Assert.Contains("delta_cntr_value >= 0", TimescaleSupport.CreatePerfmonBaselineSql, StringComparison.Ordinal); + + /* #3527: v is the PER-SECOND rate. The perfmon_baseline supply materializes only + (collection_time, delta_cntr_value) — no stored interval, and a continuous aggregate cannot + grow a column without forfeiting the history the 4-day raw tier can't refill — so the divisor + is derived from LAG(collection_time) over the collapsed series (the WaitMsPerSec idiom), + computed in the SAME windowed CTE (window-before-filter holds for it too), with the + interval-less first row filtered alongside the restart exclusion. The DOUBLE PRECISION cast + is the io-arm rule: STDDEV_SAMP over numeric can overflow System.Decimal. */ + var intervalAt = sql.IndexOf("extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) AS interval_sec", StringComparison.Ordinal); + Assert.True(intervalAt >= 0 && intervalAt < fromCteAt, "interval_sec must be derived inside the windowed CTE"); + Assert.Contains("delta_cntr_value::DOUBLE PRECISION / interval_sec AS v", sql, StringComparison.Ordinal); + var intervalFilterAt = sql.IndexOf("interval_sec > 0", StringComparison.Ordinal); + Assert.True(intervalFilterAt > fromCteAt, "the interval filter must sit OUTSIDE the windowed CTE, with the exclusion"); + } + + /// + /// #3527: the batch-request WINDOW statistic must be requests/sec — the per-interval + /// delta_cntr_value divided by the row's measured sample_interval_seconds — or the + /// BatchRequestFloor/Fallback thresholds (defined in requests/sec) admit 60-300x-inflated + /// deltas and the comparison against the per-second baseline is cross-unit. Interval <= 0 + /// rows carry NO knowable delta and must be filtered, never read as a rate of 0 or as the + /// raw delta. + /// + [Fact] + public void BatchRequestWindow_DividesByMeasuredInterval_AndSkipsUnknowableRows() + { + var sql = PgAnomalyDetector.BatchRequestWindowSql; + + Assert.Contains("AVG(delta_cntr_value * 1.0 / NULLIF(sample_interval_seconds, 0))", sql, StringComparison.Ordinal); + Assert.Contains("MAX(delta_cntr_value * 1.0 / NULLIF(sample_interval_seconds, 0))", sql, StringComparison.Ordinal); + Assert.Contains("sample_interval_seconds > 0", sql, StringComparison.Ordinal); + + /* A raw AVG/MAX of the delta is exactly the #3527 defect — pin its absence. */ + Assert.DoesNotContain("AVG(delta_cntr_value)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("MAX(delta_cntr_value)", sql, StringComparison.Ordinal); + } + + /// + /// #3527 proven live, both halves in one place: the BatchRequests BASELINE arm derives its + /// per-second unit from LAG(collection_time) over the perfmon_baseline supply, and the DETECTOR's + /// window read divides by the stored measured interval — so the two sides meet in the same + /// requests/sec unit and the absolute bars judge honest rates. + /// + /// History (one Monday-10:00 bucket, 12 collections at 300s spacing, delta 30000 each — + /// 100 req/sec): c1 has no prior (interval NULL → dropped), c6 is a restart zero (prior 30000 > + /// 1000 → excluded), c7 is a genuine idle zero (prior 0 → kept at 0/sec). 10 samples, mean + /// (9x100 + 0)/10 = 90 — in requests/sec, where the raw-delta unit would read 27000. + /// + /// Window (the following Monday): three rows at stored interval 60, delta 600000 — 10000 + /// req/sec — plus one interval-0 row with a wild delta that must be SKIPPED, not rated. The one + /// planted history day leaves the bucket untrustworthy (Full tier needs 3 distinct days), so the + /// detector fires on the absolute BatchRequestFallback bar (5000 req/sec): peak 10000 clears it + /// honestly. Pre-#3527 the raw deltas cleared every bar by orders of magnitude regardless of + /// workload; post-fix the emitted Value, peak/avg metadata, and baseline_mean are all per-second. + /// + [Fact] + public async Task EndToEnd_BatchRequestArm_PerSecondBaselineAndWindow_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live batch-request test."); + + var ct = TestContext.Current.CancellationToken; + const int batchServerId = TestServerId + 2; // own id — this test cleans its own rows + const string batchServerName = "batch-per-second-e2e"; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + await using (var cleanup = new NpgsqlCommand( + $"DELETE FROM perfmon_stats WHERE server_id = {batchServerId}; " + + $"DELETE FROM wait_stats WHERE server_id = {batchServerId};", connection)) + { + await cleanup.ExecuteNonQueryAsync(ct); + } + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var bodySucceeded = false; + try + { + var day = DateTime.UtcNow.Date.AddDays(-8); + while (day.DayOfWeek != DayOfWeek.Monday) day = day.AddDays(-1); + var historyStart = DateTime.SpecifyKind(day.AddHours(10), DateTimeKind.Unspecified); + + for (var i = 0; i < 12; i++) + { + var delta = (i == 5 || i == 6) ? 0L : 30000L; + await InsertAsync(connection, + "INSERT INTO perfmon_stats (collection_id, collection_time, server_id, server_name, object_name, counter_name, instance_name, cntr_value, delta_cntr_value, sample_interval_seconds) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + (long)(200 + i), historyStart.AddMinutes(5 * i), batchServerId, batchServerName, + "SQLServer:SQL Statistics", "Batch Requests/sec", "", delta * 2, delta, 300); + } + + /* The baseline supply must exist (see the wait test's note) — plain fallback views. */ + await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct); + + var provider = new PgBaselineProvider(postgres); + var analysisTime = historyStart.AddDays(7); + + var baseline = await provider.GetBaselineAsync(batchServerId, MetricNames.BatchRequests, analysisTime); + Assert.Equal(10L, baseline.SampleCount); + Assert.Equal(90.0, baseline.Mean, 0.001); + Assert.Equal(BaselineTier.Full, baseline.Tier); + Assert.Equal(10, baseline.HourOfDay); + Assert.Equal((int)DayOfWeek.Monday, baseline.DayOfWeek); + + /* Canary for the HasBaselineData gate — OUTSIDE the analysis window so the wait + detector's own window read stays empty and it emits nothing. */ + await InsertAsync(connection, + "INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) VALUES ($1, $2, $3, $4, $5, $6, $7)", + 300L, historyStart, batchServerId, batchServerName, TestWaitType, 1L, 100L); + + /* The anomalous current window: 10000 req/sec (delta 600000 over a measured 60s), + plus one interval-0 row whose wild delta must never be rated. */ + for (var i = 0; i < 3; i++) + { + await InsertAsync(connection, + "INSERT INTO perfmon_stats (collection_id, collection_time, server_id, server_name, object_name, counter_name, instance_name, cntr_value, delta_cntr_value, sample_interval_seconds) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + (long)(400 + i), analysisTime.AddMinutes(5 * (i + 1)), batchServerId, batchServerName, + "SQLServer:SQL Statistics", "Batch Requests/sec", "", 1200000L, 600000L, 60); + } + await InsertAsync(connection, + "INSERT INTO perfmon_stats (collection_id, collection_time, server_id, server_name, object_name, counter_name, instance_name, cntr_value, delta_cntr_value, sample_interval_seconds) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + 403L, analysisTime.AddMinutes(20), batchServerId, batchServerName, + "SQLServer:SQL Statistics", "Batch Requests/sec", "", 0L, 999999999L, 0); + + var detector = new PgAnomalyDetector(postgres, provider); + var context = new AnalysisContext + { + ServerId = batchServerId, + ServerName = batchServerName, + TimeRangeStart = analysisTime, + TimeRangeEnd = analysisTime.AddMinutes(30), + ServerUtcOffset = TimeSpan.Zero + }; + + var anomalies = await detector.DetectAnomaliesAsync(context); + + var fact = Assert.Single(anomalies); + Assert.Equal("ANOMALY_BATCH_REQUESTS", fact.Key); + Assert.Equal(10000.0, fact.Value, 0.001); // per-second, not 600000 + Assert.Equal(10000.0, fact.Metadata["peak_batch_requests"], 0.001); + Assert.Equal(10000.0, fact.Metadata["avg_batch_requests"], 0.001); + Assert.Equal(3.0, fact.Metadata["window_samples"]); // the interval-0 row is NOT a sample + Assert.Equal(90.0, fact.Metadata["baseline_mean"], 0.001); // same unit as the window + Assert.Equal(1.0, fact.Metadata["baseline_low_quality"]); // one distinct day → absolute bar + Assert.Equal(2.0, fact.Metadata["fallback_exceedance"], 0.001); // 10000 / the 5000 req/sec bar + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + { + using (var command = new NpgsqlCommand( + $"DELETE FROM perfmon_stats WHERE server_id = {batchServerId}; " + + $"DELETE FROM wait_stats WHERE server_id = {batchServerId};", cleanup)) + { + await command.ExecuteNonQueryAsync(cleanupCt); + } + await DropBaselineFallbackViewsAsync(cleanup, cleanupCt); + }); + } } [Fact] diff --git a/Darling/Darling.Tests/DarlingCliCommandsTests.cs b/Darling/Darling.Tests/DarlingCliCommandsTests.cs index e6ec32957..1cc5bbf4e 100644 --- a/Darling/Darling.Tests/DarlingCliCommandsTests.cs +++ b/Darling/Darling.Tests/DarlingCliCommandsTests.cs @@ -145,16 +145,40 @@ public void FormatProbeLine_PostgresTarget_ReportsPostgresFactsAndNoSqlServerOne Assert.DoesNotContain("Unknown (0)", line, StringComparison.Ordinal); } - /// An Aurora writer clears every gate, so the count says so rather than listing nothing. + /// + /// An Aurora writer clears every gate but ONE, and the line names it. Until #3604 Aurora was a strict + /// superset of what the PostgreSQL collectors read and the line said "all N apply"; pg_wait_sampling + /// is now gated off Aurora — the engine cannot preload the module and has pg_wait_stats instead — + /// so the pre-flight is where an operator first sees that one collector, by name, does not run there. + /// Counted from the catalog and the collectors' own gates rather than hard-coded, so a second Aurora + /// gap shows up here as a changed count rather than a silently passing pin. + /// [Fact] - public void FormatProbeLine_AuroraWriter_ReportsEveryPostgresCollectorApplies() + public void FormatProbeLine_AuroraWriter_ReportsEveryPostgresCollectorButTheOneAuroraCannotHave() { - var expected = CollectorCatalog.All.Count(d => d.TargetEngine == CollectorTargetEngine.PostgreSql); + var target = PostgresProbe().ToTargetInfo(); + var postgres = CollectorCatalog.All.Where(d => d.TargetEngine == CollectorTargetEngine.PostgreSql).ToList(); + var skipped = postgres.Where(d => !CollectorCatalog.AppliesTo(d, target)).Select(d => d.Name).ToList(); + Assert.Equal(new[] { PgWaitSamplingCollector.Instance.Name }, skipped); var line = DarlingCliCommands.FormatProbeLine("aurora-writer", PostgresProbe()); - Assert.Contains($"all {expected} PostgreSQL collectors apply", line, StringComparison.Ordinal); - Assert.DoesNotContain("skipped", line, StringComparison.Ordinal); + Assert.Contains($"{postgres.Count - 1} of {postgres.Count} PostgreSQL collectors apply", line, StringComparison.Ordinal); + Assert.Contains("skipped: pg_wait_sampling", line, StringComparison.Ordinal); + } + + /// The line an Aurora writer used to get, every PostgreSQL collector applying, is now the STOCK + /// writer's with the extension present — the shape #3604 made the finest stock tier. + [Fact] + public void FormatProbeLine_StockWriterWithTheExtension_ReportsEveryPostgresCollectorApplies() + { + var expected = CollectorCatalog.All.Count(d => d.TargetEngine == CollectorTargetEngine.PostgreSql); + var probe = PostgresProbe() with { IsAurora = false, HasPgWaitSamplingExtension = true }; + + /* pg_wait_stats and pg_cpu_utilization are Aurora-only, so a stock target skips those two instead. */ + var line = DarlingCliCommands.FormatProbeLine("stock-writer", probe); + Assert.Contains($"{expected - 2} of {expected} PostgreSQL collectors apply", line, StringComparison.Ordinal); + Assert.DoesNotContain("pg_wait_sampling", line, StringComparison.Ordinal); } /// diff --git a/Darling/Darling.Tests/DarlingComposeTests.cs b/Darling/Darling.Tests/DarlingComposeTests.cs index 6a0c9cb99..21ea8af89 100644 --- a/Darling/Darling.Tests/DarlingComposeTests.cs +++ b/Darling/Darling.Tests/DarlingComposeTests.cs @@ -297,6 +297,16 @@ SQL Server target when the engine half is skipped. */ PostgresMajorVersion = 17, PostgresVersionNum = 170_005, IsInRecovery = false, }; + /* #3604: pg_wait_sampling is gated OFF Aurora (the module cannot be preloaded there; pg_wait_stats is + the instrument on that engine), so its measures surface for a stock writer instead. Every other + PostgreSQL measure still clears the Aurora writer, and the two shapes together still cover the + whole PostgreSQL measure set - which is the property this test exists for. */ + var stockWriterWithExtension = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, IsAurora = false, HasPgWaitSamplingExtension = true, + PostgresMajorVersion = 17, PostgresVersionNum = 170_005, IsInRecovery = false, + }; + var pg = PostgresMeasures(); Assert.NotEmpty(pg); foreach (var measure in pg) @@ -308,6 +318,16 @@ SQL Server target when the engine half is skipped. */ $"pg measure '{measure.Key}' source '{measure.SourceTable}' engine gate must exclude SQL Server."); Assert.False(CollectorCatalog.AppliesTo(collector!, sqlServer), $"pg measure '{measure.Key}' source '{measure.SourceTable}' must not apply to a SQL Server target."); + + if (string.Equals(measure.SourceTable, PgWaitSamplingCollector.Instance.TargetTable, StringComparison.Ordinal)) + { + Assert.False(CollectorCatalog.AppliesTo(collector!, auroraWriter), + $"pg measure '{measure.Key}' reads pg_wait_sampling, which #3604 gated off Aurora - it must not surface there."); + Assert.True(CollectorCatalog.AppliesTo(collector!, stockWriterWithExtension), + $"pg measure '{measure.Key}' source '{measure.SourceTable}' must apply to a stock writer with the extension."); + continue; + } + Assert.True(CollectorCatalog.AppliesTo(collector!, auroraWriter), $"pg measure '{measure.Key}' source '{measure.SourceTable}' must apply to a modern Aurora writer."); } diff --git a/Darling/Darling.Tests/DarlingDailySummaryRangeTests.cs b/Darling/Darling.Tests/DarlingDailySummaryRangeTests.cs index 844bb1ef2..87c73c385 100644 --- a/Darling/Darling.Tests/DarlingDailySummaryRangeTests.cs +++ b/Darling/Darling.Tests/DarlingDailySummaryRangeTests.cs @@ -82,9 +82,10 @@ public async Task TheCalendar_BandsEachDaySeparately_ShowsCollectionGaps_AndAnch /* ── two collected days with a hole between them ── - Today collected cleanly. Two days ago collected AND recorded an ERROR run, which bands that - day Critical. Yesterday is deliberately left alone: it is the gap, and the point of the read - is that a gap is visible as an ABSENT day rather than as a quiet one. + Today collected cleanly. Two days ago collected AND recorded an ERROR run — one of two runs, + a 50% error share past the 20% bar, which bands that day Warning (#3539 A2: the share's + tier, never Critical). Yesterday is deliberately left alone: it is the gap, and the point of + the read is that a gap is visible as an ABSENT day rather than as a quiet one. */ await SeedRunAsync(connection, ct, DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow), "wait_stats", "SUCCESS"); await SeedRunAsync(connection, ct, twoDaysAgo.AddHours(12), "wait_stats", "SUCCESS"); @@ -115,8 +116,12 @@ nobody would spot in a table. */ Assert.Equal(today.ToString("yyyy-MM-dd"), days[1].GetProperty("summary_date").GetString()); /* The two days disagree, which is what makes this a calendar rather than one verdict. */ - Assert.Equal("Critical", days[0].GetProperty("overall_health").GetString()); + Assert.Equal("Warning", days[0].GetProperty("overall_health").GetString()); Assert.Equal(1, days[0].GetProperty("collection_errors").GetInt64()); + /* #3539 A2/A3, additive: the share's denominator and the blocking rate ride the payload, so the + band's figures are on the surface that bands — a finished day rates over 24 hours. */ + Assert.Equal(2, days[0].GetProperty("collection_runs").GetInt64()); + Assert.Equal(0.0, days[0].GetProperty("blocking_rate_per_hour").GetDouble()); Assert.Equal("Healthy", days[1].GetProperty("overall_health").GetString()); Assert.Equal(0, days[1].GetProperty("collection_errors").GetInt64()); diff --git a/Darling/Darling.Tests/DarlingDeltaSeederTests.cs b/Darling/Darling.Tests/DarlingDeltaSeederTests.cs index d17c0eade..8343383c6 100644 --- a/Darling/Darling.Tests/DarlingDeltaSeederTests.cs +++ b/Darling/Darling.Tests/DarlingDeltaSeederTests.cs @@ -18,12 +18,19 @@ namespace Darling.Tests; /// -/// Pins Darling's restart continuity (the Postgres twin of Lite's DuckDB delta seeding): the four -/// seed queries' latest-row form and column lists, and the shared-core inheritance. The -/// end-to-end test runs only when DARLING_TEST_PG points at a dev Postgres: it inserts two -/// wait_stats rows for a fake server at two collection times, runs SeedFromStoreAsync, and proves -/// the LATEST stored row became the delta baseline — so the first collection after a service -/// restart produces a real delta instead of 0. +/// Pins Darling's restart continuity (the Postgres twin of Lite's DuckDB delta seeding): every seed +/// query's form and column list, and the shared-core inheritance. The end-to-end tests run only when +/// DARLING_TEST_PG points at a dev Postgres: they insert two collections per family for a fake server, +/// run SeedFromStoreAsync, and prove the LATEST stored row became the delta baseline — so the first +/// collection after a service restart produces a real delta instead of 0. +/// +/// #3540 A4 widened the seed from four families to every delta family the service monitors, plus +/// the per-group pass window the #2235 series-age rescue reads. Two query shapes are pinned here: the +/// original latest-collection-per-server row-value probe (now also latch_stats and spinlock_stats), and +/// the latest-row-per-key DISTINCT ON form for the families whose collectors do not write every +/// key every pass (procedure_stats, pg_wait_stats, pg_statement_stats — and query_stats since V128, #3540, +/// once the store persisted the two statement offsets its key is made of; a pre-V128 row seeds the pass +/// window and no key). The census in Lite.Tests holds the set. /// /* Live-fixture tests share one Postgres store; the collection serializes them so cross-test row churn (inserts/purges/deletes) cannot race another class's assertions. */ @@ -96,14 +103,138 @@ seeded with a null timestamp and the gap policy could not reject a stale one. */ sql, StringComparison.Ordinal); } + /// + /// #3540 A4: the two families that mirror wait_stats exactly (every key written every pass, keyed by + /// name) take wait_stats' exact shape. Column lists are the collector's counters, in its order. + /// + [Fact] + public void SeedSql_LatchAndSpinlock_LatestRowFormAndColumns() + { + var latch = DarlingDeltaCalculator.LatchStatsSeedSql; + Assert.Contains("SELECT server_id, latch_class, waiting_requests_count, wait_time_ms, max_wait_time_ms, collection_time", + latch, StringComparison.Ordinal); + Assert.Contains("FROM latch_stats", latch, StringComparison.Ordinal); + Assert.Contains("(server_id, collection_time) IN (", latch, StringComparison.Ordinal); + + var spin = DarlingDeltaCalculator.SpinlockStatsSeedSql; + Assert.Contains("SELECT server_id, spinlock_name, collisions, spins, sleep_time, backoffs, collection_time", + spin, StringComparison.Ordinal); + Assert.Contains("FROM spinlock_stats", spin, StringComparison.Ordinal); + Assert.Contains("(server_id, collection_time) IN (", spin, StringComparison.Ordinal); + /* spins_per_collision is a DMV-computed ratio, never delta'd, so it is not a baseline. */ + Assert.DoesNotContain("spins_per_collision", spin, StringComparison.Ordinal); + } + + /// + /// The procedure_stats key is built IN the SQL, exactly as ProcedureStatsCollector.WritePayload + /// builds it (plan_handle ?? $"{db}.{schema}.{object}", null parts formatting as empty), so + /// DISTINCT ON partitions by the key the collector will present and the reader uses it verbatim. A key + /// assembled differently seeds a baseline nothing ever reads — silently. + /// + [Fact] + public void SeedSql_ProcedureStats_BuildsTheCollectorsKeyAndTakesTheLatestRowPerKey() + { + var sql = DarlingDeltaCalculator.ProcedureStatsSeedSql; + Assert.Contains("SELECT DISTINCT ON (server_id, delta_key)", sql, StringComparison.Ordinal); + Assert.Contains( + "COALESCE(plan_handle, COALESCE(database_name, '') || '.' || COALESCE(schema_name, '') || '.' || COALESCE(object_name, '')) AS delta_key", + sql, StringComparison.Ordinal); + Assert.Contains("execution_count, total_worker_time, total_elapsed_time,", sql, StringComparison.Ordinal); + Assert.Contains("total_logical_reads, total_logical_writes, total_physical_reads, total_spills,", sql, StringComparison.Ordinal); + Assert.Contains("FROM procedure_stats", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY server_id, delta_key, collection_time DESC", sql, StringComparison.Ordinal); + } + + /// + /// The PostgreSQL pair's keys are the collectors' own: the numeric event id for waits (the name changes + /// case across Aurora majors), and the full (queryid, database_id, user_id, toplevel) identity for + /// statements. The datid in that key is the collector's choice as written (#3540 A11c); the seed + /// reproduces it rather than redesigning it. + /// + [Fact] + public void SeedSql_PostgresPair_PartitionByTheCollectorsKeyAndTakeTheLatestRowPerKey() + { + var waits = DarlingDeltaCalculator.PgWaitStatsSeedSql; + Assert.Contains("SELECT DISTINCT ON (server_id, wait_event_id)", waits, StringComparison.Ordinal); + Assert.Contains("server_id, wait_event_id, waits, wait_time_us, collection_time", waits, StringComparison.Ordinal); + Assert.Contains("FROM pg_wait_stats", waits, StringComparison.Ordinal); + Assert.Contains("ORDER BY server_id, wait_event_id, collection_time DESC", waits, StringComparison.Ordinal); + Assert.DoesNotContain("wait_event,", waits, StringComparison.Ordinal); + + var statements = DarlingDeltaCalculator.PgStatementStatsSeedSql; + Assert.Contains("SELECT DISTINCT ON (server_id, queryid, database_id, user_id, toplevel)", statements, StringComparison.Ordinal); + Assert.Contains("calls, total_exec_time_ms, rows_returned, collection_time", statements, StringComparison.Ordinal); + Assert.Contains("FROM pg_statement_stats", statements, StringComparison.Ordinal); + Assert.Contains("ORDER BY server_id, queryid, database_id, user_id, toplevel, collection_time DESC", statements, StringComparison.Ordinal); + } + + /// + /// query_stats is key-seeded since V128 (#3540): the store persists both statement offsets now, so + /// the read partitions by the collector's FULL key — sql_handle, both offsets, plan_handle — and + /// returns each key's latest row inside the window, with the eight counters the collector's eight + /// series difference. The offsets are selected RAW (no COALESCE, no arithmetic) because the seeder + /// rebuilds the key from them with the collector's own interpolation, and there is no offset filter + /// in the SQL: a pre-V128 row (NULL offsets) is read for the pass window and skipped for keys in C#, + /// so one read serves both halves. + /// + [Fact] + public void SeedSql_QueryStats_PartitionsByTheFullDeltaKeyAndSelectsTheOffsetsRaw() + { + var sql = DarlingDeltaCalculator.QueryStatsSeedSql; + Assert.Contains("SELECT DISTINCT ON (server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle)", sql, StringComparison.Ordinal); + Assert.Contains("server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle,", sql, StringComparison.Ordinal); + Assert.Contains("execution_count, total_worker_time, total_elapsed_time,", sql, StringComparison.Ordinal); + Assert.Contains("total_logical_reads, total_logical_writes, total_physical_reads, total_rows, total_spills,", sql, StringComparison.Ordinal); + Assert.Contains("FROM query_stats", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle, collection_time DESC", sql, StringComparison.Ordinal); + Assert.DoesNotContain("IS NOT NULL", sql, StringComparison.Ordinal); + Assert.DoesNotContain("COALESCE", sql, StringComparison.Ordinal); + Assert.DoesNotContain("GROUP BY", sql, StringComparison.Ordinal); + } + public static TheoryData SeedQueries() => new() { { DarlingDeltaCalculator.WaitStatsSeedSql, "wait_stats" }, { DarlingDeltaCalculator.FileIoStatsSeedSql, "file_io_stats" }, { DarlingDeltaCalculator.PerfmonStatsSeedSql, "perfmon_stats" }, { DarlingDeltaCalculator.MemoryGrantStatsSeedSql, "memory_grant_stats" }, + { DarlingDeltaCalculator.LatchStatsSeedSql, "latch_stats" }, + { DarlingDeltaCalculator.SpinlockStatsSeedSql, "spinlock_stats" }, }; + /// The latest-row-per-key shape (#3540 A4, query_stats since V128): one table read, one bound, DISTINCT ON. + public static TheoryData PerKeySeedQueries() => new() + { + { DarlingDeltaCalculator.ProcedureStatsSeedSql, "procedure_stats" }, + { DarlingDeltaCalculator.QueryStatsSeedSql, "query_stats" }, + { DarlingDeltaCalculator.PgWaitStatsSeedSql, "pg_wait_stats" }, + { DarlingDeltaCalculator.PgStatementStatsSeedSql, "pg_statement_stats" }, + }; + + /// + /// The per-key shape's bound: the cutoff appears exactly ONCE, on its only table read, before the + /// DISTINCT ON's ORDER BY — there is no inner aggregate to bind a second time, and a second table read + /// would be the unbounded scan #1772 removed. The live chunk-exclusion pin below proves the one bound + /// keeps TimescaleDB on the window's chunk. + /// + [Theory] + [MemberData(nameof(PerKeySeedQueries))] + public void PerKeySeedSql_BoundsItsOnlyTableRead_OnceBeforeTheOrdering(string sql, string table) + { + Assert.Equal(1, CountOccurrences(sql, "collection_time >= $1")); + Assert.Equal(1, CountOccurrences(sql, "FROM " + table)); + Assert.Contains("SELECT DISTINCT ON (server_id,", sql, StringComparison.Ordinal); + Assert.EndsWith("collection_time DESC", sql.TrimEnd(), StringComparison.Ordinal); + Assert.True( + sql.IndexOf("collection_time >= $1", StringComparison.Ordinal) + < sql.IndexOf("ORDER BY server_id,", StringComparison.Ordinal), + "the bound must sit on the table read, ahead of the ordering that picks the latest row per key"); + Assert.DoesNotContain("$2", sql, StringComparison.Ordinal); + /* No MAX(collection_time) probe: that shape returns the latest COLLECTION, which for these families + is missing every key whose counter was idle or fell out of the TOP (n) on the last pass. */ + Assert.DoesNotContain("MAX(collection_time)", sql, StringComparison.Ordinal); + } + /// /// The #1772 pin, mirrored by Lite's LiteDeltaSeederTests: BOTH halves carry the cutoff. Bounding /// only the outer read still lets the inner MAX() aggregate every chunk of the hypertable; @@ -281,8 +412,8 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup } /// - /// Every seed query actually RUNS against the real store schema — all four, not just the one the - /// end-to-end test drives. + /// Every seed query actually RUNS against the real store schema — all ten, not just the ones the + /// end-to-end tests drive. /// /// This closes a hole that would reproduce the exact field symptom. The memory-grant seed swallows /// its own exceptions entirely (a deliberate tolerance for a table that may not exist yet after a schema @@ -308,6 +439,14 @@ public async Task EverySeedQuery_RunsAgainstTheRealSchema_AgainstDevPostgres() (DarlingDeltaCalculator.FileIoStatsSeedSql, 12), (DarlingDeltaCalculator.PerfmonStatsSeedSql, 6), (DarlingDeltaCalculator.MemoryGrantStatsSeedSql, 6), + /* #3540 A4 */ + (DarlingDeltaCalculator.LatchStatsSeedSql, 6), + (DarlingDeltaCalculator.SpinlockStatsSeedSql, 7), + (DarlingDeltaCalculator.ProcedureStatsSeedSql, 10), + /* #3540 V128: 5 key parts + 8 counters + collection_time */ + (DarlingDeltaCalculator.QueryStatsSeedSql, 14), + (DarlingDeltaCalculator.PgWaitStatsSeedSql, 5), + (DarlingDeltaCalculator.PgStatementStatsSeedSql, 9), }; foreach (var (sql, columns) in queries) @@ -323,16 +462,364 @@ list the reader indexes by ordinal is the one the store actually returns. */ } } + /// + /// #3540 A4, end to end on the real schema: every family the seed gained produces a REAL delta on the + /// first post-restart call, the per-key shape restores a key the latest pass did not write, the + /// query_stats pass window arms the #2235 series-age rescue on that first pass, and an original + /// family's pass window is seeded too. Every expected value below was worked by hand from the rows + /// and reproduced against PG18 + TimescaleDB 2.28.1 before this was written. + /// + /// Two servers: the seeded one, and a second whose only rows predate the window so the + /// same pass proves the bound is doing the discriminating (a first sighting, and no rescue). + /// + [Fact] + public async Task EndToEnd_EveryFamilyAndThePassWindow_SeedFromStore_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live delta-seeding test."); + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await PgMigrations.MigrateAsync(connection, TestContext.Current.CancellationToken); + + var bodySucceeded = false; + try + { + await DeleteFamilyRowsAsync(connection, TestContext.Current.CancellationToken); + + var now = DateTime.UtcNow; + var stale = Naive(now - CollectorDeltaCalculator.SeedLookback - TimeSpan.FromMinutes(5)); + var older = Naive(now.AddMinutes(-4)); + var latest = Naive(now.AddMinutes(-2)); + + /* latch_stats / spinlock_stats: every key every pass; the stale server's only pass is outside. */ + await LatchAsync(connection, TestServerId, stale, 1, 1, 1); + await LatchAsync(connection, TestServerId, older, 100, 1000, 50); + await LatchAsync(connection, TestServerId, latest, 140, 1500, 60); + await LatchAsync(connection, StaleServerId, stale, 5, 50, 5); + await SpinlockAsync(connection, older, 10, 100, 5, 2); + await SpinlockAsync(connection, latest, 20, 200, 8, 3); + + /* procedure_stats: 0x01 in both passes; 0x02 only in the OLDER pass (fell out of the TOP (150)); + a null-handle row keyed by db.schema.object; and a stale 0x02 row outside the window. */ + await ProcAsync(connection, older, "0x01", "db", "dbo", "p1", 10); + await ProcAsync(connection, latest, "0x01", "db", "dbo", "p1", 15); + await ProcAsync(connection, older, "0x02", "db", "dbo", "p2", 7); + await ProcAsync(connection, latest, null, "db", "dbo", "proc3", 3); + await ProcAsync(connection, stale, "0x02", "db", "dbo", "p2", 1); + + /* query_stats: three PRE-V128 passes (no offsets), one stale — the pass window must come from the + two inside; plus V128 rows carrying the offsets (#3540): the whole-batch statement (0, -1) in + both passes, a second statement of the same batch/plan (100, 240) only in the older pass, and a + null-handle row. */ + foreach (var t in new[] { stale, older, latest }) + { + await QueryStatsAsync(connection, t); + } + await KeyedQueryStatsAsync(connection, older, "0xSH1", 0, -1, "0xPH1", 10); + await KeyedQueryStatsAsync(connection, latest, "0xSH1", 0, -1, "0xPH1", 15); + await KeyedQueryStatsAsync(connection, older, "0xSH1", 100, 240, "0xPH1", 7); + await KeyedQueryStatsAsync(connection, latest, null, 0, -1, null, 3); + + /* pg_wait_stats: 1001 both passes; 1002 idle at the latest pass, so its newest row is the older. */ + await PgWaitAsync(connection, older, 1001, 5, 500); + await PgWaitAsync(connection, latest, 1001, 10, 900); + await PgWaitAsync(connection, older, 1002, 3, 300); + + /* pg_statement_stats: statement 11 both passes; statement 12 (toplevel false) idle at the latest. */ + await PgStatementAsync(connection, older, 11, 16384, 10, true, 100, 1234.9, 50); + await PgStatementAsync(connection, latest, 11, 16384, 10, true, 120, 1500.7, 60); + await PgStatementAsync(connection, older, 12, 16384, 10, false, 7, 70.2, 7); + + /* wait_stats: an ORIGINAL family, to prove its pass window is seeded too. */ + await InsertWaitStatsRowAsync(connection, older, waitingTasks: 10, waitTimeMs: 2000, signalWaitTimeMs: 500); + await InsertWaitStatsRowAsync(connection, latest, waitingTasks: 40, waitTimeMs: 5000, signalWaitTimeMs: 800); + + var deltas = new DarlingDeltaCalculator(); + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + await deltas.SeedFromStoreAsync(postgres, null, TestContext.Current.CancellationToken); + } + + const int Gap = CollectorDeltaCalculator.DefaultMaxGapSeconds; + var pass = now; + + /* latch: baseline is the LATEST pass (1500), not the older (1000) or the stale (1). */ + Assert.Equal(100, deltas.CalculateDeltaWithInterval(TestServerId, "latch_stats_wait_time", "BUFFER", 1600, out var latchInterval, pass, Gap)); + Assert.InRange(latchInterval, 118, 122); + Assert.Equal(10, deltas.CalculateDelta(TestServerId, "latch_stats_waiting_requests", "BUFFER", 150, pass, Gap)); + Assert.Equal(0, deltas.CalculateDelta(TestServerId, "latch_stats_max_wait", "BUFFER", 60, pass, Gap)); + + /* spinlock: all four counters. */ + Assert.Equal(5, deltas.CalculateDelta(TestServerId, "spinlock_stats_collisions", "LOCK_HASH", 25, pass, Gap)); + Assert.Equal(60, deltas.CalculateDelta(TestServerId, "spinlock_stats_spins", "LOCK_HASH", 260, pass, Gap)); + Assert.Equal(1, deltas.CalculateDelta(TestServerId, "spinlock_stats_sleep_time", "LOCK_HASH", 9, pass, Gap)); + Assert.Equal(4, deltas.CalculateDelta(TestServerId, "spinlock_stats_backoffs", "LOCK_HASH", 7, pass, Gap)); + + /* procedure_stats: 0x01 from the latest pass; 0x02 from the OLDER pass (the latest-collection + shape would have missed it entirely and this would be 0); the null-handle fallback key. */ + Assert.Equal(3, deltas.CalculateDelta(TestServerId, "proc_stats_exec", "0x01", 18, pass, Gap)); + Assert.Equal(30, deltas.CalculateDelta(TestServerId, "proc_stats_worker", "0x01", 180, pass, Gap)); + Assert.Equal(2, deltas.CalculateDeltaWithInterval(TestServerId, "proc_stats_exec", "0x02", 9, out var procInterval, pass, Gap)); + Assert.InRange(procInterval, 238, 242); + Assert.Equal(2, deltas.CalculateDelta(TestServerId, "proc_stats_exec", "db.dbo.proc3", 5, pass, Gap)); + + /* pg_wait_stats: the idle-at-latest event is seeded from its older row. */ + Assert.Equal(2, deltas.CalculateDelta(TestServerId, "pg_wait_stats_waits", "1001", 12, pass, Gap)); + Assert.Equal(100, deltas.CalculateDelta(TestServerId, "pg_wait_stats_time", "1001", 1000, pass, Gap)); + Assert.Equal(1, deltas.CalculateDelta(TestServerId, "pg_wait_stats_waits", "1002", 4, pass, Gap)); + + /* pg_statement_stats: the collector's key, and its long-truncation of the stored double. */ + Assert.Equal(10, deltas.CalculateDelta(TestServerId, "pg_statement_stats_calls", "11|16384|10|1", 130, pass, Gap)); + Assert.Equal(100, deltas.CalculateDelta(TestServerId, "pg_statement_stats_time", "11|16384|10|1", 1600, pass, Gap)); + Assert.Equal(1, deltas.CalculateDelta(TestServerId, "pg_statement_stats_rows", "11|16384|10|1", 61, pass, Gap)); + Assert.Equal(1, deltas.CalculateDelta(TestServerId, "pg_statement_stats_calls", "12|16384|10|0", 8, pass, Gap)); + + /* query_stats KEYS (#3540, V128): the latest pass is the baseline for the key both passes wrote, + spelled with the raw -1 as the collector spells it; the statement that fell out of the TOP (n) + is restored from its OLDER row over ~240 s; a null handle formats as empty on both sides; and + the pre-V128 rows seeded NOTHING — not even under a normalizing guess (gap policy off, so only a + first sighting reads 0; a seeded baseline of 1 would return 4). */ + Assert.Equal(3, deltas.CalculateDeltaWithInterval(TestServerId, "query_stats_exec", "0xSH1:0:-1:0xPH1", 18, out var keyedInterval, pass, Gap)); + Assert.InRange(keyedInterval, 118, 122); + Assert.Equal(210, deltas.CalculateDelta(TestServerId, "query_stats_spills", "0xSH1:0:-1:0xPH1", 1260, pass, Gap)); + Assert.Equal(2, deltas.CalculateDeltaWithInterval(TestServerId, "query_stats_exec", "0xSH1:100:240:0xPH1", 9, out var fellOutInterval, pass, Gap)); + Assert.InRange(fellOutInterval, 238, 242); + Assert.Equal(2, deltas.CalculateDelta(TestServerId, "query_stats_exec", ":0:-1:", 5, pass, Gap)); + Assert.Equal(0, deltas.CalculateDelta(TestServerId, "query_stats_exec", "sh:0:0:ph", 5, pass, 0)); + Assert.Equal(0, deltas.CalculateDelta(TestServerId, "query_stats_exec", "sh:0:-1:ph", 5, pass, 0)); + + /* The series-age rescue on the FIRST post-restart pass: the pass window is seeded from EVERY + query_stats row, the pre-V128 ones included, so a plan compiled 30 s ago (inside the ~120 s + since the last pre-restart pass) is credited in full with a real interval, while a plan older + than that gap baselines honestly. Unseeded, both are (0, 0) — the defect #3614 closed and V128 + must not reopen on the first restart after the upgrade, when the window holds only such rows. */ + Assert.Equal(900, deltas.CalculateDeltaWithSeriesAge(TestServerId, "query_stats_worker", "sh:0:99:newplan", 900, 30, out var rescueInterval, pass, Gap)); + Assert.InRange(rescueInterval, 118, 122); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(TestServerId, "query_stats_exec", "sh:0:99:oldplan", 900, 3_000, out var oldInterval, pass, Gap)); + Assert.Equal(0, oldInterval); + + /* An original family's pass window is seeded too, proven through the same rescue path. */ + Assert.Equal(77, deltas.CalculateDeltaWithSeriesAge(TestServerId, "wait_stats_tasks", "NEW_WAIT", 77, 10, out _, pass, Gap)); + + /* The stale server: no key seed (first sighting even with the gap policy off) and no pass window. */ + Assert.Equal(0, deltas.CalculateDelta(StaleServerId, "latch_stats_wait_time", "BUFFER", 999, pass, 0)); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(StaleServerId, "query_stats_worker", "k", 900, 30, out _, pass, Gap)); + + /* ClearServer drops both seeded halves — the re-add path both hosts now wire (#3540 A4). */ + deltas.ClearServer(TestServerId); + Assert.Equal(0, deltas.CalculateDelta(TestServerId, "latch_stats_wait_time", "BUFFER", 1700, pass.AddSeconds(30), 0)); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(TestServerId, "query_stats_worker", "sh:0:99:another", 500, 5, out _, pass.AddSeconds(30), Gap)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteFamilyRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The per-key shape's bound, asked of the planner the way the #1772 pin asks it of the row-value + /// shape: with one bound on one table read, DISTINCT ON over the window must still exclude the + /// two-day-old chunk and read the window's. pg_statement_stats is the family with the most rows per + /// pass on the dogfood fleet, so it is the one whose shape matters most; the other two per-key + /// queries are the same shape over the same kind of index. + /// + [Fact] + public async Task PerKeySeedRead_ExcludesChunksOlderThanTheWindow_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live chunk-exclusion test."); + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await PgMigrations.MigrateAsync(connection, TestContext.Current.CancellationToken); + + var timescaleEnabled = await LiveTimescaleProbe.TryEnableAsync(connectionString!, TestContext.Current.CancellationToken); + Assert.SkipWhen(!timescaleEnabled, "TimescaleDB is not enabled on DARLING_TEST_PG — there are no chunks to exclude."); + + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, TestContext.Current.CancellationToken); + + var bodySucceeded = false; + try + { + await DeleteFamilyRowsAsync(connection, TestContext.Current.CancellationToken); + + var insideWindow = Naive(DateTime.UtcNow.AddMinutes(-1)); + var twoDaysBack = Naive(DateTime.UtcNow.AddDays(-2)); + await PgStatementAsync(connection, insideWindow, 11, 16384, 10, true, 120, 1500.7, 60); + await PgStatementAsync(connection, twoDaysBack, 11, 16384, 10, true, 1, 1, 1); + + var recentChunk = await ChunkHoldingRowAsync(connection, "pg_statement_stats", insideWindow); + var oldChunk = await ChunkHoldingRowAsync(connection, "pg_statement_stats", twoDaysBack); + + Assert.SkipWhen(recentChunk is null || oldChunk is null, "the sentinel rows did not land — nothing to prove."); + Assert.SkipWhen(string.Equals(recentChunk, oldChunk, StringComparison.Ordinal), + $"pg_statement_stats put both timestamps in {recentChunk} — this store has no separate chunk to exclude."); + + var plan = await ExplainAsync(connection, DarlingDeltaCalculator.PgStatementStatsSeedSql, CollectorDeltaCalculator.SeedCutoff()); + + Assert.DoesNotContain(oldChunk!, plan, StringComparison.Ordinal); + Assert.Contains(recentChunk!, plan, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteFamilyRowsAsync(cleanup, cleanupCt)); + } + } + + /* Naive-UTC storage by convention across the product; Npgsql 6+ rejects Kind=Utc against `timestamp`. */ + private static DateTime Naive(DateTime t) => DateTime.SpecifyKind(t, DateTimeKind.Unspecified); + + /// A second server whose only rows predate the lookback window. + private const int StaleServerId = -545454; + + private static async Task LatchAsync(NpgsqlConnection connection, int serverId, DateTime t, long requests, long waitMs, long maxWaitMs) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO latch_stats (collection_id, collection_time, server_id, server_name, latch_class, waiting_requests_count, wait_time_ms, max_wait_time_ms) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', 'BUFFER', $3, $4, $5)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(serverId); + cmd.Parameters.AddWithValue(requests); + cmd.Parameters.AddWithValue(waitMs); + cmd.Parameters.AddWithValue(maxWaitMs); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private static async Task SpinlockAsync(NpgsqlConnection connection, DateTime t, long collisions, long spins, long sleepTime, long backoffs) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO spinlock_stats (collection_id, collection_time, server_id, server_name, spinlock_name, collisions, spins, spins_per_collision, sleep_time, backoffs) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', 'LOCK_HASH', $3, $4, 10.0, $5, $6)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + cmd.Parameters.AddWithValue(collisions); + cmd.Parameters.AddWithValue(spins); + cmd.Parameters.AddWithValue(sleepTime); + cmd.Parameters.AddWithValue(backoffs); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + /// Counters are multiples of the execution count so every group's expected delta is derivable. + private static async Task ProcAsync(NpgsqlConnection connection, DateTime t, string? planHandle, string db, string schema, string obj, long executions) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO procedure_stats (collection_id, collection_time, server_id, server_name, database_name, schema_name, object_name, object_type, " + + "execution_count, total_worker_time, total_elapsed_time, total_logical_reads, total_logical_writes, total_physical_reads, total_spills, plan_handle) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', $3, $4, $5, 'P', $6, $7, $8, $9, $10, $11, $12, $13)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + cmd.Parameters.AddWithValue(db); + cmd.Parameters.AddWithValue(schema); + cmd.Parameters.AddWithValue(obj); + cmd.Parameters.AddWithValue(executions); + cmd.Parameters.AddWithValue(executions * 10); + cmd.Parameters.AddWithValue(executions * 20); + cmd.Parameters.AddWithValue(executions * 30); + cmd.Parameters.AddWithValue(executions * 40); + cmd.Parameters.AddWithValue(executions * 50); + cmd.Parameters.AddWithValue(executions * 60); + cmd.Parameters.AddWithValue((object?)planHandle ?? DBNull.Value); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private static async Task QueryStatsAsync(NpgsqlConnection connection, DateTime t) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, query_hash, sql_handle, plan_handle, execution_count) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', 'qh', 'sh', 'ph', 1)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + /// A V128 query_stats row (#3540): both offsets stored raw; the eight counters are multiples of the + /// execution count so every group's expected delta is derivable. + private static async Task KeyedQueryStatsAsync(NpgsqlConnection connection, DateTime t, string? sqlHandle, int start, int end, string? planHandle, long executions) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, query_hash, sql_handle, plan_handle, " + + "statement_start_offset, statement_end_offset, " + + "execution_count, total_worker_time, total_elapsed_time, total_logical_reads, total_logical_writes, total_physical_reads, total_rows, total_spills) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', 'qh', $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + cmd.Parameters.AddWithValue((object?)sqlHandle ?? DBNull.Value); + cmd.Parameters.AddWithValue((object?)planHandle ?? DBNull.Value); + cmd.Parameters.AddWithValue(start); + cmd.Parameters.AddWithValue(end); + cmd.Parameters.AddWithValue(executions); + cmd.Parameters.AddWithValue(executions * 10); + cmd.Parameters.AddWithValue(executions * 20); + cmd.Parameters.AddWithValue(executions * 30); + cmd.Parameters.AddWithValue(executions * 40); + cmd.Parameters.AddWithValue(executions * 50); + cmd.Parameters.AddWithValue(executions * 60); + cmd.Parameters.AddWithValue(executions * 70); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private static async Task PgWaitAsync(NpgsqlConnection connection, DateTime t, long eventId, long waits, long waitTimeUs) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO pg_wait_stats (collection_id, collection_time, server_id, server_name, wait_type_id, wait_event_id, wait_type, wait_event, waits, wait_time_us) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', 1, $3, 'IPC', 'X', $4, $5)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + cmd.Parameters.AddWithValue(eventId); + cmd.Parameters.AddWithValue(waits); + cmd.Parameters.AddWithValue(waitTimeUs); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private static async Task PgStatementAsync(NpgsqlConnection connection, DateTime t, long queryId, long databaseId, long userId, bool topLevel, long calls, double totalMs, long rows) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO pg_statement_stats (collection_id, collection_time, server_id, server_name, queryid, database_id, user_id, toplevel, calls, total_exec_time_ms, rows_returned) " + + "VALUES (1, $1, $2, 'delta-seed-e2e', $3, $4, $5, $6, $7, $8, $9)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(TestServerId); + cmd.Parameters.AddWithValue(queryId); + cmd.Parameters.AddWithValue(databaseId); + cmd.Parameters.AddWithValue(userId); + cmd.Parameters.AddWithValue(topLevel); + cmd.Parameters.AddWithValue(calls); + cmd.Parameters.AddWithValue(totalMs); + cmd.Parameters.AddWithValue(rows); + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + private static async Task DeleteFamilyRowsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) + { + foreach (var table in new[] { "wait_stats", "latch_stats", "spinlock_stats", "procedure_stats", "query_stats", "pg_wait_stats", "pg_statement_stats" }) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM {table} WHERE server_id IN ({TestServerId}, {StaleServerId})", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } + } + /// /// The chunk a planted row actually lives in. tableoid on a hypertable read resolves to the /// CHUNK, because the chunk is the table the row is stored in; on a plain table it resolves to the /// table itself, which is what lets the caller detect an unpartitioned store and skip. Returned /// unqualified, matching how EXPLAIN names relations. /// - private static async Task ChunkHoldingRowAsync(NpgsqlConnection connection, DateTime collectionTime) + private static Task ChunkHoldingRowAsync(NpgsqlConnection connection, DateTime collectionTime) + => ChunkHoldingRowAsync(connection, "wait_stats", collectionTime); + + private static async Task ChunkHoldingRowAsync(NpgsqlConnection connection, string table, DateTime collectionTime) { using var cmd = new NpgsqlCommand( - "SELECT tableoid::regclass::text FROM wait_stats WHERE server_id = $1 AND collection_time = $2", connection); + $"SELECT tableoid::regclass::text FROM {table} WHERE server_id = $1 AND collection_time = $2", connection); cmd.Parameters.AddWithValue(TestServerId); cmd.Parameters.AddWithValue(collectionTime); var name = await cmd.ExecuteScalarAsync(TestContext.Current.CancellationToken) as string; diff --git a/Darling/Darling.Tests/DarlingFleetReaderTests.cs b/Darling/Darling.Tests/DarlingFleetReaderTests.cs index 54d419822..77cfe3f09 100644 --- a/Darling/Darling.Tests/DarlingFleetReaderTests.cs +++ b/Darling/Darling.Tests/DarlingFleetReaderTests.cs @@ -193,6 +193,7 @@ public void FleetTagForestSql_SelectsTheWholeHierarchy_OrderedForStableSiblings( [InlineData(nameof(DarlingFleetReader.FleetThreadsSql))] [InlineData(nameof(DarlingFleetReader.FleetBlockingSql))] [InlineData(nameof(DarlingFleetReader.FleetDeadlockSql))] + [InlineData(nameof(DarlingFleetReader.FleetPgDeadlockSql))] [InlineData(nameof(DarlingFleetReader.FleetLastCollectionSql))] [InlineData(nameof(DarlingFleetReader.FleetCollectionHealthSql))] public void EveryFleetSql_IsPgDialect_NoTSql(string constName) @@ -230,14 +231,20 @@ public void FleetServerCard_SerializesSnakeCase_WithStringBands() CpuSeverity = HealthSeverity.Critical, MemorySeverity = HealthSeverity.Healthy, BlockingCount = 4, + BlockingRatePerHour = 4.0, BlockingSeverity = HealthSeverity.Warning, DeadlockCount = 1, DeadlockLastSeen = new DateTime(2026, 7, 18, 3, 15, 0, DateTimeKind.Unspecified), DeadlockSeverity = HealthSeverity.Critical, ThreadsSeverity = HealthSeverity.Unknown, FailedCollectorCount = 0, + CollectorCount = 40, CollectorSeverity = HealthSeverity.Healthy, OverallMetricSeverity = HealthSeverity.Critical, + /* #3528: deliberately measured < total, so the value pins below cannot pass off a card that + serialized one count under both keys. */ + MeasuredMetricCount = 1, + MetricCount = 6, }; var json = JsonSerializer.Serialize(card, DarlingFleetReader.JsonOptions); @@ -247,15 +254,23 @@ public void FleetServerCard_SerializesSnakeCase_WithStringBands() "\"server_id\"", "\"display_name\"", "\"server_name\"", "\"engine_edition\"", "\"is_azure_sql_db\"", "\"is_azure_mi\"", "\"is_silenced\"", "\"tags\"", "\"band\"", "\"status\"", "\"is_online\"", "\"last_collection\"", "\"cpu_percent\"", "\"total_cpu_percent\"", - "\"cpu_severity\"", "\"memory_severity\"", "\"blocking_count\"", "\"blocking_severity\"", + "\"cpu_severity\"", "\"memory_severity\"", "\"blocking_count\"", "\"blocking_rate_per_hour\"", + "\"blocking_severity\"", "\"deadlock_count\"", "\"deadlock_last_seen\"", "\"deadlock_rate_per_hour\"", "\"deadlock_severity\"", "\"threads_severity\"", - "\"failed_collector_count\"", "\"collector_severity\"", "\"overall_metric_severity\"", + "\"failed_collector_count\"", "\"collector_count\"", "\"collector_severity\"", "\"overall_metric_severity\"", + "\"measured_metric_count\"", "\"metric_count\"", }) { Assert.Contains(field, json, StringComparison.Ordinal); } + /* #3528: the coverage counts ride every card so a consumer can qualify the band label + ("Healthy — 1 of 6 measured") — values pinned, not just keys, so the two cannot be swapped or + collapsed into one. */ + JsonAssert.Contains("\"measured_metric_count\": 1", json); + JsonAssert.Contains("\"metric_count\": 6", json); + /* Bands / severities serialize as strings, not ordinals — the frontend maps a name to a color. */ JsonAssert.Contains("\"band\": \"Critical\"", json); JsonAssert.Contains("\"cpu_severity\": \"Critical\"", json); @@ -401,19 +416,36 @@ public void ADeadlockReaderThatReadNothing_DoesNotCount_AndNamesItsCause( => Assert.Equal(expected, FleetDeadlockCoverage.ClassifyDeadlockSource(isPostgres: false, band)); /// - /// The issue's own case: a PostgreSQL target is never covered, and its collector's band cannot change - /// that. pg_deadlocks can be perfectly HEALTHY on all fifty targets and this total still counts - /// none of it — the rows are in a different table. That is why PostgreSQL is asked before any band. + /// #3539 reversed #3017's PostgreSQL arm: a PostgreSQL target IS covered when its deadlock-source + /// collector (pg_database_stats) read, on exactly the terms a SQL Server's deadlocks + /// collector is — degraded still counts, silent and denied do not — and the covered arm is + /// PostgresTarget rather than Read only because the instrument differs (a counter + /// difference, not a graph). The pre-#3539 answer, PostgresTarget on the engine alone, would now + /// call a server whose collector never ran "counted". /// [Theory] - [InlineData(CollectorHealthClassifier.Healthy)] - [InlineData(CollectorHealthClassifier.NoPermissions)] - [InlineData(CollectorHealthClassifier.Stopped)] - [InlineData(null)] - public void APostgresTarget_IsNeverCovered_WhateverItsCollectorSays(string? band) - => Assert.Equal( - FleetDeadlockSource.PostgresTarget, - FleetDeadlockCoverage.ClassifyDeadlockSource(isPostgres: true, band)); + [InlineData(CollectorHealthClassifier.Healthy, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.Warning, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.Stale, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.Failing, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.NoPermissions, FleetDeadlockSource.CollectorDenied)] + [InlineData(CollectorHealthClassifier.Stopped, FleetDeadlockSource.CollectorSilent)] + [InlineData(CollectorHealthClassifier.NeverRun, FleetDeadlockSource.CollectorSilent)] + [InlineData(null, FleetDeadlockSource.CollectorSilent)] + public void APostgresTarget_IsCoveredOnItsOwnCollectorsTerms(string? band, FleetDeadlockSource expected) + => Assert.Equal(expected, FleetDeadlockCoverage.ClassifyDeadlockSource(isPostgres: true, band)); + + /// The one predicate both roll-ups reduce servers_read with: the two covered arms and + /// nothing else. Enumerated over the whole enum so a value added later lands uncovered by default. + [Fact] + public void ExactlyTheTwoCoveredArmsCount() + { + Assert.True(FleetDeadlockCoverage.IsCovered(FleetDeadlockSource.Read)); + Assert.True(FleetDeadlockCoverage.IsCovered(FleetDeadlockSource.PostgresTarget)); + Assert.False(FleetDeadlockCoverage.IsCovered(FleetDeadlockSource.CollectorSilent)); + Assert.False(FleetDeadlockCoverage.IsCovered(FleetDeadlockSource.CollectorDenied)); + Assert.Equal(2, Enum.GetValues().Count(FleetDeadlockCoverage.IsCovered)); + } /// /// A card that sets nothing reads as UNCOVERED, and that is the load-bearing default. DeadlockSource @@ -450,52 +482,69 @@ public void BuildRollup_ReducesCoverageFromTheCards_WithEveryCauseAttributed() { Card(1, band: CollectorHealthClassifier.Healthy), Card(2, band: CollectorHealthClassifier.Failing), - Card(3, isPostgres: true), - Card(4, isPostgres: true), + Card(3, isPostgres: true, band: CollectorHealthClassifier.Healthy), + Card(4, isPostgres: true, band: CollectorHealthClassifier.Stale), Card(5, band: CollectorHealthClassifier.Stopped), Card(6, band: CollectorHealthClassifier.NoPermissions), Card(7, band: null), + /* #3539: a PostgreSQL target whose pg_database_stats collector left no band is SILENT, not + a PostgreSQL bucket entry - the engine no longer answers on its own. */ + Card(8, isPostgres: true, band: null), }, Now, Now.AddHours(-1), Now); var coverage = rollup.DeadlockCoverage; - Assert.Equal(2, coverage.ServersRead); - Assert.Equal(7, coverage.ServersTotal); + /* Four read: two SQL Servers through their deadlocks collector, two PostgreSQL targets through + pg_database_stats (#3539). */ + Assert.Equal(4, coverage.ServersRead); + Assert.Equal(8, coverage.ServersTotal); + /* The PostgreSQL sub-count names the instrument for two of the four read. */ Assert.Equal(2, coverage.PostgresServers); - Assert.Equal(2, coverage.ServersCollectorSilent); // STOPPED + the null band + Assert.Equal(3, coverage.ServersCollectorSilent); // STOPPED + the null band + the bandless PostgreSQL target Assert.Equal(1, coverage.ServersCollectorDenied); /* Every server is accounted for exactly once — an unattributed server would mean coverage that - reports a gap it cannot explain, which is the same shape as a total that reports no denominator. */ + reports a gap it cannot explain, which is the same shape as a total that reports no denominator. + Three terms, not four: postgres_servers is a SUBSET of servers_read since #3539, and a consumer + still summing it in would over-count the fleet by every PostgreSQL target. */ Assert.Equal( coverage.ServersTotal, - coverage.ServersRead + coverage.PostgresServers - + coverage.ServersCollectorSilent + coverage.ServersCollectorDenied); + coverage.ServersRead + coverage.ServersCollectorSilent + coverage.ServersCollectorDenied); + Assert.True(coverage.PostgresServers <= coverage.ServersRead); /* And it agrees with the field the fleet already reported. */ Assert.Equal(rollup.TotalServers, coverage.ServersTotal); } /// - /// The measured case, end to end: a PostgreSQL-only fleet reports total_deadlocks: 0 with zero - /// coverage beside it, and the note sends the reader to the tool that can actually answer. + /// The measured case, end to end (#3539 reversed its direction): a PostgreSQL-only fleet whose + /// pg_database_stats collectors are running reports FULL coverage, its deadlocks summed into + /// total_deadlocks, and the note names the instrument and the tool that has the graphs — the + /// pre-#3539 sentence, "cannot count at all", is gone. /// [Fact] - public void APostgresOnlyFleet_ReportsZeroCoverage_AndNamesTheToolThatCanAnswer() + public void APostgresOnlyFleet_IsCovered_AndTheNoteNamesTheInstrument() { var rollup = DarlingFleetReader.BuildRollup( - new[] { Card(1, isPostgres: true), Card(2, isPostgres: true), Card(3, isPostgres: true) }, + new[] + { + Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy, deadlockCount: 2), + Card(2, isPostgres: true, band: CollectorHealthClassifier.Healthy), + Card(3, isPostgres: true, band: CollectorHealthClassifier.Failing, deadlockCount: 1), + }, Now, Now.AddHours(-1), Now); - Assert.Equal(0, rollup.TotalDeadlocks); - Assert.Equal(0, rollup.DeadlockCoverage.ServersRead); + Assert.Equal(3, rollup.TotalDeadlocks); + Assert.Equal(3, rollup.DeadlockCoverage.ServersRead); Assert.Equal(3, rollup.DeadlockCoverage.PostgresServers); var note = rollup.DeadlockCoverage.Note; - Assert.Contains("read a deadlock source for 0 of 3", note, StringComparison.Ordinal); + Assert.Contains("read a deadlock source for 3 of 3", note, StringComparison.Ordinal); + Assert.Contains("3 of those are PostgreSQL targets counted from the server's own pg_stat_database.deadlocks counter", note, StringComparison.Ordinal); Assert.Contains("get_pg_deadlocks", note, StringComparison.Ordinal); + Assert.DoesNotContain("cannot count", note, StringComparison.Ordinal); /* The two causes that do not apply are absent, so a reader is not handed three actions when one is called for. */ @@ -503,6 +552,29 @@ called for. */ Assert.DoesNotContain("needs a grant", note, StringComparison.Ordinal); } + /// + /// The cross-server PostgreSQL deadlock read is a per-series counter DIFFERENCE, clamped, summed — + /// pinned on the SQL's text because the alternative, SUM(deadlocks), is a plausible-looking + /// one-liner that returns a lifetime counter multiplied by the sample count (measured: eight million + /// "deadlocks" on a store with none in the window). The live test runs it; this stops a rewrite from + /// quietly reintroducing the sum. + /// + [Fact] + public void ThePostgresDeadlockRead_DifferencesTheCounterPerDatabaseSeries_AndNeverSumsTheColumn() + { + var sql = DarlingFleetReader.FleetPgDeadlockSql; + + Assert.Contains("FROM pg_database_stats", sql, StringComparison.Ordinal); + Assert.Contains("deadlocks - LAG(deadlocks) OVER (PARTITION BY server_id, database_name ORDER BY collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("SUM(GREATEST(raw_delta, 0))", sql, StringComparison.Ordinal); + Assert.Contains("MAX(collection_time) FILTER (WHERE raw_delta > 0)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(deadlocks)", sql, StringComparison.Ordinal); + /* Windowed on the partitioning column, both bounds, like the SQL Server twin. */ + Assert.Contains("collection_time >= $1", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $2", sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY server_id", sql, StringComparison.Ordinal); + } + /// /// The sentence this whole issue turns on. The two windows genuinely diverge — /// total_deadlocks is counted between the caller's bounds (one hour by default), while coverage is @@ -548,7 +620,7 @@ public void TheCoverageSerializes_BesideTheTotal_WithTheCauseAsAName() { var json = JsonSerializer.Serialize( DarlingFleetReader.BuildRollup( - new[] { Card(1, isPostgres: true), Card(2, band: CollectorHealthClassifier.Healthy) }, + new[] { Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(2, band: CollectorHealthClassifier.Healthy) }, Now, Now.AddHours(-1), Now), DarlingFleetReader.JsonOptions); @@ -564,12 +636,17 @@ public void TheCoverageSerializes_BesideTheTotal_WithTheCauseAsAName() JsonAssert.Contains("\"deadlock_source\": \"PostgresTarget\"", json); JsonAssert.Contains("\"deadlock_source\": \"Read\"", json); - JsonAssert.Contains("\"servers_read\": 1", json); + /* Both covered (#3539): servers_read counts the PostgreSQL target, and postgres_servers names it + as the counter-read one of the two. */ + JsonAssert.Contains("\"servers_read\": 2", json); + JsonAssert.Contains("\"postgres_servers\": 1", json); } private static readonly DateTime Now = new(2026, 9, 5, 12, 0, 0, DateTimeKind.Unspecified); - private static FleetServerCard Card(int id, bool isPostgres = false, string? band = null) => + /// The ENGINE'S deadlock-source collector band — deadlocks on a SQL Server, + /// pg_database_stats on a PostgreSQL target (#3539); the card carries one field for it. + private static FleetServerCard Card(int id, bool isPostgres = false, string? band = null, int deadlockCount = 0) => new() { ServerId = id, @@ -577,6 +654,7 @@ private static FleetServerCard Card(int id, bool isPostgres = false, string? ban ServerName = "target-" + id.ToString(CultureInfo.InvariantCulture), IsPostgres = isPostgres, DeadlockCollectorBand = band, + DeadlockCount = deadlockCount, }; } @@ -674,9 +752,15 @@ public async Task GetFleetOverview_PerServerFallbackAndBanding_AgainstDevPostgre await InsertServerAsync(connection, XeServerId, XeName, 5, ct); await InsertServerAsync(connection, DmvServerId, DmvName, 3, ct); - /* XE server: 2 XE reports + 1 DMV snapshot -> fallback prefers XE (count 2). */ - await InsertBlockedProcessAsync(connection, XeServerId, XeName, at, ct); - await InsertBlockedProcessAsync(connection, XeServerId, XeName, at.AddMinutes(1), ct); + /* XE server: 5 XE reports + 1 DMV snapshot -> fallback prefers XE (count 5). Five inside the + one-hour card window is 5/hr, the blocking band's Warning tier exactly (#3539 A3) — two + would be the measured quiet mode and Healthy by count, and this server has to sit in the + WARNING band for the cross-band ordering assertion below. */ + for (var i = 0; i < 5; i++) + { + await InsertBlockedProcessAsync(connection, XeServerId, XeName, at.AddSeconds(i), ct); + } + await InsertDmvBlockingAsync(connection, XeServerId, XeName, at, ct); /* DMV-only server: 3 DMV snapshots (fallback), and enough deadlocks to clear the rate tier. */ @@ -701,8 +785,12 @@ public async Task GetFleetOverview_PerServerFallbackAndBanding_AgainstDevPostgre var xe = result.Cards.Single(c => c.ServerId == XeServerId); var dmv = result.Cards.Single(c => c.ServerId == DmvServerId); - /* XE preferred: 2 events, no deadlock -> Warning band. */ - Assert.Equal(2, xe.BlockingCount); + /* XE preferred: 5 events (5.0/hr over the card's hour, the Warning tier), no deadlock -> + Warning band, and the rate rides the card beside the count (#3539 A3). */ + Assert.Equal(5, xe.BlockingCount); + Assert.Equal(5.0, xe.BlockingRatePerHour); + Assert.Equal(TimeSpan.FromHours(1), xe.BlockingWindow); + Assert.Equal(HealthSeverity.Warning, xe.BlockingSeverity); Assert.Equal(0, xe.DeadlockCount); Assert.Equal(FleetHealthBand.Warning, xe.Band); Assert.True(xe.IsOnline); @@ -712,9 +800,11 @@ public async Task GetFleetOverview_PerServerFallbackAndBanding_AgainstDevPostgre Assert.True(xe.IsAzureSqlDb); Assert.False(xe.IsAzureManagedInstance); - /* DMV fallback: 3 events -> Warning on the blocking axis, 25 deadlocks/hr -> Critical on the - deadlock axis, and worst-wins makes the card Critical. */ + /* DMV fallback: 3 snapshots (3/hr, Healthy by count — the fallback's unit is coarser and the + rows carry no wait), 25 deadlocks/hr -> Critical on the deadlock axis, and worst-wins makes + the card Critical. */ Assert.Equal(3, dmv.BlockingCount); + Assert.Equal(HealthSeverity.Healthy, dmv.BlockingSeverity); Assert.Equal(25, dmv.DeadlockCount); Assert.Equal(FleetHealthBand.Critical, dmv.Band); diff --git a/Darling/Darling.Tests/DarlingLockWaitTrendTests.cs b/Darling/Darling.Tests/DarlingLockWaitTrendTests.cs index 8d61e8a05..14c784441 100644 --- a/Darling/Darling.Tests/DarlingLockWaitTrendTests.cs +++ b/Darling/Darling.Tests/DarlingLockWaitTrendTests.cs @@ -122,13 +122,15 @@ to fix collection that is working perfectly. /* The reset row is dropped rather than charted as a negative wait. */ Assert.DoesNotContain(trend, r => r.GetProperty("wait_type").GetString() == "LCK_M_U"); - /* Four rows: two wait types x two collections. The FIRST collection of each type has no prior - sample to difference against, so its interval is NULL and its rate is 0 rather than the raw - delta — the LAG is per wait type, which is what stops one type's cadence describing another. */ - Assert.Equal(4, trend.Length); + /* Two rows: two wait types x the SECOND collection only. The FIRST collection of each type has + no prior sample to difference against and (a pre-V127 row) no stored interval, so its rate is + NULL and the row is dropped — it used to be charted as 0.00, a fabricated idle point (#3540). + The LAG is per wait type, which is what stops one type's cadence describing another. */ + Assert.Equal(2, trend.Length); + Assert.DoesNotContain(trend, r => r.GetProperty("collection_time").GetString()! + .StartsWith(first.ToString("yyyy-MM-ddTHH:mm:ss"), StringComparison.Ordinal)); Assert.Equal(100d, RateOf(trend, "LCK_M_X", second), 3); - Assert.Equal(0d, RateOf(trend, "LCK_M_X", first), 3); /* The fractional rate. Asserted as > 0 as well as by value, because "0.05" and "0" differ by a cast and the point of the assertion is that the cast is there. */ @@ -153,8 +155,9 @@ cast and the point of the assertion is that the cast is there. */ dataSource, ServerName, 1, pastSecond.ToString("yyyy-MM-ddTHH:mm:ss") + "Z")).RootElement; var anchoredRows = anchored.GetProperty("trend").EnumerateArray().ToArray(); - Assert.Equal(2, anchoredRows.Length); - Assert.All(anchoredRows, r => Assert.Equal("LCK_M_IX", r.GetProperty("wait_type").GetString())); + /* One row: the first anchored collection has no prior and is not a point (#3540). */ + var anchoredRow = Assert.Single(anchoredRows); + Assert.Equal("LCK_M_IX", anchoredRow.GetProperty("wait_type").GetString()); Assert.Equal(30d, RateOf(anchoredRows, "LCK_M_IX", pastSecond), 3); bodySucceeded = true; diff --git a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs index b02884432..51e0fbeff 100644 --- a/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpAlertToolsTests.cs @@ -20,6 +20,7 @@ using Npgsql; using PerformanceMonitor.Alerting; using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; using PerformanceMonitor.Notifications; @@ -90,7 +91,7 @@ private static (string Name, bool Optional)[] McpParams(string toolName) } [Theory] - [InlineData("get_alert_history", "server_name,hours_back,limit,as_of")] + [InlineData("get_alert_history", "server_name,hours_back,limit,as_of,include_dismissed")] [InlineData("get_mute_rules", "enabled_only")] [InlineData("update_alert_settings", "settings_json")] [InlineData("create_mute_rule", "server_name,metric_name,database_pattern,query_text_pattern,wait_type_pattern,job_name_pattern,reason,expires_at")] @@ -137,11 +138,15 @@ public void ParamContract_WriteTools_RequireTheirTarget(string toolName, string /* ---------------- read SQL pins ---------------- */ [Fact] - public void AlertHistorySql_ReadsLog_ExcludesDismissed_ServerScoped() + public void AlertHistorySql_ReadsLog_ExcludesDismissedByDefault_ServerScoped() { var sql = Reader.AlertHistorySql; Assert.Contains("FROM config_alert_log", sql, StringComparison.Ordinal); - Assert.Contains("dismissed = FALSE", sql, StringComparison.Ordinal); + /* #3541 A3: the exclusion is the DEFAULT arm of a caller's choice, not a hidden constant — $5 lifts + it. The literal stays so the grid's read and this one keep saying the same words. */ + Assert.Contains("(dismissed = FALSE OR $5)", sql, StringComparison.Ordinal); + /* And the row SAYS which population it belongs to, so an include-dismissed page can label each row. */ + Assert.Contains("dismissed\n", sql.Replace("\r\n", "\n"), StringComparison.Ordinal); /* #2495: BOTH window edges are bound, so server_id and the cap moved up one ordinal each. */ Assert.Contains("alert_time >= $1", sql, StringComparison.Ordinal); Assert.Contains("alert_time <= $2", sql, StringComparison.Ordinal); @@ -151,16 +156,93 @@ public void AlertHistorySql_ReadsLog_ExcludesDismissed_ServerScoped() } [Fact] - public void AlertHistoryAllServersSql_ReadsLog_ExcludesDismissed_NoServerFilter() + public void AlertHistoryAllServersSql_ReadsLog_ExcludesDismissedByDefault_NoServerFilter() { var sql = Reader.AlertHistoryAllServersSql; Assert.Contains("FROM config_alert_log", sql, StringComparison.Ordinal); - Assert.Contains("dismissed = FALSE", sql, StringComparison.Ordinal); + Assert.Contains("(dismissed = FALSE OR $4)", sql, StringComparison.Ordinal); Assert.DoesNotContain("server_id =", sql, StringComparison.Ordinal); /* fleet-wide */ Assert.Contains("alert_time <= $2", sql, StringComparison.Ordinal); /* #2495 upper edge */ Assert.Contains("LIMIT $3", sql, StringComparison.Ordinal); } + /// + /// #3541 A3: the hidden filter is MEASURED, not just disclosed. The count reads the same table over the + /// same window and scope as the history read, with the predicate inverted, so it is exactly the rows the + /// default read removed — and it must NOT carry a LIMIT, or a busy window would under-count what it hid. + /// + [Theory] + [InlineData(nameof(Reader.DismissedAlertCountSql), true)] + [InlineData(nameof(Reader.DismissedAlertCountAllServersSql), false)] + public void DismissedAlertCountSql_CountsTheRowsTheDefaultReadHides_Unbounded(string sqlName, bool serverScoped) + { + var sql = sqlName == nameof(Reader.DismissedAlertCountSql) ? Reader.DismissedAlertCountSql : Reader.DismissedAlertCountAllServersSql; + + Assert.Contains("SELECT COUNT(*)", sql, StringComparison.Ordinal); + Assert.Contains("FROM config_alert_log", sql, StringComparison.Ordinal); + Assert.Contains("alert_time >= $1", sql, StringComparison.Ordinal); + Assert.Contains("alert_time <= $2", sql, StringComparison.Ordinal); + Assert.Contains("dismissed = TRUE", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT", sql, StringComparison.Ordinal); + Assert.Equal(serverScoped, sql.Contains("server_id = $3", StringComparison.Ordinal)); + } + + /// + /// include_dismissed is an APPENDED optional, after as_of, defaulting to the grid's own read + /// — a caller who never sends it reads what they always read, the /api/read dispatch (which passes + /// as_of by name) is untouched, and no positional C# caller is re-bound. The description must name + /// the filter in both directions: what it hides and how to lift it. + /// + [Fact] + public void GetAlertHistory_IncludeDismissed_IsAnAppendedOptional_AndTheDescriptionNamesTheFilter() + { + var method = typeof(DarlingMcpAlertTools).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == "get_alert_history"); + var names = McpParams("get_alert_history").Select(p => p.Name).ToArray(); + + var flag = method.GetParameters().Single(p => p.Name == "include_dismissed"); + Assert.True(flag.HasDefaultValue); + Assert.False((bool)flag.DefaultValue!); + Assert.True(Array.IndexOf(names, "include_dismissed") > Array.IndexOf(names, "as_of")); + + var description = method.GetCustomAttribute()!.Description; + Assert.Contains("EXCLUDES DISMISSED ALERTS", description, StringComparison.Ordinal); + Assert.Contains("dismissed_excluded_count", description, StringComparison.Ordinal); + Assert.Contains("include_dismissed", description, StringComparison.Ordinal); + } + + /// + /// #3541 A14 rider (from #3594's residuals): the web mirror of this read could NOT lift the dismissed + /// filter. /api/read/get_alert_history dispatched without include_dismissed and its + /// /api/catalog entry did not advertise it, so a browser or API caller received + /// dismissed_excluded_count — "N rows were hidden" — with no wire key to un-hide them. Both halves + /// are pinned: the catalog names the parameter as an optional boolean defaulting to the tool's own default, + /// and the dispatch passes it BY NAME from the query string through the same QueryBool every other + /// optional boolean uses (source-text pin, because the dispatch is a lambda over an HttpContext and the + /// tool would need a store to observe the flag downstream). + /// + [Fact] + public void WebRead_GetAlertHistory_AdvertisesAndDispatchesIncludeDismissed() + { + var descriptor = DarlingWebEndpoints.CatalogDescriptors["get_alert_history"]; + var param = descriptor.Params.Single(p => p.Name == "include_dismissed"); + Assert.Equal("bool", param.Type); + Assert.False(param.Required); + Assert.Equal(false, param.Default); + + /* The catalog prose says what the flag does — the description is the only thing a web caller reads. */ + Assert.Contains("include_dismissed", descriptor.Description, StringComparison.Ordinal); + Assert.Contains("dismissed", descriptor.Description, StringComparison.Ordinal); + + /* And the dispatch line carries it, by name, off the query string. */ + var source = CSharpSourceWalker.StripCommentsAndStrings(ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "DarlingWebEndpoints.cs")); + var dispatchLine = source.Split('\n').Single(l => + l.Contains("DarlingMcpAlertTools.GetAlertHistory(", StringComparison.Ordinal)); + Assert.Contains("include_dismissed: QueryBool(c,", dispatchLine, StringComparison.Ordinal); + Assert.Contains("as_of: AsOf(c)", dispatchLine, StringComparison.Ordinal); + } + /// /// #2391: #2349's four knobs are readable and writable through the MCP. They reached 3.5.0 with the /// store plane only — clamped on read in DarlingAlertSettings, columns in V79 — but with no group @@ -212,6 +294,37 @@ public void FileGrowthWriteBounds_MatchTheEngineClamps() Assert.Contains("\"file_growth.lookback_minutes\", 5, 1440", tools, StringComparison.Ordinal); } + /// + /// #3539 A8c: file_growth.rise_mb became a RATE (megabytes per hour, averaged over + /// lookback_minutes) without changing its key, its column or its integer — so the wire contract is + /// pinned as UNCHANGED here: the read emits the row's value under the same key, the write accepts the same + /// key into the same column, and the meaning lives in both tool descriptions, which is where an agent reads + /// it. A rename would have broken every client that reads or writes the setting to say something the + /// description says just as well. + /// + [Fact] + public void FileGrowthRise_KeepsItsKeyAndColumn_AndBothDescriptionsSayItIsPerHour() + { + var payload = SerializedSettingsPayload(SampleSettingsRow()); + Assert.Equal(1024, payload["file_growth"]!["rise_mb"]!.GetValue()); + Assert.Equal(60, payload["file_growth"]!["lookback_minutes"]!.GetValue()); + + var (targets, error) = ParseAsPartialUpdate((JsonObject)JsonNode.Parse( + "{\"file_growth\":{\"rise_mb\":2048}}")!); + Assert.Null(error); + Assert.Equal(new[] { (DarlingMcpAlertTools.AlertSettingsTable, "file_growth_rise_mb") }, targets.ToArray()); + + /* The unit, on both descriptions — read off the attributes the MCP host serves, the way the + get_alert_history pin above reads its own. The census in FileGrowthRiseUnitCensusTests holds the + phrase across every surface; this is the MCP half stated where the MCP contract is pinned. */ + Assert.Contains("rise_mb is megabytes per HOUR", ToolDescription("get_alert_settings"), StringComparison.Ordinal); + Assert.Contains("file_growth.rise_mb is megabytes per HOUR", ToolDescription("update_alert_settings"), StringComparison.Ordinal); + } + + private static string ToolDescription(string toolName) => + ToolMethods().Single(m => m.GetCustomAttribute()!.Name == toolName) + .GetCustomAttribute()!.Description; + /// /// The SELECT's column count and the positional read must agree. Every field on /// AlertSettingsReadRow is read by ORDINAL, so a column inserted anywhere but the end re-maps @@ -748,7 +861,11 @@ SQL Server numbers above (3 and 5) — equal pairs would let a payload that emit /* #3466 (V124): inside the write bound [15, 1440], deliberately NOT the shipped 60 — a sample equal to the default would let a payload that dropped the column and fell back to the default still match — and enabled deliberately FALSE against the shipped TRUE for the same reason. */ - FleetSweepEnabled: false, FleetSweepIntervalMinutes: 240); + FleetSweepEnabled: false, FleetSweepIntervalMinutes: 240, + /* #3528 (V126): inside the write bound (>= 0), deliberately NOT the shipped 50 — a sample equal + to the default would let a payload that dropped the column and fell back to the default still + match. */ + SelfDiskFreeWarnGb: 75); [Fact] public void AlertSettingsSql_ReadsSingleGlobalRow() @@ -764,6 +881,8 @@ public void AlertSettingsSql_ReadsSingleGlobalRow() [Theory] [InlineData(nameof(Reader.AlertHistorySql))] [InlineData(nameof(Reader.AlertHistoryAllServersSql))] + [InlineData(nameof(Reader.DismissedAlertCountSql))] + [InlineData(nameof(Reader.DismissedAlertCountAllServersSql))] [InlineData(nameof(Reader.AlertSettingsSelectSql))] public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { @@ -771,6 +890,8 @@ public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { nameof(Reader.AlertHistorySql) => Reader.AlertHistorySql, nameof(Reader.AlertHistoryAllServersSql) => Reader.AlertHistoryAllServersSql, + nameof(Reader.DismissedAlertCountSql) => Reader.DismissedAlertCountSql, + nameof(Reader.DismissedAlertCountAllServersSql) => Reader.DismissedAlertCountAllServersSql, _ => Reader.AlertSettingsSelectSql, }; var lower = sql.ToLowerInvariant(); @@ -1834,6 +1955,20 @@ await DarlingMcpTestData.ExecAsync(connection, ct, VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", when, ServerId, ServerName, "High CPU", 92.5, 80.0, true, "email", null, false, "CPU sustained above threshold"); + /* #3539 A8e: a Poison Wait row that FIRED Warning, with the tier persisted the way both SKUs' + deliverers persist it (the serializer's Severity member), and a legacy Deadlocks row carrying + no context at all. */ + var gradedContext = new AlertContext { SeverityOverride = AlertSeverityLevel.Warning }; + gradedContext.Details.Add(new AlertDetailItem { Heading = "THREADPOOL" }); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO config_alert_log (alert_time, server_id, server_name, metric_name, current_value, threshold_value, alert_sent, notification_type, send_error, muted, detail_text, context_json) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", + when.AddMinutes(-1), ServerId, ServerName, "Poison Wait", 61000.0, 60000.0, true, "webhook", null, false, "THREADPOOL", AlertContextSerializer.Serialize(gradedContext)); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO config_alert_log (alert_time, server_id, server_name, metric_name, current_value, threshold_value, alert_sent, notification_type, send_error, muted, detail_text) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", + when.AddMinutes(-2), ServerId, ServerName, "Deadlocks Detected", 1.0, 1.0, true, "email", null, false, null); + /* Seed the single global settings row — every column has a default, so id alone suffices. BOTH singletons, because #3314 made get_alert_settings read the delivery cooldown off config_notification: the service seeds the two in one pass, and the tool reports `unavailable` @@ -1852,6 +1987,29 @@ await DarlingMcpTestData.ExecAsync(connection, ct, var scoped = await DarlingMcpAlertTools.GetAlertHistory(postgres, ServerName); DarlingMcpTestData.AssertEnvelope(scoped, ServerName, "alerts"); Assert.Contains("High CPU", scoped, StringComparison.Ordinal); + /* #3541 A3: three planted, undismissed rows — the page says so, says the filter applied and hid + nothing, and carries no `total_` key. */ + JsonAssert.Contains("\"alerts_returned\": 3", scoped); + JsonAssert.Contains("\"truncated\": false", scoped); + JsonAssert.Contains("\"dismissed_excluded\": true", scoped); + JsonAssert.Contains("\"dismissed_excluded_count\": 0", scoped); + Assert.DoesNotContain("total_alerts", scoped, StringComparison.Ordinal); + + /* #3539 A8e: the tier the alert FIRED at, per row, and where it came from. The Poison Wait row + reads the Warning it fired at off its context ("fired") — not the red its name implies — while + the two rows with no context are classified by name and say so. Asserted on the row objects + rather than by substring, so a "warning" from one row cannot satisfy a pin about another. */ + using (var page = JsonDocument.Parse(scoped)) + { + var byMetric = page.RootElement.GetProperty("alerts").EnumerateArray() + .ToDictionary(a => a.GetProperty("metric_name").GetString()!, a => a); + Assert.Equal("warning", byMetric["Poison Wait"].GetProperty("severity").GetString()); + Assert.Equal(AlertHistoryRowSeverity.SourceFired, byMetric["Poison Wait"].GetProperty("severity_source").GetString()); + Assert.Equal("critical", byMetric["Deadlocks Detected"].GetProperty("severity").GetString()); + Assert.Equal(AlertHistoryRowSeverity.SourceMetricName, byMetric["Deadlocks Detected"].GetProperty("severity_source").GetString()); + Assert.Equal("warning", byMetric["High CPU"].GetProperty("severity").GetString()); + Assert.Equal(AlertHistoryRowSeverity.SourceMetricName, byMetric["High CPU"].GetProperty("severity_source").GetString()); + } var fleet = await DarlingMcpAlertTools.GetAlertHistory(postgres); Assert.False(fleet.StartsWith("Error during", StringComparison.Ordinal), fleet); diff --git a/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs b/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs index 47182387e..8979288fb 100644 --- a/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpBlockingToolsTests.cs @@ -189,6 +189,33 @@ public void BlockedProcessReportsSql_ReadsBaseTable_XmlAndPairColumns_WindowsOnC Assert.Contains("blocking_spid", sql, StringComparison.Ordinal); Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY event_time DESC", sql, StringComparison.Ordinal); + /* #3541 A3: the cap is the CALLER'S ($4), not the 200 the reader used to hide under a tool that + advertised `limit`. */ + Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT 200", sql, StringComparison.Ordinal); + } + + /// + /// #3541 A3: get_blocked_process_xml pages over rows that CARRY a report, and the predicate is in + /// the SQL — filtering for XML in C# after a capped fetch was the defect (a run of graph-less rows at the + /// newest end read as "no XML in the window"). Pinned as the SAME projection as the unfiltered read plus + /// exactly the predicate, so the two consts cannot drift a column apart. + /// + [Fact] + public void BlockedProcessReportsWithXmlSql_IsTheUnfilteredRead_PlusTheXmlPredicate_InSql() + { + var plain = DarlingBlockingReader.BlockedProcessReportsSql; + var withXml = DarlingBlockingReader.BlockedProcessReportsWithXmlSql; + + Assert.Contains("AND blocked_process_report_xml IS NOT NULL", withXml, StringComparison.Ordinal); + Assert.Contains("AND blocked_process_report_xml <> ''", withXml, StringComparison.Ordinal); + Assert.DoesNotContain("blocked_process_report_xml IS NOT NULL", plain, StringComparison.Ordinal); + + /* Everything up to the window predicate is byte-identical. */ + const string Cut = "AND collection_time <= $3"; + Assert.Equal(plain[..(plain.IndexOf(Cut, StringComparison.Ordinal) + Cut.Length)], + withXml[..(withXml.IndexOf(Cut, StringComparison.Ordinal) + Cut.Length)]); + Assert.EndsWith("ORDER BY event_time DESC\nLIMIT $4", withXml.Replace("\r\n", "\n"), StringComparison.Ordinal); } [Fact] @@ -199,6 +226,8 @@ public void DmvBlockingSnapshotsSql_ReadsView_NoXmlColumn() Assert.DoesNotContain("blocked_process_report_xml", sql, StringComparison.Ordinal); /* the DMV snapshot has no report XML */ Assert.Contains("contentious_object", sql, StringComparison.Ordinal); Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT 200", sql, StringComparison.Ordinal); } [Fact] @@ -210,6 +239,40 @@ public void RecentDeadlocksSql_ReadsBaseTable_GraphXml_OrdersByDeadlockTime() Assert.Contains("deadlock_graph_xml", sql, StringComparison.Ordinal); Assert.Contains("victim_process_id", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY deadlock_time DESC", sql, StringComparison.Ordinal); + /* #3541 A3: the cap is the caller's, not the 50 a caller asking for 100 deadlocks never saw. */ + Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT 50", sql, StringComparison.Ordinal); + } + + /// Same shape as the blocked-process pair: get_deadlock_detail's limit counts graphs + /// because the graph predicate is in the SQL, and the two consts share one body. + [Fact] + public void RecentDeadlocksWithGraphSql_IsTheUnfilteredRead_PlusTheGraphPredicate_InSql() + { + var plain = DarlingBlockingReader.RecentDeadlocksSql; + var withGraph = DarlingBlockingReader.RecentDeadlocksWithGraphSql; + + Assert.Contains("AND deadlock_graph_xml IS NOT NULL", withGraph, StringComparison.Ordinal); + Assert.Contains("AND deadlock_graph_xml <> ''", withGraph, StringComparison.Ordinal); + Assert.DoesNotContain("deadlock_graph_xml IS NOT NULL", plain, StringComparison.Ordinal); + + const string Cut = "AND collection_time <= $3"; + Assert.Equal(plain[..(plain.IndexOf(Cut, StringComparison.Ordinal) + Cut.Length)], + withGraph[..(withGraph.IndexOf(Cut, StringComparison.Ordinal) + Cut.Length)]); + Assert.EndsWith("ORDER BY deadlock_time DESC\nLIMIT $4", withGraph.Replace("\r\n", "\n"), StringComparison.Ordinal); + } + + /// + /// The fingerprint scan ceiling (#3541 A3): #2159 promised the dedup_key filter runs over the window + /// BEFORE limit, and a hidden 200-row cap was quietly breaking it. The ceiling has to be materially + /// wider than that cap or the promise is still hollow, and bounded because a scan row carries the graph + /// or both SQL texts; 5,000 is what the analysis pair-row readers fetch WITHOUT the XML. + /// + [Fact] + public void FingerprintScanCeiling_IsWiderThanTheOldHiddenCap_AndBounded() + { + Assert.True(DarlingBlockingReader.FingerprintScanCeiling >= 1000, "the scan ceiling is not materially wider than the 200-row cap #2159's promise was hollow under"); + Assert.True(DarlingBlockingReader.FingerprintScanCeiling <= 5000, "the scan carries XML per row; the analysis readers fetch 5,000 WITHOUT it"); } [Fact] @@ -253,6 +316,12 @@ public void LockWaitTrendSql_FiltersLockWaits_LagsPerWaitType_AndDividesAsDouble Assert.Contains("wait_type LIKE 'LCK%'", sql, StringComparison.Ordinal); Assert.Contains("LAG(collection_time) OVER (PARTITION BY wait_type ORDER BY collection_time)", sql, StringComparison.Ordinal); Assert.Contains("CAST(delta_wait_time_ms AS double precision) / interval_seconds", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first (0, the unknowable marker, → NULL through NULLIF); the LAG only for + pre-V127 rows; no ELSE 0 on the rate, so an unknowable interval reads NULL and never 0.00. */ + Assert.Contains("CASE WHEN sample_interval_seconds IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(sample_interval_seconds, 0)", sql, StringComparison.Ordinal); + Assert.Contains("END AS interval_seconds", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0 END", sql, StringComparison.Ordinal); /* A negative delta is the counter reset across a restart, not a negative wait. */ Assert.Contains("WHERE delta_wait_time_ms >= 0", sql, StringComparison.Ordinal); @@ -280,14 +349,18 @@ public void LockWaitTrend_AnchorCarriesTheSharedDescription() [Theory] [InlineData(nameof(DarlingBlockingReader.BlockedProcessReportsSql))] + [InlineData(nameof(DarlingBlockingReader.BlockedProcessReportsWithXmlSql))] [InlineData(nameof(DarlingBlockingReader.DmvBlockingSnapshotsSql))] [InlineData(nameof(DarlingBlockingReader.RecentDeadlocksSql))] + [InlineData(nameof(DarlingBlockingReader.RecentDeadlocksWithGraphSql))] public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) { var sql = sqlName switch { nameof(DarlingBlockingReader.BlockedProcessReportsSql) => DarlingBlockingReader.BlockedProcessReportsSql, + nameof(DarlingBlockingReader.BlockedProcessReportsWithXmlSql) => DarlingBlockingReader.BlockedProcessReportsWithXmlSql, nameof(DarlingBlockingReader.DmvBlockingSnapshotsSql) => DarlingBlockingReader.DmvBlockingSnapshotsSql, + nameof(DarlingBlockingReader.RecentDeadlocksWithGraphSql) => DarlingBlockingReader.RecentDeadlocksWithGraphSql, _ => DarlingBlockingReader.RecentDeadlocksSql, }; var lower = sql.ToLowerInvariant(); @@ -397,6 +470,11 @@ await DarlingMcpTestData.ExecAsync(connection, ct, var blocking = await DarlingMcpBlockingTools.GetBlocking(postgres, ServerName); DarlingMcpTestData.AssertEnvelope(blocking, ServerName, "events"); Assert.Contains("dbo.Posts", blocking, StringComparison.Ordinal); + /* #3541 A3: two planted rows (one XE, one DMV on a different pair) merge to a two-row page well + under the default limit, so the page says so — and no `total_` key is on it. */ + JsonAssert.Contains("\"events_returned\": 2", blocking); + JsonAssert.Contains("\"truncated\": false", blocking); + Assert.DoesNotContain("total_events", blocking, StringComparison.Ordinal); DarlingMcpTestData.AssertEnvelope(await DarlingMcpBlockingTools.GetDeadlocks(postgres, ServerName), ServerName, "deadlocks"); var detail = await DarlingMcpBlockingTools.GetDeadlockDetail(postgres, ServerName); diff --git a/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs b/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs index 603b9308a..642248029 100644 --- a/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpConfigHistoryToolsTests.cs @@ -143,7 +143,7 @@ public void QueryStoreHealthSql_LatestSnapshot_SelectsPayloadOrder() Assert.Contains("MAX(capture_time)", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY database_name", sql, StringComparison.Ordinal); Assert.Contains( - "database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes", + "database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes, capture_time", sql, StringComparison.Ordinal); } diff --git a/Darling/Darling.Tests/DarlingMcpCustomAlertToolsTests.cs b/Darling/Darling.Tests/DarlingMcpCustomAlertToolsTests.cs index 81667d3c7..e6e6854a0 100644 --- a/Darling/Darling.Tests/DarlingMcpCustomAlertToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpCustomAlertToolsTests.cs @@ -313,12 +313,30 @@ await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 1, enabled: Assert.Equal(name, paused.RootElement.GetProperty("name").GetString()); } - /* PARTIAL update at version 2 - swap the definition only. enabled (false) must be preserved, version 3. */ + /* PARTIAL update at version 2 - swap the definition only. enabled (false) must be preserved, version 3. + #3541 A14: so must the OMITTED description - the half of the shared vocabulary this tool always had. */ using (var redefined = JsonDocument.Parse( await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 2, definition: GoodRuleV2))) { Assert.Equal(3, redefined.RootElement.GetProperty("version").GetInt32()); Assert.False(redefined.RootElement.GetProperty("enabled").GetBoolean()); + Assert.Equal("made over MCP", redefined.RootElement.GetProperty("description").GetString()); + } + + /* #3541 A14: the half it lacked - an EMPTY string clears the description (version 4), the same + explicit clear update_custom_view now takes; and the next omission keeps the cleared null (5). */ + using (var cleared = JsonDocument.Parse( + await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 3, description: ""))) + { + Assert.Equal(4, cleared.RootElement.GetProperty("version").GetInt32()); + Assert.Equal(JsonValueKind.Null, cleared.RootElement.GetProperty("description").ValueKind); + } + + using (var stillCleared = JsonDocument.Parse( + await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 4, enabled: false))) + { + Assert.Equal(5, stillCleared.RootElement.GetProperty("version").GetInt32()); + Assert.Equal(JsonValueKind.Null, stillCleared.RootElement.GetProperty("description").ValueKind); } /* stale update (still presenting version 1) - conflict, not a silent clobber. */ @@ -331,7 +349,7 @@ await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 2, definiti /* update with an INVALID definition on a live row - invalid, and the row is untouched. */ Assert.Equal("invalid", DarlingMcpTestData.StatusOf( - await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 3, definition: BadMeasureRule))); + await DarlingMcpCustomAlertTools.UpdateCustomAlertRule(postgres, id, 5, definition: BadMeasureRule))); /* delete - deleted; then get + delete-again - not_found. */ Assert.Equal("deleted", DarlingMcpTestData.StatusOf(await DarlingMcpCustomAlertTools.DeleteCustomAlertRule(postgres, id))); diff --git a/Darling/Darling.Tests/DarlingMcpCustomViewToolsTests.cs b/Darling/Darling.Tests/DarlingMcpCustomViewToolsTests.cs index b207cfa0c..f58a8271e 100644 --- a/Darling/Darling.Tests/DarlingMcpCustomViewToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpCustomViewToolsTests.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Reflection; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using ModelContextProtocol.Server; @@ -183,6 +184,92 @@ public async Task UpdateCustomView_InvalidDefinition_ReturnsInvalid_WithoutTouch Assert.Equal("invalid", DarlingMcpTestData.StatusOf(result)); } + /* ---------------- #3541 A14: one write vocabulary for an optional text field ---------------- */ + + /// + /// The rule itself, as a truth table: omitted (null) keeps the current value — including a current null — + /// an empty or whitespace-only string clears, and text replaces. The helper is the alert-rule tool's, and + /// the census below pins that the view tool calls the same one. + /// + [Theory] + [InlineData(null, "kept", "kept")] + [InlineData(null, null, null)] + [InlineData("", "kept", null)] + [InlineData(" ", "kept", null)] + [InlineData("new", "kept", "new")] + [InlineData("new", null, "new")] + public void ResolveOptionalText_OmittedKeeps_EmptyClears_TextReplaces(string? sent, string? current, string? expected) + { + Assert.Equal(expected, DarlingMcpCustomAlertTools.ResolveOptionalText(sent, current)); + } + + /// + /// Both update tools route their optional description through the ONE helper — the cross-tool census the + /// contract asks for. Before this, the two tools disagreed (the view tool wrote an omitted description as + /// NULL; the rule tool kept it), and a reader of either description had no way to know which rule the other + /// followed. Read off the stripped source so a comment naming the helper cannot satisfy it; each tool's + /// UpdateAsync call must pass the helper's result where its description argument goes. + /// + [Fact] + public void BothUpdateTools_RouteDescriptionThroughTheSharedVocabulary() + { + var viewSource = CSharpSourceWalker.StripCommentsAndStrings(RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpCustomViewTools.cs")); + var ruleSource = CSharpSourceWalker.StripCommentsAndStrings(RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpCustomAlertTools.cs")); + + /* The view tool: reads the row first (there is no "current" to keep without it), then passes the + helper's result — never the bare parameter — to the store. */ + var viewUpdate = viewSource[viewSource.IndexOf("UpdateCustomView(", StringComparison.Ordinal)..]; + viewUpdate = viewUpdate[..viewUpdate.IndexOf("DeleteCustomView(", StringComparison.Ordinal)]; + Assert.Contains("store.GetAsync(view_id)", viewUpdate, StringComparison.Ordinal); + Assert.Contains("DarlingMcpCustomAlertTools.ResolveOptionalText(description, currentOk.View.Description)", viewUpdate, StringComparison.Ordinal); + Assert.DoesNotMatch(@"UpdateAsync\(\s*view_id,\s*name,\s*description,", viewUpdate); + + /* The rule tool: the same helper over its own current row. */ + var ruleUpdate = ruleSource[ruleSource.IndexOf("UpdateCustomAlertRule(", StringComparison.Ordinal)..]; + ruleUpdate = ruleUpdate[..ruleUpdate.IndexOf("DeleteCustomAlertRule(", StringComparison.Ordinal)]; + Assert.Contains("ResolveOptionalText(description, row.Description)", ruleUpdate, StringComparison.Ordinal); + Assert.DoesNotContain("description ?? row.Description", ruleUpdate, StringComparison.Ordinal); + + /* And the helper is declared exactly once, in the rule tool (the newer contract's home). */ + Assert.Single(Regex.Matches(ruleSource, @"internal static string\? ResolveOptionalText\(")); + Assert.Empty(Regex.Matches(viewSource, @"static string\? ResolveOptionalText\(")); + } + + /// The two descriptions and their two `description` parameter descriptions tell the SAME story, in + /// the words a caller will search for. A vocabulary shared in code and not in prose is shared with nobody. + [Fact] + public void BothUpdateTools_DescribeTheSameDescriptionVocabulary() + { + static (string Tool, string Param) Prose(string toolName) + { + var method = typeof(T) + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == toolName); + var tool = method.GetCustomAttribute()!.Description; + var param = method.GetParameters().Single(p => p.Name == "description").GetCustomAttribute()!.Description; + return (tool, param); + } + + var view = Prose("update_custom_view"); + var rule = Prose("update_custom_alert_rule"); + + foreach (var (tool, param) in new[] { view, rule }) + { + Assert.Contains("write vocabulary", tool, StringComparison.Ordinal); + Assert.Contains("Omit to keep the current description; send an empty string \"\" to clear it.", param, StringComparison.Ordinal); + } + + /* Each names the other, so a reader of one is pointed at the shared rule. */ + Assert.Contains("update_custom_alert_rule", view.Tool, StringComparison.Ordinal); + Assert.Contains("update_custom_view", rule.Tool, StringComparison.Ordinal); + + /* The release-old denial is gone: the rule tool no longer says the description cannot be cleared. */ + Assert.DoesNotContain("cannot clear it", rule.Param, StringComparison.Ordinal); + Assert.DoesNotContain("full replacement of name/description/definition", view.Tool, StringComparison.Ordinal); + } + [Fact] public async Task RunCustomViewPanel_BadJson_ReturnsInvalid_WithoutTouchingTheStore() { @@ -321,6 +408,23 @@ await DarlingMcpCustomViewTools.CreateCustomView(postgres, name, GoodDashboard, await DarlingMcpCustomViewTools.UpdateCustomView(postgres, id, name, GoodDashboardV2, 1, "edited over MCP"))) { Assert.Equal(2, updated.RootElement.GetProperty("version").GetInt32()); + Assert.Equal("edited over MCP", updated.RootElement.GetProperty("description").GetString()); + } + + /* #3541 A14: an OMITTED description is unchanged — this exact call used to write NULL. Version 3. */ + using (var kept = JsonDocument.Parse( + await DarlingMcpCustomViewTools.UpdateCustomView(postgres, id, name, GoodDashboardV2, 2))) + { + Assert.Equal(3, kept.RootElement.GetProperty("version").GetInt32()); + Assert.Equal("edited over MCP", kept.RootElement.GetProperty("description").GetString()); + } + + /* ... and an EMPTY string is the explicit clear. Version 4. */ + using (var cleared = JsonDocument.Parse( + await DarlingMcpCustomViewTools.UpdateCustomView(postgres, id, name, GoodDashboardV2, 3, ""))) + { + Assert.Equal(4, cleared.RootElement.GetProperty("version").GetInt32()); + Assert.Equal(JsonValueKind.Null, cleared.RootElement.GetProperty("description").ValueKind); } /* stale update (still presenting version 1) — conflict, not a silent clobber. */ diff --git a/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs b/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs index 6aaf61d39..465b963e6 100644 --- a/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpDataToolsTests.cs @@ -556,6 +556,10 @@ public void WaitStatsSql_AggregatesDeltas_HeaviestFirst() Assert.Contains("SUM(delta_waiting_tasks)", sql, StringComparison.Ordinal); Assert.Contains("GROUP BY wait_type", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY SUM(delta_wait_time_ms) DESC", sql, StringComparison.Ordinal); + /* #3541 A3: the cap is the caller's ($4), not the 50 that sat under a limit the tool accepts up to + 1,000 — the shape DarlingPgWaitReader already fixed for the PostgreSQL twin. */ + Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT 50", sql, StringComparison.Ordinal); } [Fact] @@ -566,6 +570,12 @@ public void WaitTrendSql_PerSecondRate_ForOneType() Assert.Contains("wait_type = $2", sql, StringComparison.Ordinal); Assert.Contains("LAG(collection_time)", sql, StringComparison.Ordinal); Assert.Contains("wait_time_ms_per_second", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first (0, the unknowable marker, → NULL through NULLIF); the LAG only for + pre-V127 rows; no ELSE 0 on the rate, so an unknowable interval reads NULL and never 0.00. */ + Assert.Contains("CASE WHEN sample_interval_seconds IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(sample_interval_seconds, 0)", sql, StringComparison.Ordinal); + Assert.Contains("END AS interval_seconds", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0 END", sql, StringComparison.Ordinal); } [Fact] @@ -592,6 +602,22 @@ public void FileIoSql_LatestSnapshot_CarriesRawDeltas() Assert.Contains("delta_stall_write_ms", sql, StringComparison.Ordinal); Assert.Contains("delta_reads", sql, StringComparison.Ordinal); Assert.Contains("MAX(collection_time)", sql, StringComparison.Ordinal); + /* #3540: the interval rides along so the tool can report latency as null on the unknowable marker. */ + Assert.Contains("sample_interval_seconds", sql, StringComparison.Ordinal); + } + + /// #3540: a file whose latest row is the calculator's unknowable marker (stored interval 0) reports + /// null latencies, not "0.00 ms"; a pre-V127 row (NULL interval) and a measured row keep the stall/op reading. + [Fact] + public void FileIoRow_ReportsUnknowable_OnlyForAStoredZeroInterval() + { + static DarlingDataReader.FileIoRow Row(int? interval) => new( + "AppDb", "AppDb_data", "ROWS", "D:\\AppDb.mdf", 100, DeltaReads: 0, DeltaWrites: 0, DeltaReadBytes: 0, + DeltaWriteBytes: 0, DeltaStallReadMs: 0, DeltaStallWriteMs: 0, SampleIntervalSeconds: interval); + + Assert.True(Row(0).IsUnknowable); + Assert.False(Row(null).IsUnknowable); + Assert.False(Row(60).IsUnknowable); } [Fact] @@ -648,7 +674,28 @@ public void TopProceduresSql_AggregatesDeltas_OptionalDbFilter_ReadsBaseTable() Assert.Contains("FROM procedure_stats", sql, StringComparison.Ordinal); Assert.Contains("GROUP BY database_name, schema_name, object_name, object_type", sql, StringComparison.Ordinal); Assert.Contains("$5::text IS NULL OR database_name = $5", sql, StringComparison.Ordinal); - Assert.Contains("SUM(delta_elapsed_time) DESC", sql, StringComparison.Ordinal); + Assert.Contains("SUM(delta_worker_time) DESC", sql, StringComparison.Ordinal); + } + + /* #3523: every by-CPU read RANKED by summed elapsed time — on a wait-bound server the real CPU + consumers could be absent from the page entirely, and attributed_cpu_ratio then read as "hidden + CPU" when it meant "wrong sort key". Both the ranking cut (the CTE's ORDER BY ... LIMIT) and the + post-WAITFOR-trim final ordering must key on CPU; the viewer's Duration grids keep their elapsed + ranking by design and are pinned separately in ViewerQueriesTests. */ + [Theory] + [InlineData(nameof(DarlingDataReader.TopQueriesSql))] + [InlineData(nameof(DarlingDataReader.TopQueriesByHostObjectSql))] + [InlineData(nameof(DarlingDataReader.TopProceduresSql))] + public void ByCpuReads_RankByWorkerTime_NeverElapsed(string sqlName) + { + var sql = SqlByName(sqlName); + Assert.Contains("ORDER BY SUM(delta_worker_time) DESC", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(delta_elapsed_time) DESC", sql, StringComparison.Ordinal); + Assert.DoesNotContain("total_elapsed_us DESC", sql, StringComparison.Ordinal); + if (sqlName != nameof(DarlingDataReader.TopProceduresSql)) + { + Assert.Contains("ORDER BY r.total_cpu_us DESC", sql, StringComparison.Ordinal); + } } [Fact] @@ -727,6 +774,7 @@ exactly like the viewer's UTC-offset read. */ [InlineData(nameof(DarlingDataReader.TempDbTrendSql))] [InlineData(nameof(DarlingDataReader.LatestPerfmonStatsSql))] [InlineData(nameof(DarlingDataReader.TopQueriesSql))] + [InlineData(nameof(DarlingDataReader.TopQueriesByHostObjectSql))] [InlineData(nameof(DarlingDataReader.TopProceduresSql))] [InlineData(nameof(DarlingDataReader.QueryStoreTopSql))] [InlineData(nameof(DarlingDataReader.ServerListSql))] @@ -757,6 +805,7 @@ public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) nameof(DarlingDataReader.TempDbTrendSql) => DarlingDataReader.TempDbTrendSql, nameof(DarlingDataReader.LatestPerfmonStatsSql) => DarlingDataReader.LatestPerfmonStatsSql, nameof(DarlingDataReader.TopQueriesSql) => DarlingDataReader.TopQueriesSql, + nameof(DarlingDataReader.TopQueriesByHostObjectSql) => DarlingDataReader.TopQueriesByHostObjectSql, nameof(DarlingDataReader.TopProceduresSql) => DarlingDataReader.TopProceduresSql, nameof(DarlingDataReader.QueryStoreTopSql) => DarlingDataReader.QueryStoreTopSql, nameof(DarlingDataReader.ServerListSql) => DarlingDataReader.ServerListSql, diff --git a/Darling/Darling.Tests/DarlingMcpHealthParserToolsTests.cs b/Darling/Darling.Tests/DarlingMcpHealthParserToolsTests.cs index aa5fc6216..65a996b25 100644 --- a/Darling/Darling.Tests/DarlingMcpHealthParserToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpHealthParserToolsTests.cs @@ -12,6 +12,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using ModelContextProtocol.Server; @@ -113,27 +114,44 @@ public void DatabaseNameMapSql_LatestNamePerId_FromSizeStatsView() } /// - /// #2484: the probe that lets an empty parse-on-read answer say WHICH nothing it found must read the - /// SAME source the read itself reads. A probe on the base table would report a server as captured for - /// rows the view-backed read can never return -- picking the wrong branch in precisely the case the - /// probe exists to get right. It is also scoped to the event_type, and windowless by design. + /// #2484 → #3541 A12: the probes that let an empty parse-on-read answer say WHICH nothing it found must + /// read the SAME source the read itself reads. A probe on the base table would report a server as + /// captured for rows the view-backed read can never return -- picking the wrong branch in precisely the + /// case the probe exists to get right. Both are windowless by design (a time bound would make them + /// answer the same question the read just did) and both are a MAX over collection_time, so they + /// say WHEN as well as whether — the message needs the when. The type-scoped one is scoped to + /// event_type; the source witness deliberately is not, because it answers "has this server's ring + /// buffer ever been read into the store", which is about the session, not the category. Neither is + /// collection_log: the log records a SUCCESS for a run that read a dead session and stored + /// nothing, which is exactly the shape being mis-reported. /// [Fact] - public void HasAnyEventOfTypeSql_ProbesTheSameView_ScopedToType_AndIgnoresTheWindow() + public void TheWitnessProbes_ReadTheSameView_AreWindowless_AndSayWhen() { - var sql = DarlingSystemHealthReader.HasAnyEventOfTypeSql; - Assert.Contains("FROM v_system_health_events", sql, StringComparison.Ordinal); - Assert.Contains("WHERE server_id = $1", sql, StringComparison.Ordinal); - Assert.Contains("event_type = $2", sql, StringComparison.Ordinal); - Assert.Contains("LIMIT 1", sql, StringComparison.Ordinal); - /* Windowless: a time bound here would make the probe answer the same question the read just did. */ - Assert.DoesNotContain("event_time", sql, StringComparison.Ordinal); + var source = DarlingSystemHealthReader.LastCaptureSql; + Assert.Contains("SELECT MAX(collection_time)", source, StringComparison.Ordinal); + Assert.Contains("FROM v_system_health_events", source, StringComparison.Ordinal); + Assert.Contains("WHERE server_id = $1", source, StringComparison.Ordinal); + Assert.Contains("event_xml IS NOT NULL", source, StringComparison.Ordinal); + Assert.DoesNotContain("event_type", source, StringComparison.Ordinal); + Assert.DoesNotContain("event_time", source, StringComparison.Ordinal); + Assert.DoesNotContain("collection_log", source, StringComparison.Ordinal); + + var ofType = DarlingSystemHealthReader.LastCaptureOfTypeSql; + Assert.Contains("SELECT MAX(collection_time)", ofType, StringComparison.Ordinal); + Assert.Contains("FROM v_system_health_events", ofType, StringComparison.Ordinal); + Assert.Contains("WHERE server_id = $1", ofType, StringComparison.Ordinal); + Assert.Contains("event_type = $2", ofType, StringComparison.Ordinal); + Assert.Contains("event_xml IS NOT NULL", ofType, StringComparison.Ordinal); + Assert.DoesNotContain("event_time", ofType, StringComparison.Ordinal); + Assert.DoesNotContain("collection_log", ofType, StringComparison.Ordinal); } [Theory] [InlineData(nameof(DarlingSystemHealthReader.SystemHealthEventsByTypeSql))] [InlineData(nameof(DarlingSystemHealthReader.DatabaseNameMapSql))] - [InlineData(nameof(DarlingSystemHealthReader.HasAnyEventOfTypeSql))] + [InlineData(nameof(DarlingSystemHealthReader.LastCaptureSql))] + [InlineData(nameof(DarlingSystemHealthReader.LastCaptureOfTypeSql))] public void Reads_ArePostgresDialect_PositionalParams(string sqlName) { var sql = (string)typeof(DarlingSystemHealthReader).GetField(sqlName)!.GetValue(null)!; @@ -342,8 +360,8 @@ public void SevereError_DatabaseNameResolution_MatchesViewer() /// /// Gated (DARLING_TEST_PG) live round-trip for the health-parser tools. Plants raw system_health_events rows /// (real captured-event fixtures) across the categories + a database_size_stats mapping row, then asserts each -/// tool shreds + gates + resolves and returns its data-bearing envelope; an empty store returns the "empty" -/// miss. +/// tool shreds + gates + resolves and returns its data-bearing envelope with the source witness; a category +/// never captured on a live session is the healthy "empty"; an empty store is "unavailable" (#3541 A12). /// [Collection("live-postgres")] public sealed class DarlingMcpHealthParserToolsLivePostgresTests @@ -403,16 +421,42 @@ await DarlingMcpTestData.ExecAsync(connection, ct, DarlingMcpTestData.AssertEnvelope(await DarlingMcpHealthParserTools.GetIOIssues(postgres, ServerName), ServerName, "issues"); DarlingMcpTestData.AssertEnvelope(await DarlingMcpHealthParserTools.GetMemoryNodeOOM(postgres, ServerName), ServerName, "events"); - /* memory_conditions / memory_broker have no planted LOW rows → the "empty" miss (not a throw). */ - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetMemoryConditions(postgres, ServerName))); - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetMemoryBroker(postgres, ServerName))); + /* memory_conditions has planted sp_server_diagnostics rows and none is LOW → rung 1 of the + #3541 A12 ladder: "empty", captured and gated out, with the witness saying the source was + observed. memory_broker's event type was never planted while OTHER types were → rung 3: + still "empty" (the session IS being read; the engine recorded no broker event), never + "unavailable". Both are the healthy answer and both must say so with the witness attached. */ + var conditions = JsonDocument.Parse(await DarlingMcpHealthParserTools.GetMemoryConditions(postgres, ServerName)).RootElement; + Assert.Equal("empty", conditions.GetProperty("status").GetString()); + Assert.True(conditions.GetProperty("source_observed").GetBoolean()); + Assert.Equal(t.ToString("o"), conditions.GetProperty("last_captured_at").GetString()); + Assert.True(conditions.GetProperty("events_in_window").GetInt32() > 0); + Assert.Contains("Events ARE being captured", conditions.GetProperty("message").GetString()!, StringComparison.Ordinal); + + var broker = JsonDocument.Parse(await DarlingMcpHealthParserTools.GetMemoryBroker(postgres, ServerName)).RootElement; + Assert.Equal("empty", broker.GetProperty("status").GetString()); + Assert.True(broker.GetProperty("source_observed").GetBoolean()); + Assert.Equal(0, broker.GetProperty("events_in_window").GetInt32()); + Assert.Equal(JsonValueKind.Null, broker.GetProperty("last_captured_of_type_at").ValueKind); + Assert.Contains("the absence is a measurement", broker.GetProperty("message").GetString()!, StringComparison.Ordinal); + + /* The data envelope carries the same witness pair. */ + var scheduler = JsonDocument.Parse(await DarlingMcpHealthParserTools.GetSchedulerIssues(postgres, ServerName)).RootElement; + Assert.True(scheduler.GetProperty("source_observed").GetBoolean()); + Assert.Equal(t.ToString("o"), scheduler.GetProperty("last_captured_at").GetString()); /* an unknown server resolves to the listing error. */ Assert.StartsWith("Could not resolve server.", await DarlingMcpHealthParserTools.GetSystemHealth(postgres, "darling-no-such-server"), StringComparison.Ordinal); - /* an empty store returns the miss. */ + /* An empty store is rung 4: nothing of any type was ever captured, so this is NOT a clean bill — + "unavailable" with source_observed false (#3541 A12). It used to answer "empty", the same word + the healthy branches above earn, which is the defect. */ await DeleteRowsAsync(connection, ct, keepServer: true); - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSchedulerIssues(postgres, ServerName))); + var dead = JsonDocument.Parse(await DarlingMcpHealthParserTools.GetSchedulerIssues(postgres, ServerName)).RootElement; + Assert.Equal("unavailable", dead.GetProperty("status").GetString()); + Assert.False(dead.GetProperty("source_observed").GetBoolean()); + Assert.Equal(JsonValueKind.Null, dead.GetProperty("last_captured_at").ValueKind); + Assert.Contains("NOT an all-clear", dead.GetProperty("message").GetString()!, StringComparison.Ordinal); bodySucceeded = true; } diff --git a/Darling/Darling.Tests/DarlingMcpHealthToolsTests.cs b/Darling/Darling.Tests/DarlingMcpHealthToolsTests.cs index 086dbf613..583122703 100644 --- a/Darling/Darling.Tests/DarlingMcpHealthToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpHealthToolsTests.cs @@ -126,6 +126,25 @@ public void DailySummarySql_DayBucketed_AllSources() Assert.Contains("FROM v_memory_pressure_events", sql, StringComparison.Ordinal); Assert.Contains("FROM config_alert_log", sql, StringComparison.Ordinal); Assert.Contains("day_spine", sql, StringComparison.Ordinal); + + /* #3541 A9: the presence count is the LAST projection, after collection_runs, so the thirteen positional + reads before it stay put — and it counts the seven signal joins, never the collection log or the + alert log, whose survival is the reason a day can outlive its signals. The same pin is written for + Lite's copy in DailySummaryCpuBarPinTests' neighbourhood by construction: both SQLs are read by the + SAME ordinal (13) and judged by the SAME DailySummaryRetention.StateFor. */ + Assert.True(sql.IndexOf("AS collection_runs", StringComparison.Ordinal) < sql.IndexOf("AS signal_sources_present", StringComparison.Ordinal), + "signal_sources_present must trail collection_runs so the positional reads before it stay put"); + Assert.Equal(DailySummaryRetention.SignalSourceCount, System.Text.RegularExpressions.Regex.Matches(sql, @"CASE WHEN (\w+)\.d IS NULL THEN 0 ELSE 1 END").Count); + Assert.DoesNotContain("CASE WHEN cl.d IS NULL", sql, StringComparison.Ordinal); + Assert.DoesNotContain("CASE WHEN al.d IS NULL", sql, StringComparison.Ordinal); + Assert.EndsWith("ORDER BY s.d", sql.TrimEnd(), StringComparison.Ordinal); + + /* And Lite's copy carries the same arm, in the same position, read at the same ordinal. */ + var lite = RepoFile.ReadRepoFile("Lite", "Services", "LocalDataService.DailySummary.cs"); + Assert.True(lite.IndexOf("AS collection_runs", StringComparison.Ordinal) < lite.IndexOf("AS signal_sources_present", StringComparison.Ordinal)); + Assert.Equal(DailySummaryRetention.SignalSourceCount, System.Text.RegularExpressions.Regex.Matches(lite, @"CASE WHEN (\w+)\.d IS NULL THEN 0 ELSE 1 END").Count); + Assert.Contains("SignalSourcesPresent = reader.IsDBNull(13)", lite, StringComparison.Ordinal); + Assert.Contains("SignalSourcesPresent = reader.IsDBNull(13)", RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingHealthReader.cs"), StringComparison.Ordinal); } [Theory] @@ -158,11 +177,21 @@ public void DailySummaryRow_BandsThroughSharedCalculator() { var date = new DateTime(2026, 7, 9, 0, 0, 0, DateTimeKind.Unspecified); - /* A deadlock day is Critical (the shared calculator's rule). */ - var critical = new Reader.DailySummaryReadRow(date, 0m, "", 0, DeadlockCount: 1, 0, 0, 0, 0, 0, 0, 0, HasData: true); + /* A day at a critical deadlock RATE is Critical (#3525): 480 over the row's 24-hour window is + 20/hr, the card band's Critical tier. One deadlock in a day is 0.04/hr and no longer paints the + cell red — the count trigger this replaced read 87.9% of production days Critical. */ + var critical = new Reader.DailySummaryReadRow(date, 0m, "", 0, DeadlockCount: 480, 0, 0, 0, 0, 0, 0, 0, HasData: true); Assert.Equal(DailyHealthBand.Critical, critical.HealthBand); Assert.Equal("Critical", critical.OverallHealth); + var oneDeadlock = new Reader.DailySummaryReadRow(date, 0m, "", 0, DeadlockCount: 1, 0, 0, 0, 0, 0, 0, 0, HasData: true); + Assert.Equal(DailyHealthBand.Healthy, oneDeadlock.HealthBand); + + /* And the band honours the tiers the read stamped from the store (#3368's knobs): the same 20/hr + day under raised tiers is not Critical. */ + var raised = critical with { RateTiers = new DeadlockRateThresholds(100.0, 500.0) }; + Assert.Equal(DailyHealthBand.Healthy, raised.HealthBand); + /* A collected-but-quiet day is Healthy. */ var healthy = new Reader.DailySummaryReadRow(date, 12m, "CXPACKET", 3, 0, 0, 0, 0, 0, 0, 0, 0, HasData: true); Assert.Equal(DailyHealthBand.Healthy, healthy.HealthBand); @@ -173,6 +202,146 @@ public void DailySummaryRow_BandsThroughSharedCalculator() Assert.Equal("No Data", noData.OverallHealth); } + /* ---------------- #3541 A9: retention ghosts (no live PG) ---------------- */ + + /// + /// The row the reader stamps Purged bands No Data whatever the spine still holds for it. This is + /// the defect in one row: HasData: true (a spine row exists — the run record outlives the signals + /// by 30 days), CollectionRuns non-zero, every signal a COALESCEd zero — and before the state + /// existed that banded Healthy. The same row judged Collected is the Healthy it always was, so the state + /// is the ONLY thing that moved the verdict. + /// + [Fact] + public void DailySummaryRow_PurgedOrPastHorizon_IsNoData_NeverHealthy() + { + var date = new DateTime(2026, 7, 9, 0, 0, 0, DateTimeKind.Unspecified); + var shell = new Reader.DailySummaryReadRow(date, 0m, "", 0, 0, 0, 0, 0, 0, 0, 0, 0, HasData: true) { CollectionRuns = 1_440 }; + + Assert.Equal(DailyHealthBand.Healthy, shell.HealthBand); + Assert.Equal(DailyHealthBand.NoData, (shell with { DataState = DailySummaryDataState.Purged }).HealthBand); + Assert.Equal("No Data", (shell with { DataState = DailySummaryDataState.Purged }).OverallHealth); + Assert.Equal(DailyHealthBand.NoData, (shell with { DataState = DailySummaryDataState.PastHorizon }).HealthBand); + /* Inside retention a zero is a measurement: a day with no run record keeps its band (an alert-only + day is Warning, as PerformanceCalendarDataTests has always pinned on Lite), with the caveat on the row. */ + Assert.Equal(DailyHealthBand.Healthy, (shell with { DataState = DailySummaryDataState.NoRunRecord, CollectionRuns = 0 }).HealthBand); + Assert.Equal(DailyHealthBand.Warning, (shell with { DataState = DailySummaryDataState.NoRunRecord, CollectionRuns = 0, AlertCount = 1 }).HealthBand); + + /* A purged day with a real, surviving alert is STILL No Data: the composite band needs every input, + and Warning-on-alerts-alone would understate a day whose deadlocks are gone. */ + var withAlert = shell with { AlertCount = 3, DataState = DailySummaryDataState.Purged }; + Assert.Equal(DailyHealthBand.NoData, withAlert.HealthBand); + Assert.False(withAlert.ToSignals().HasData); + } + + /// + /// The horizon is the SHORTEST effective retention among the sources — on a default store the signal + /// collectors' shared 30 (), never the collection + /// log's 60 or the alert log's 90, which are precisely the horizons that let a spine row outlive its + /// signals. A fleet override on one signal collector moves it; raising every collector past the log + /// leaves the log as the floor. + /// + [Fact] + public void ShortestSignalRetention_IsTheSignalsDefault_AndFollowsFleetOverrides() + { + var none = System.Array.Empty(); + Assert.Equal(PerformanceMonitor.Darling.Service.DarlingRetention.DataRetentionBaseDays, Reader.ShortestSignalRetentionDays(none)); + Assert.Equal(30, PerformanceMonitor.Darling.Service.DarlingRetention.DataRetentionBaseDays); + Assert.True(Reader.ShortestSignalRetentionDays(none) < PerformanceMonitor.Darling.Service.DarlingRetention.CollectionLogRetentionDays); + Assert.True(Reader.ShortestSignalRetentionDays(none) < PerformanceMonitor.Darling.Service.DarlingRetention.AlertHistoryRetentionDays); + + /* Every signal collector the aggregate reads has a schedule entry — the resolver indexes by name. */ + foreach (var collector in Reader.DailySummarySignalCollectors) + Assert.True(CollectorScheduleDefaults.All.ContainsKey(collector), $"{collector} has no CollectorScheduleDefaults entry"); + + var shortened = new[] { new PerformanceMonitor.Darling.Service.ScheduleOverride(null, "deadlocks", null, 10, true) }; + Assert.Equal(10, Reader.ShortestSignalRetentionDays(shortened)); + + /* A PER-SERVER override does not move a shared-table purge, so it does not move the horizon. */ + var perServer = new[] { new PerformanceMonitor.Darling.Service.ScheduleOverride(42, "deadlocks", null, 10, true) }; + Assert.Equal(30, Reader.ShortestSignalRetentionDays(perServer)); + + /* cpu_utilization is floored at the baseline window exactly as the purge floors it. */ + var cpuShort = new[] { new PerformanceMonitor.Darling.Service.ScheduleOverride(null, "cpu_utilization", null, 5, true) }; + Assert.Equal(30, Reader.ShortestSignalRetentionDays(cpuShort)); + + var lengthened = Reader.DailySummarySignalCollectors + .Select(c => new PerformanceMonitor.Darling.Service.ScheduleOverride(null, c, null, 365, true)).ToArray(); + Assert.Equal(PerformanceMonitor.Darling.Service.DarlingRetention.CollectionLogRetentionDays, Reader.ShortestSignalRetentionDays(lengthened)); + } + + /// + /// The fleet-override read names the same table and the same fleet predicate the purge's resolver uses, + /// so the horizon this tool publishes is the horizon the purge enforces. + /// + [Fact] + public void FleetRetentionOverridesSql_ReadsTheFleetRows_OfTheSignalCollectors() + { + var sql = Reader.FleetRetentionOverridesSql; + Assert.Contains("FROM config_collector_schedules", sql, StringComparison.Ordinal); + Assert.Contains("server_id IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("collector_name = ANY($1)", sql, StringComparison.Ordinal); + Assert.Contains("retention_days IS NOT NULL", sql, StringComparison.Ordinal); + } + + /// + /// The descriptions carry the vocabulary an agent will branch on: the horizon field, the count of days + /// before it, the purged state, and the promise that such a day is never Healthy. + /// + [Fact] + public void DailySummaryDescriptions_NameTheHorizon_ThePurgedState_AndTheExactDateFormat() + { + var range = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "get_daily_summary_range"); + var rangeText = range.GetCustomAttribute()!.Description; + Assert.Contains("retention_horizon", rangeText, StringComparison.Ordinal); + Assert.Contains("days_before_horizon", rangeText, StringComparison.Ordinal); + Assert.Contains("data_state=purged", rangeText, StringComparison.Ordinal); + Assert.Contains("NEVER Healthy", rangeText, StringComparison.Ordinal); + + var single = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "get_daily_summary"); + Assert.Contains("data_state=purged", single.GetCustomAttribute()!.Description, StringComparison.Ordinal); + var date = single.GetParameters().Single(p => p.Name == "summary_date"); + Assert.Contains("yyyy-MM-dd ONLY", date.GetCustomAttribute()!.Description, StringComparison.Ordinal); + } + + /// + /// summary_date is EXACT ISO-8601 on both SKUs' tools (through the shared parser): the spelling + /// the description promised parses, the ambiguous 01/02/2026 the general parser used to accept + /// as 2 January is refused, and the refusal names the one accepted form. + /// + [Theory] + [InlineData("2026-07-09", true)] + [InlineData(" 2026-07-09 ", true)] + [InlineData("01/02/2026", false)] + [InlineData("07/09/2026", false)] + [InlineData("2026-7-9", false)] + [InlineData("2026-07-09T00:00:00Z", false)] + [InlineData("July 9, 2026", false)] + public void SummaryDate_IsParsedExactly_OrRefusedNamingTheFormat(string input, bool accepted) + { + var error = McpHelpers.ParseSummaryDate(input, out var date); + if (accepted) + { + Assert.Null(error); + Assert.Equal(new DateTime(2026, 7, 9), date!.Value); + Assert.Equal(DateTimeKind.Utc, date.Value.Kind); + } + else + { + Assert.Null(date); + Assert.StartsWith($"Invalid summary_date value '{input}'", error, StringComparison.Ordinal); + Assert.Contains("yyyy-MM-dd", error, StringComparison.Ordinal); + } + } + + [Fact] + public void SummaryDate_Absent_MeansToday_ResolvedByTheReader() + { + Assert.Null(McpHelpers.ParseSummaryDate(null, out var none)); + Assert.Null(none); + Assert.Null(McpHelpers.ParseSummaryDate(" ", out var blank)); + Assert.Null(blank); + } + /* ---------------- advertised MCP schema ---------------- */ private static System.Collections.Generic.List BuildToolSchemas() @@ -271,7 +440,19 @@ await DarlingMcpTestData.ExecAsync(connection, ct, postgres, ServerName, boundary.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); DarlingMcpTestData.AssertEnvelope(onItsOwnDay, ServerName, "overall_health"); - Assert.Contains("Critical", onItsOwnDay, StringComparison.Ordinal); + /* The row's VISIBILITY to the explicit-date read is what this test pins, so assert the evidence + first. The band it carries changed twice on purpose: #3525 made one deadlock across a 24h day + (0.04/hr) Healthy rather than Critical, and #3541 A9 then withheld the verdict altogether for + THIS row — a fixed day two months back is before the store's 30-day retention horizon, and a + deadlock row surviving there means the purge has not reached the day (data_state + past_horizon: the row is real, the zeros beside it may not be, so No Data rather than a green + cell). A day with no run record INSIDE the horizon reads no_run_record and keeps its band. */ + Assert.Contains("\"deadlock_count\":1", onItsOwnDay, StringComparison.Ordinal); + var judged = JsonDocument.Parse(onItsOwnDay).RootElement; + Assert.Equal("past_horizon", judged.GetProperty("data_state").GetString()); + Assert.Equal("No Data", judged.GetProperty("overall_health").GetString()); + Assert.Contains("1 of 7 signal sources", judged.GetProperty("data_note").GetString(), StringComparison.Ordinal); + Assert.DoesNotContain("Healthy", onItsOwnDay, StringComparison.Ordinal); Assert.Contains("2026-07-20", onItsOwnDay, StringComparison.Ordinal); /* The bug: the same rows are invisible to an implicit "today", which is what the sibling test @@ -318,6 +499,17 @@ await DarlingMcpTestData.ExecAsync(connection, ct, VALUES ($1,$2,$3,$4,$5,$6,$7)", CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(when), ServerId, ServerName, DarlingMcpTestData.Naive(when), 85, 10); + /* One 15-second blocked-process report (#3539 A2/A3): the day's Warning has to come from a signal + whose band does not depend on the time of day this test runs. The high-CPU bar scales with the + still-forming day's elapsed portion (one sample is Warning below four hours and Healthy past + them; six is Critical below 4.8 hours and Warning past them), so no hot-sample count is + Warning at every hour of the day. The blocking WAIT arm is rate-independent: 15 s is Warning + over any window, and nothing here can reach Critical. */ + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, wait_time_ms) +VALUES ($1,$2,$3,$4,$5,$6)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(when), ServerId, ServerName, DarlingMcpTestData.Naive(when), 15_000L); + await DarlingMcpTestData.ExecAsync(connection, ct, @"INSERT INTO memory_stats (collection_id, collection_time, server_id, server_name, total_physical_memory_mb, available_physical_memory_mb, total_server_memory_mb, target_server_memory_mb, buffer_pool_mb, plan_cache_mb, system_memory_state, sql_memory_model) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", @@ -352,7 +544,14 @@ has already rolled over — the tool then correctly returns its empty envelope a var daily = await DarlingMcpHealthTools.GetDailySummary( postgres, ServerName, when.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); DarlingMcpTestData.AssertEnvelope(daily, ServerName, "overall_health"); - Assert.Contains("Critical", daily, StringComparison.Ordinal); /* the deadlock makes the day Critical */ + /* #3525: one deadlock is 0.04/hr against a 24h day — below the rate tiers, so it no longer + makes the day Critical. The 15 s block is the blocking band's Warning wait arm at any elapsed + window (#3539 A2/A3), so the day reads Warning whatever the clock says, and the planted + deadlock stays visible as evidence — as does the run total the error share divides by. */ + Assert.Contains("\"deadlock_count\":1", daily, StringComparison.Ordinal); + Assert.Contains("\"overall_health\":\"Warning\"", daily, StringComparison.Ordinal); + Assert.Contains("\"collection_runs\":1", daily, StringComparison.Ordinal); + Assert.Contains("\"max_block_duration_ms\":15000", daily, StringComparison.Ordinal); bodySucceeded = true; } @@ -365,7 +564,7 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = private static async Task DeleteRowsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) { - var sql = string.Join(" ", new[] { "cpu_utilization_stats", "memory_stats", "wait_stats", "deadlocks", "collection_log" } + var sql = string.Join(" ", new[] { "cpu_utilization_stats", "memory_stats", "wait_stats", "deadlocks", "blocked_process_reports", "collection_log" } .Select(tbl => $"DELETE FROM {tbl} WHERE server_id = {ServerId};")) + $" DELETE FROM servers WHERE server_id = {ServerId};"; using var cleanup = new NpgsqlCommand(sql, connection); diff --git a/Darling/Darling.Tests/DarlingMcpLatchSpinlockToolsTests.cs b/Darling/Darling.Tests/DarlingMcpLatchSpinlockToolsTests.cs index 0296d9ea3..a5d03825f 100644 --- a/Darling/Darling.Tests/DarlingMcpLatchSpinlockToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpLatchSpinlockToolsTests.cs @@ -87,6 +87,12 @@ public void LatchStatsTopNSql_TopByWaitTime_PerSecondFromLag_LatestSnapshot() Assert.Contains("DISTINCT ON (latch_class)", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY a.total_delta_wait_time_ms DESC", sql, StringComparison.Ordinal); Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first (0, the unknowable marker, → NULL through NULLIF); the LAG only for + pre-V127 rows; no ELSE 0 on the rate, so an unknowable interval reads NULL and never 0.00. */ + Assert.Contains("CASE WHEN sample_interval_seconds IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(sample_interval_seconds, 0)", sql, StringComparison.Ordinal); + Assert.Contains("END AS interval_seconds", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0 END", sql, StringComparison.Ordinal); } [Fact] @@ -101,6 +107,12 @@ public void SpinlockStatsTopNSql_TopByCollisions_PerSecondFromLag() Assert.Contains("DISTINCT ON (spinlock_name)", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY a.total_delta_collisions DESC", sql, StringComparison.Ordinal); Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first (0, the unknowable marker, → NULL through NULLIF); the LAG only for + pre-V127 rows; no ELSE 0 on the rate, so an unknowable interval reads NULL and never 0.00. */ + Assert.Contains("CASE WHEN sample_interval_seconds IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(sample_interval_seconds, 0)", sql, StringComparison.Ordinal); + Assert.Contains("END AS interval_seconds", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0 END", sql, StringComparison.Ordinal); } [Theory] diff --git a/Darling/Darling.Tests/DarlingMcpMemoryGrantToolsTests.cs b/Darling/Darling.Tests/DarlingMcpMemoryGrantToolsTests.cs index a377a9995..f125184e9 100644 --- a/Darling/Darling.Tests/DarlingMcpMemoryGrantToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpMemoryGrantToolsTests.cs @@ -11,6 +11,7 @@ using System.ComponentModel; using System.Linq; using System.Reflection; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using ModelContextProtocol.Server; @@ -88,6 +89,31 @@ public void ResourceSemaphoreLatestSql_LatestSnapshot_CarriesCeilingColumns() Assert.Contains("resource_semaphore_id", sql, StringComparison.Ordinal); Assert.Contains("timeout_error_count_delta", sql, StringComparison.Ordinal); Assert.Contains("forced_grant_count_delta", sql, StringComparison.Ordinal); + /* #3540 (V128): the Dashboard's sample_interval_seconds is back — the collector stores it now — and + it rides LAST so every ordinal the reader indexes is unchanged. */ + var interval = sql.IndexOf("sample_interval_seconds", StringComparison.Ordinal); + Assert.True(interval > sql.IndexOf("forced_grant_count_delta,", StringComparison.Ordinal), "the interval must be selected after the last pre-V128 column"); + /* LastIndexOf: the CTE's MAX(collection_time) probe reads the view first; the select list sits ahead + of the SECOND FROM. */ + Assert.True(interval < sql.LastIndexOf("FROM v_memory_grant_stats", StringComparison.Ordinal), "the interval must be in the SELECT list"); + Assert.Equal(1, sql.Split("sample_interval_seconds").Length - 1); + } + + /// + /// #3540 (V128): the resource-semaphore row carries the stored interval and reports it the way the file-I/O + /// row does — IsUnknowable is true ONLY for a stored 0 (the calculator's marker), never for a + /// pre-V128 NULL, which is "never recorded" rather than "unknowable". The tool then hands the caller a + /// null interval for both, with interval_known saying so. + /// + [Fact] + public void ResourceSemaphoreRow_IsUnknowable_OnlyForAStoredZeroInterval() + { + static DarlingMemoryGrantReader.ResourceSemaphoreRow Row(int? interval) => new( + DateTime.UnixEpoch, 0, 2, 100, 200, 90, 80, 10, 8, 3, 1, 5, 2, 0, 0, interval); + + Assert.True(Row(0).IsUnknowable); + Assert.False(Row(120).IsUnknowable); + Assert.False(Row(null).IsUnknowable); } [Fact] @@ -203,18 +229,37 @@ public async Task MemoryGrantTools_ReadPlantedRows_AgainstDevPostgres() await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); var t = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); - foreach (var (semaphore, pool) in new[] { ((short)0, 1), ((short)0, 2) }) + /* #3540 (V128): three semaphores at one collection — pool 1 a restart marker (stored interval 0), + pool 2 measured (120 s), pool 3 a pre-V128 row (NULL). The tool reports the interval only for + the measured one and says interval_known for exactly that one. */ + foreach (var (semaphore, pool, interval) in new[] { ((short)0, 1, (object)0), ((short)0, 2, 120), ((short)0, 3, DBNull.Value) }) { await DarlingMcpTestData.ExecAsync(connection, ct, - @"INSERT INTO memory_grant_stats (collection_id, collection_time, server_id, server_name, resource_semaphore_id, pool_id, target_memory_mb, max_target_memory_mb, total_memory_mb, available_memory_mb, granted_memory_mb, used_memory_mb, grantee_count, waiter_count, timeout_error_count, forced_grant_count, timeout_error_count_delta, forced_grant_count_delta) -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)", - CollectionIdGenerator.Next(), t, ServerId, ServerName, semaphore, pool, 8000m, 12000m, 8000m, 6000m, 2000m, 1500m, 3, 1, 4L, 2L, 1L, 0L); + @"INSERT INTO memory_grant_stats (collection_id, collection_time, server_id, server_name, resource_semaphore_id, pool_id, target_memory_mb, max_target_memory_mb, total_memory_mb, available_memory_mb, granted_memory_mb, used_memory_mb, grantee_count, waiter_count, timeout_error_count, forced_grant_count, timeout_error_count_delta, forced_grant_count_delta, sample_interval_seconds) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, semaphore, pool, 8000m, 12000m, 8000m, 6000m, 2000m, 1500m, 3, 1, 4L, 2L, 1L, 0L, interval); } var semaphoreJson = await DarlingMcpMemoryGrantTools.GetResourceSemaphore(postgres, ServerName); DarlingMcpTestData.AssertEnvelope(semaphoreJson, ServerName, "grants"); Assert.Contains("max_target_memory_mb", semaphoreJson, StringComparison.Ordinal); + using (var doc = JsonDocument.Parse(semaphoreJson)) + { + var byPool = doc.RootElement.GetProperty("grants").EnumerateArray() + .ToDictionary(g => g.GetProperty("pool_id").GetInt32()); + Assert.Equal(3, byPool.Count); + + Assert.Equal(JsonValueKind.Null, byPool[1].GetProperty("sample_interval_seconds").ValueKind); + Assert.False(byPool[1].GetProperty("interval_known").GetBoolean()); + + Assert.Equal(120, byPool[2].GetProperty("sample_interval_seconds").GetInt32()); + Assert.True(byPool[2].GetProperty("interval_known").GetBoolean()); + + Assert.Equal(JsonValueKind.Null, byPool[3].GetProperty("sample_interval_seconds").ValueKind); + Assert.False(byPool[3].GetProperty("interval_known").GetBoolean()); + } + var grantsJson = await DarlingMcpMemoryGrantTools.GetMemoryGrants(postgres, ServerName); DarlingMcpTestData.AssertEnvelope(grantsJson, ServerName, "grants"); Assert.Contains("granted_memory_mb", grantsJson, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/DarlingMcpObjectStatsToolsTests.cs b/Darling/Darling.Tests/DarlingMcpObjectStatsToolsTests.cs index 97b3307bc..fb8a82547 100644 --- a/Darling/Darling.Tests/DarlingMcpObjectStatsToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpObjectStatsToolsTests.cs @@ -110,8 +110,12 @@ public void ObjectSizeGrowthSql_RollsUpPerTable_ComputesGrowth() "GROUP BY database_name, schema_name, table_name", sql, "the rollup is no longer per table"); - Assert.Contains("growth_7d_mb", sql, StringComparison.Ordinal); - Assert.Contains("growth_30d_mb", sql, StringComparison.Ordinal); + /* #3541 A12: the growth figures are no longer derived in SQL — the raw baselines come back as their own + nullable columns and ObjectSizeGrowthRow derives each figure from exactly the baseline it names + (McpZeroIsAMeasurementTests pins the fold's absence and the derivations). */ + Assert.Contains("reserved_mb_7d_ago", sql, StringComparison.Ordinal); + Assert.Contains("reserved_mb_30d_ago", sql, StringComparison.Ordinal); + Assert.Contains("reserved_mb_oldest", sql, StringComparison.Ordinal); SqlTextPin.AssertExpresses("ORDER BY l.current_reserved_mb DESC", sql, "the biggest table no longer sorts first"); } diff --git a/Darling/Darling.Tests/DarlingMcpPgIoToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPgIoToolsTests.cs new file mode 100644 index 000000000..ff54f41ec --- /dev/null +++ b/Darling/Darling.Tests/DarlingMcpPgIoToolsTests.cs @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Wire shape for get_pg_io_stats, asserted against the real projection (#3536). +/// +/// The subject is the timing contract the trend sibling already shipped and this read lacked: +/// track_io_timing is OFF by default in PostgreSQL, so on a stock server every time counter in the +/// store is zero — and read_time_ms / reads over that is 0.000 ms, a latency that reads as an +/// impossibly fast disk rather than an unmeasured one. The zero is a fact about the configuration; printed +/// as a measurement it is the most reassuring wrong number on the surface. +/// +public class DarlingMcpPgIoToolsTests +{ + /// + /// Two combinations the way the reader hands them over: ordered by read time then by read count, so + /// over a store of zero times (timing off) the count IS the ordering. The busiest-by-reads row leads. + /// + private static List Rows(bool timed) => new() + { + new DarlingPgIoReader.PgIoRow( + BackendType: "client backend", ObjectType: "relation", Context: "normal", + Reads: 5_000, ReadTimeMs: timed ? 2_500 : 0, + Hits: 95_000, Extends: 10, ExtendTimeMs: timed ? 40 : 0, + Evictions: 5, Reuses: 0, + Writes: 200, WriteTimeMs: timed ? 90 : 0, + OpBytes: 8_192, WriteCountersTracked: true, StatsReset: null, + ReadBytes: 0, WriteBytes: 0, ExtendBytes: 0, ByteCountersTracked: false), + new DarlingPgIoReader.PgIoRow( + BackendType: "autovacuum worker", ObjectType: "relation", Context: "vacuum", + Reads: 1_000, ReadTimeMs: timed ? 700 : 0, + Hits: 3_000, Extends: 0, ExtendTimeMs: 0, + Evictions: 0, Reuses: 40, + Writes: 50, WriteTimeMs: timed ? 25 : 0, + OpBytes: 8_192, WriteCountersTracked: true, StatsReset: null, + ReadBytes: 0, WriteBytes: 0, ExtendBytes: 0, ByteCountersTracked: false), + }; + + /// + /// The page the projection takes since #3541 A7: the rows plus the WINDOW's totals. Built here as the rows' + /// own sums, which is the "page is the whole window" case every assertion in this file was written + /// against; the shares-of-a-larger-window arithmetic is ' + /// subject. + /// + private static DarlingPgIoReader.PgIoPage Page(List rows) => + new(rows, rows.Sum(r => r.Reads), rows.Sum(r => r.ReadTimeMs)); + + private static JsonElement Parse(string json) + { + using var doc = JsonDocument.Parse(json); + return doc.RootElement.Clone(); + } + + /// + /// The point of #3536. With track_io_timing off — PostgreSQL's DEFAULT — every time field + /// is null rather than the 0.0 the arithmetic produces, top to bottom: the per-row times, the per-read + /// latency, the read-time shares, and the window total. The counters beside them survive untouched, + /// because the operation counts are real measurements whatever the timing setting is. + /// + [Fact] + public void TimingUntracked_NullsEveryTimeField_AndKeepsTheCounts() + { + var root = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: false)), 20, timingSetting: false)); + + Assert.False(root.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Equal(JsonValueKind.Null, root.GetProperty("total_read_time_ms").ValueKind); + + var row = root.GetProperty("combinations")[0]; + Assert.Equal(JsonValueKind.Null, row.GetProperty("read_time_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("avg_read_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("write_time_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("extend_time_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("pct_of_total_read_time").ValueKind); + + /* The counts are measured regardless of the timing setting and must not be dragged down with it. */ + Assert.Equal(5_000, row.GetProperty("reads").GetInt64()); + Assert.Equal(95_000, row.GetProperty("hits").GetInt64()); + Assert.Equal(95.0, row.GetProperty("hit_pct").GetDouble()); + Assert.Equal(200, row.GetProperty("writes").GetInt64()); + + var note = root.GetProperty("timing_note").GetString()!; + Assert.Contains("off by DEFAULT", note, StringComparison.Ordinal); + Assert.Contains("does not measure I/O time", note, StringComparison.Ordinal); + } + + /// + /// The busiest field keeps its key — the web tile reads it by name — and busiest_basis beside it + /// says what the ranking actually used: read time when the server measures it, the read COUNT when it + /// does not. The reader's ORDER BY carries the count as its second key, so over a store of zeros the + /// count is the entire ordering rather than a tiebreak, and the payload has to say so. + /// + [Fact] + public void TheBusiestBasis_IsReadsWhenUntracked_AndReadTimeWhenTracked() + { + var untracked = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: false)), 20, timingSetting: false)); + Assert.Equal("client backend/relation/normal", untracked.GetProperty("busiest_by_read_time").GetString()); + Assert.Contains("read count", untracked.GetProperty("busiest_basis").GetString(), StringComparison.Ordinal); + Assert.Contains("does not measure I/O time", untracked.GetProperty("busiest_basis").GetString(), StringComparison.Ordinal); + + var tracked = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: true)), 20, timingSetting: true)); + Assert.Equal("client backend/relation/normal", tracked.GetProperty("busiest_by_read_time").GetString()); + Assert.Contains("read time", tracked.GetProperty("busiest_basis").GetString(), StringComparison.Ordinal); + } + + /// + /// With timing tracked the measured figures flow through unchanged — the untracked arm must not cost + /// the measuring server anything: per-read latency is time over reads, the shares add up, and the + /// window total is the sum of the rows. + /// + [Fact] + public void TimingTracked_KeepsTheMeasuredFigures() + { + var root = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: true)), 20, timingSetting: true)); + + Assert.True(root.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Equal(3_200.0, root.GetProperty("total_read_time_ms").GetDouble()); + Assert.Contains("pg_server_config", root.GetProperty("io_timing_source").GetString(), StringComparison.Ordinal); + Assert.Contains("track_io_timing on", root.GetProperty("timing_note").GetString(), StringComparison.Ordinal); + + var row = root.GetProperty("combinations")[0]; + Assert.Equal(2_500.0, row.GetProperty("read_time_ms").GetDouble()); + Assert.Equal(0.5, row.GetProperty("avg_read_ms").GetDouble()); + Assert.Equal(78.1, row.GetProperty("pct_of_total_read_time").GetDouble()); + Assert.Equal(90.0, row.GetProperty("write_time_ms").GetDouble()); + } + + /// + /// A store without the server's configuration answers from the only evidence left — whether any + /// non-zero time appears in the window — and SAYS it is inferring, exactly as the trend sibling does. + /// "We do not know" and "the server does not measure it" license different readings of a zero. + /// + [Fact] + public void AnUncollectedSetting_IsInferredFromTheData_AndSaysSo() + { + var quiet = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: false)), 20, timingSetting: null)); + Assert.False(quiet.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Contains("inferred from the data", quiet.GetProperty("io_timing_source").GetString(), StringComparison.Ordinal); + + var timed = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: true)), 20, timingSetting: null)); + Assert.True(timed.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Contains("inferred from the data", timed.GetProperty("io_timing_source").GetString(), StringComparison.Ordinal); + } + + /// + /// The setting wins over the observation in BOTH directions, mirroring the trend sibling: collected + /// configuration is the authority, and inference is only for a store that never collected it. + /// + [Fact] + public void TheCollectedSetting_OverridesTheObservation() + { + /* Setting says on, window happens to be all zeros: tracked, with honest zeros, not nulls. */ + var on = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: false)), 20, timingSetting: true)); + Assert.True(on.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Equal(0.0, on.GetProperty("combinations")[0].GetProperty("read_time_ms").GetDouble()); + + /* Setting says off, stale non-zero times in the window: untracked wins and the times are null. */ + var off = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(Rows(timed: true)), 20, timingSetting: false)); + Assert.False(off.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Equal(JsonValueKind.Null, off.GetProperty("combinations")[0].GetProperty("read_time_ms").ValueKind); + } + + /// + /// The Aurora contract is untouched by the timing one: write COUNTERS tracked/untracked is a different + /// axis from time measured/unmeasured, and a timing-on Aurora still reports null writes. The two flags + /// travel separately because their remedies are different — one is a platform fact, one is a setting. + /// + [Fact] + public void AuroraNullWrites_SurviveTheTimingGate() + { + var aurora = Rows(timed: true) + .Select(r => r with { Writes = 0, WriteTimeMs = 0, WriteCountersTracked = false }) + .ToList(); + + var root = Parse(DarlingMcpPgIoTools.BuildIoJson("srv", 24, Page(aurora), 20, timingSetting: true)); + + Assert.True(root.GetProperty("io_timing_tracked").GetBoolean()); + Assert.False(root.GetProperty("write_counters_tracked_anywhere").GetBoolean()); + Assert.Contains("Aurora", root.GetProperty("note").GetString(), StringComparison.Ordinal); + + var row = root.GetProperty("combinations")[0]; + Assert.Equal(JsonValueKind.Null, row.GetProperty("writes").ValueKind); + Assert.Equal(JsonValueKind.Null, row.GetProperty("write_time_ms").ValueKind); + /* The read side still reports: timing is on and reads are tracked everywhere. */ + Assert.Equal(2_500.0, row.GetProperty("read_time_ms").GetDouble()); + } +} diff --git a/Darling/Darling.Tests/DarlingMcpPgLoggingAuditToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPgLoggingAuditToolsTests.cs new file mode 100644 index 000000000..1d1daf583 --- /dev/null +++ b/Darling/Darling.Tests/DarlingMcpPgLoggingAuditToolsTests.cs @@ -0,0 +1,793 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// get_pg_logging_audit (#3607): the judgment, asserted against the shipped +/// over hand-built snapshots, and the wire shape against the shipped +/// projection — the DarlingMcpPgPlanToolsTests arrangement, for the same reason: a guard that rebuilt +/// either would keep passing while the real one drifted. +/// +/// The snapshots are spelled the way pg_settings renders them — integers in the base unit with +/// no suffix, booleans as on/off, the PostgreSQL 18 log_connections list verbatim — +/// because the parse is the judgment's first step and a fixture in some other shape would test a path no +/// server produces. Measured on 18.4 (the rig this lane ran): log_autovacuum_min_duration renders +/// 600000 with unit ms, log_connections stores on, true, 1, +/// all and receipt,authentication each as written. +/// +public sealed class DarlingMcpPgLoggingAuditToolsTests +{ + private static readonly DateTime Stamp = new(2026, 9, 18, 14, 0, 0, DateTimeKind.Utc); + + private static DarlingPgLoggingAuditReader.PgLoggingSettingRow Row( + string name, string? setting, string? unit = null, string context = "sighup", + string source = "default", string? boot = null, DateTime? at = null, bool pendingRestart = false) => + new(name, setting, unit, context, source, boot ?? setting, pendingRestart, at ?? Stamp); + + /// A PostgreSQL 14 with nothing turned on: every judged setting at its pre-15 default. + private static List AllOff() => new() + { + Row("log_min_duration_statement", "-1", "ms", "superuser"), + Row("log_lock_waits", "off", context: "superuser"), + Row("log_temp_files", "-1", "kB", "superuser"), + Row("log_autovacuum_min_duration", "-1", "ms"), + Row("log_checkpoints", "off"), + Row("log_connections", "off", context: "superuser-backend"), + Row("log_disconnections", "off", context: "superuser-backend"), + Row("deadlock_timeout", "1000", "ms", "superuser"), + Row("shared_preload_libraries", "", context: "postmaster"), + Row("log_line_prefix", "%m [%p] "), + Row("lc_messages", "en_US.utf8", context: "superuser", source: "configuration file"), + Row("work_mem", "4096", "kB", "user"), + }; + + /// The 18.4 rig's defaults, as measured: checkpoints on, autovacuum at ten minutes, connections empty. + private static List Pg18Defaults() + { + var rows = AllOff(); + Replace(rows, Row("log_autovacuum_min_duration", "600000", "ms")); + Replace(rows, Row("log_checkpoints", "on")); + Replace(rows, Row("log_connections", "", context: "superuser-backend")); + return rows; + } + + /// Every setting at its recommended value, so the audit has nothing to ask for. + private static List Recommended() + { + var rows = Pg18Defaults(); + Replace(rows, Row("log_min_duration_statement", "1000", "ms", "superuser", "configuration file", "-1")); + Replace(rows, Row("log_lock_waits", "on", context: "superuser", source: "configuration file", boot: "off")); + Replace(rows, Row("log_temp_files", "0", "kB", "superuser", "configuration file", "-1")); + Replace(rows, Row("log_autovacuum_min_duration", "0", "ms", source: "configuration file", boot: "600000")); + Replace(rows, Row("log_connections", "all", context: "superuser-backend", source: "configuration file", boot: "")); + Replace(rows, Row("log_disconnections", "on", context: "superuser-backend", source: "configuration file", boot: "off")); + return rows; + } + + private static void Replace(List rows, DarlingPgLoggingAuditReader.PgLoggingSettingRow row) + { + rows.RemoveAll(r => r.Name == row.Name); + rows.Add(row); + } + + private static DarlingPgLoggingAudit.Facet FacetOf(DarlingPgLoggingAudit.Result audit, string setting) => + Assert.Single(audit.Facets, f => f.Setting == setting); + + /* ───────────────────────── the verdicts ───────────────────────── */ + + /// + /// The issue's own scenario: a target with everything off. Seven facets, seven off, every one + /// named in the summary, and every remedy is the self-hosted form because nothing in the snapshot says + /// otherwise. + /// + [Fact] + public void ATargetWithEverythingOff_IsOffSevenTimes_AndEveryRowNamesItsRemedy() + { + var audit = DarlingPgLoggingAudit.Audit(AllOff()); + + Assert.Equal(DarlingPgLoggingAudit.JudgedSettings, audit.Facets.Select(f => f.Setting).ToArray()); + Assert.All(audit.Facets, f => Assert.Equal(DarlingPgLoggingAudit.Off, f.Verdict)); + Assert.False(audit.Managed); + Assert.Contains("no rds.* parameter", audit.HostingEvidence, StringComparison.Ordinal); + Assert.Equal(Stamp, audit.CapturedAt); + + foreach (var facet in audit.Facets) + { + Assert.StartsWith("ALTER SYSTEM SET " + facet.Setting + " = ", facet.Remedy, StringComparison.Ordinal); + Assert.Contains("SELECT pg_reload_conf();", facet.Remedy, StringComparison.Ordinal); + Assert.DoesNotContain("parameter group", facet.Remedy, StringComparison.Ordinal); + Assert.Null(facet.ScopeNote); + } + } + + /// + /// The three threshold settings, at each of their three states. -1 is off, 0 is + /// everything, a positive value is partial — and the partial row SAYS what is under the line, + /// with the threshold in it, because "partial" alone is the collapse this read exists to avoid. + /// + [Theory] + [InlineData("log_min_duration_statement", "ms", "-1", DarlingPgLoggingAudit.Off)] + [InlineData("log_min_duration_statement", "ms", "0", DarlingPgLoggingAudit.Instrumented)] + [InlineData("log_min_duration_statement", "ms", "1000", DarlingPgLoggingAudit.Partial)] + [InlineData("log_temp_files", "kB", "-1", DarlingPgLoggingAudit.Off)] + [InlineData("log_temp_files", "kB", "0", DarlingPgLoggingAudit.Instrumented)] + [InlineData("log_temp_files", "kB", "10240", DarlingPgLoggingAudit.Partial)] + [InlineData("log_autovacuum_min_duration", "ms", "-1", DarlingPgLoggingAudit.Off)] + [InlineData("log_autovacuum_min_duration", "ms", "0", DarlingPgLoggingAudit.Instrumented)] + [InlineData("log_autovacuum_min_duration", "ms", "300000", DarlingPgLoggingAudit.Partial)] + public void AThresholdSetting_IsOffAtMinusOne_EverythingAtZero_AndPartialAbove( + string setting, string unit, string value, string expected) + { + var rows = AllOff(); + Replace(rows, Row(setting, value, unit, "superuser", "configuration file", "-1")); + + var facet = FacetOf(DarlingPgLoggingAudit.Audit(rows), setting); + + Assert.Equal(expected, facet.Verdict); + Assert.Equal(value, facet.Value); + Assert.Equal(unit, facet.Unit); + + if (expected == DarlingPgLoggingAudit.Partial) + { + Assert.Contains(value, facet.CostNote, StringComparison.Ordinal); + Assert.Contains("write nothing", facet.CostNote, StringComparison.Ordinal); + } + } + + /// + /// partial is the recommended posture for log_min_duration_statement, and the row says + /// so instead of asking for a change. Sending an agent to "fix" a 1000 ms threshold to 0 is the + /// worst outcome this read could produce: 0 logs every statement the server runs, and #2565 measured + /// the capture-everything shape of that mechanism at 31 percent of throughput. So the threshold row's + /// remedy is "no change", its cost_note calls it the recommendation, and the 0 row — instrumented by the + /// verdict's own definition — carries the measured cost and a remedy that moves it to a threshold. + /// + [Fact] + public void AStatementThreshold_IsTheRecommendation_AndZeroCarriesTheMeasuredCost() + { + var rows = AllOff(); + Replace(rows, Row("log_min_duration_statement", "1000", "ms", "superuser", "configuration file", "-1")); + var threshold = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_min_duration_statement"); + + Assert.Equal(DarlingPgLoggingAudit.Partial, threshold.Verdict); + Assert.StartsWith("No change needed", threshold.Remedy, StringComparison.Ordinal); + Assert.Contains("recommended posture", threshold.CostNote, StringComparison.Ordinal); + + Replace(rows, Row("log_min_duration_statement", "0", "ms", "superuser", "configuration file", "-1")); + var everything = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_min_duration_statement"); + + Assert.Equal(DarlingPgLoggingAudit.Instrumented, everything.Verdict); + Assert.Contains("31 percent", everything.CostNote, StringComparison.Ordinal); + Assert.Contains("#2565", everything.CostNote, StringComparison.Ordinal); + Assert.StartsWith("ALTER SYSTEM SET log_min_duration_statement = 1000;", everything.Remedy, StringComparison.Ordinal); + } + + /// + /// PostgreSQL 15 moved log_autovacuum_min_duration's default from off to ten minutes, and a server + /// sitting on that default is the shape worth naming: partial, and the cost_note says the default + /// sees only the outlier runs. The same threshold set DELIBERATELY (value differs from boot_val) gets the + /// neutral wording — the audit does not accuse a choice of being a default. + /// + [Fact] + public void TheTenMinuteAutovacuumDefault_IsNamedAsTheDefault_AndADeliberateThresholdIsNot() + { + var onDefault = FacetOf(DarlingPgLoggingAudit.Audit(Pg18Defaults()), "log_autovacuum_min_duration"); + Assert.Equal(DarlingPgLoggingAudit.Partial, onDefault.Verdict); + Assert.Contains("own default since 15", onDefault.CostNote, StringComparison.Ordinal); + Assert.Contains("600000", onDefault.CostNote, StringComparison.Ordinal); + + var rows = Pg18Defaults(); + Replace(rows, Row("log_autovacuum_min_duration", "300000", "ms", source: "configuration file", boot: "600000")); + var deliberate = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_autovacuum_min_duration"); + Assert.Equal(DarlingPgLoggingAudit.Partial, deliberate.Verdict); + Assert.DoesNotContain("own default since 15", deliberate.CostNote, StringComparison.Ordinal); + Assert.Contains("300000", deliberate.CostNote, StringComparison.Ordinal); + + /* Review's case: an administrator who WRITES 600000 into postgresql.conf has made a choice that + happens to equal the boot value. PostgreSQL says source = 'configuration file', and so does this - + a text comparison against boot_val would have called the choice inaction, the anti-pattern + DarlingPgServerConfigReader.CurrentConfigSql documents avoiding. */ + Replace(rows, Row("log_autovacuum_min_duration", "600000", "ms", source: "configuration file", boot: "600000")); + var explicitDefault = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_autovacuum_min_duration"); + Assert.Equal(DarlingPgLoggingAudit.Partial, explicitDefault.Verdict); + Assert.DoesNotContain("own default since 15", explicitDefault.CostNote, StringComparison.Ordinal); + } + + /// + /// log_connections across the 17/18 boundary, in the spellings 18.4 stores verbatim. Anything + /// that produces lines is instrumented; the empty string — 18's default and its "off" — is off. + /// + [Theory] + [InlineData("on", true)] + [InlineData("true", true)] + [InlineData("1", true)] + [InlineData("all", true)] + [InlineData("receipt,authentication", true)] + [InlineData("off", false)] + [InlineData("false", false)] + [InlineData("0", false)] + [InlineData("", false)] + public void LogConnections_ReadsBothTheBooleanAndThe18ListForm(string value, bool producesLines) + { + Assert.Equal(producesLines, DarlingPgLoggingAudit.ConnectionLogging(value)); + + var rows = AllOff(); + Replace(rows, Row("log_connections", value, context: "superuser-backend")); + var facet = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_connections"); + + Assert.Equal(producesLines ? DarlingPgLoggingAudit.Instrumented : DarlingPgLoggingAudit.Off, facet.Verdict); + Assert.Equal(value, facet.Value); + } + + /// + /// A setting missing from the snapshot is unknown — not off, not defaulted, not inferred from the + /// major — and it is kept OUT of the off list, because "off" is an action and "we do not know" is not. + /// + [Fact] + public void AnAbsentSetting_IsUnknown_NotInferred_AndNotCountedAsOff() + { + var rows = AllOff(); + rows.RemoveAll(r => r.Name == "log_lock_waits"); + + var audit = DarlingPgLoggingAudit.Audit(rows); + var facet = FacetOf(audit, "log_lock_waits"); + + Assert.Equal(DarlingPgLoggingAudit.Unknown, facet.Verdict); + Assert.Null(facet.Value); + Assert.Null(facet.Source); + Assert.Contains("not in the stored snapshot", facet.CostNote, StringComparison.Ordinal); + Assert.StartsWith("No remedy is offered", facet.Remedy, StringComparison.Ordinal); + + using var doc = JsonDocument.Parse(DarlingMcpPgLoggingAuditTools.BuildAuditJson("srv", audit)); + var root = doc.RootElement; + Assert.Equal(1, root.GetProperty("unknown_count").GetInt32()); + Assert.Equal(6, root.GetProperty("off_count").GetInt32()); + Assert.Equal(new[] { "log_lock_waits" }, root.GetProperty("unknown_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + Assert.DoesNotContain("log_lock_waits", root.GetProperty("off_settings").EnumerateArray().Select(e => e.GetString())); + } + + /// + /// A value the parse cannot read is unknown too, rather than a guess in either direction — and + /// its message says the row IS in the snapshot with an unreadable value, which is a different fact from + /// the row being absent and must not be reported as it (review on #3643). Close to unreachable, since + /// pg_settings renders well-formed values, which is exactly why the message is pinned: nothing + /// else would ever exercise it. + /// + [Fact] + public void AnUnparseableValue_IsUnknown_AndSaysItIsInTheSnapshot() + { + var rows = AllOff(); + Replace(rows, Row("log_temp_files", "lots", "kB", "superuser")); + Replace(rows, Row("log_checkpoints", "maybe")); + + var audit = DarlingPgLoggingAudit.Audit(rows); + foreach (var facet in new[] { FacetOf(audit, "log_temp_files"), FacetOf(audit, "log_checkpoints") }) + { + Assert.Equal(DarlingPgLoggingAudit.Unknown, facet.Verdict); + Assert.Contains("IS in the stored snapshot", facet.CostNote, StringComparison.Ordinal); + Assert.Contains("'" + facet.Value + "'", facet.CostNote, StringComparison.Ordinal); + Assert.DoesNotContain("not in the stored snapshot", facet.CostNote, StringComparison.Ordinal); + Assert.Contains("could not read", facet.Remedy, StringComparison.Ordinal); + } + } + + /// + /// pending_restart reaches the row and the summary (review on #3643). It is the one case where the + /// value judged is provably not the value the server will have: the file already holds another and the + /// running server has not restarted, so the remedy is written against a value that changes at the next + /// restart. get_pg_server_config reports it loudly for the same reason; dropping it here would + /// have left the audit's own remedy unqualified on exactly the row where it needs qualifying. + /// + [Fact] + public void APendingRestart_IsCarriedOnTheRow_AndNamedInTheSummary() + { + var rows = AllOff(); + Replace(rows, Row("log_lock_waits", "off", context: "superuser", source: "configuration file", pendingRestart: true)); + + var audit = DarlingPgLoggingAudit.Audit(rows); + var pending = FacetOf(audit, "log_lock_waits"); + Assert.True(pending.PendingRestart); + Assert.Contains("pending_restart is TRUE", pending.RestartNote, StringComparison.Ordinal); + Assert.Contains("RUNNING one", pending.RestartNote, StringComparison.Ordinal); + Assert.Contains("does not carry the file's value", pending.RestartNote, StringComparison.Ordinal); + + var plain = FacetOf(audit, "log_checkpoints"); + Assert.False(plain.PendingRestart); + Assert.Null(plain.RestartNote); + + using var doc = JsonDocument.Parse(DarlingMcpPgLoggingAuditTools.BuildAuditJson("srv", audit)); + var root = doc.RootElement; + Assert.Equal(1, root.GetProperty("pending_restart_count").GetInt32()); + Assert.Equal(new[] { "log_lock_waits" }, root.GetProperty("pending_restart_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + var row = root.GetProperty("facets").EnumerateArray().Single(f => f.GetProperty("setting").GetString() == "log_lock_waits"); + Assert.True(row.GetProperty("pending_restart").GetBoolean()); + Assert.False(string.IsNullOrWhiteSpace(row.GetProperty("restart_note").GetString())); + Assert.Contains("restart_note", root.GetProperty("note").GetString(), StringComparison.Ordinal); + } + + /// + /// Everything at its recommended value: nothing off, nothing unknown, and every remedy declines to ask + /// for a change — including the two threshold rows whose verdict is partial by design. + /// + [Fact] + public void AFullyInstrumentedTarget_HasNothingOff_AndNoRemedyAsksForAChange() + { + var audit = DarlingPgLoggingAudit.Audit(Recommended()); + + Assert.DoesNotContain(audit.Facets, f => f.Verdict is DarlingPgLoggingAudit.Off or DarlingPgLoggingAudit.Unknown); + Assert.Equal(DarlingPgLoggingAudit.Partial, FacetOf(audit, "log_min_duration_statement").Verdict); + Assert.Equal(6, audit.Facets.Count(f => f.Verdict == DarlingPgLoggingAudit.Instrumented)); + Assert.All(audit.Facets, f => Assert.StartsWith("No change needed", f.Remedy, StringComparison.Ordinal)); + } + + /* ───────────────────────── the remedy per flavour ───────────────────────── */ + + /// + /// One rds.* parameter flips every remedy to the parameter-group form. The registry's + /// engine token cannot make this call — MonitoredEngineKind.Postgres is both self-hosted and RDS + /// for PostgreSQL — and the two need opposite instructions: ALTER SYSTEM is refused on RDS and + /// Aurora, and a parameter group does not exist on a server somebody administers. The evidence is in + /// the same snapshot the audit already reads, and the response names it. + /// + [Fact] + public void AnRdsParameterInTheSnapshot_WordsEveryRemedyForAParameterGroup() + { + var rows = AllOff(); + rows.Add(Row("rds.extensions", "auto_explain,pg_stat_statements,...", context: "postmaster")); + rows.Add(Row("rds.superuser_reserved_connections", "2", context: "postmaster")); + + var audit = DarlingPgLoggingAudit.Audit(rows); + + Assert.True(audit.Managed); + Assert.Contains("2 rds.* parameter(s)", audit.HostingEvidence, StringComparison.Ordinal); + + foreach (var facet in audit.Facets) + { + Assert.StartsWith("Set " + facet.Setting + " = ", facet.Remedy, StringComparison.Ordinal); + Assert.Contains("parameter group", facet.Remedy, StringComparison.Ordinal); + Assert.Contains("ALTER SYSTEM is refused", facet.Remedy, StringComparison.Ordinal); + Assert.Contains("WITHOUT a reboot", facet.Remedy, StringComparison.Ordinal); + Assert.DoesNotContain("pg_reload_conf", facet.Remedy, StringComparison.Ordinal); + } + + /* And the rds.* rows themselves are evidence, not facets. */ + Assert.DoesNotContain(audit.Facets, f => f.Setting.StartsWith("rds.", StringComparison.Ordinal)); + } + + /// + /// The remedy's reload-versus-restart clause comes from the setting's OWN context, not from a table + /// here: superuser-backend settings apply to connections opened after the reload and the remedy + /// says so, a postmaster context (none of the judged settings has one today, so it is forced in + /// the fixture) says restart, and everything else says reload. + /// + [Fact] + public void TheChangeClause_FollowsTheSettingsContext() + { + var rows = AllOff(); + Replace(rows, Row("log_checkpoints", "off", context: "postmaster")); + var audit = DarlingPgLoggingAudit.Audit(rows); + + var perBackend = FacetOf(audit, "log_connections"); + Assert.Equal("reload; applies to connections opened after it", perBackend.ChangeNeeds); + Assert.Contains("new connections take the new one", perBackend.Remedy, StringComparison.Ordinal); + + var forcedStatic = FacetOf(audit, "log_checkpoints"); + Assert.Equal("restart", forcedStatic.ChangeNeeds); + Assert.Contains("RESTART", forcedStatic.Remedy, StringComparison.Ordinal); + + var plain = FacetOf(audit, "log_lock_waits"); + Assert.Equal("reload", plain.ChangeNeeds); + Assert.Contains("a reload, not a restart", plain.Remedy, StringComparison.Ordinal); + + rows.Add(Row("rds.extensions", "x", context: "postmaster")); + var managed = DarlingPgLoggingAudit.Audit(rows); + Assert.Contains("needs a reboot", FacetOf(managed, "log_checkpoints").Remedy, StringComparison.Ordinal); + Assert.Contains("to connections opened after it", FacetOf(managed, "log_connections").Remedy, StringComparison.Ordinal); + } + + /// + /// A per-role or per-database override is the monitoring connection's value, not the server's — the + /// limit the readiness collector states for lc_messages, applied to every GUC here. The row + /// carries a scope_note only when the source says so. + /// + [Fact] + public void ARoleOrDatabaseOverride_GetsAScopeNote_AndAServerSettingDoesNot() + { + var rows = AllOff(); + Replace(rows, Row("log_min_duration_statement", "0", "ms", "superuser", "user", "-1")); + Replace(rows, Row("log_lock_waits", "on", context: "superuser", source: "database", boot: "off")); + Replace(rows, Row("log_temp_files", "0", "kB", "superuser", "configuration file", "-1")); + + var audit = DarlingPgLoggingAudit.Audit(rows); + + Assert.Contains("source is 'user'", FacetOf(audit, "log_min_duration_statement").ScopeNote, StringComparison.Ordinal); + Assert.Contains("source is 'database'", FacetOf(audit, "log_lock_waits").ScopeNote, StringComparison.Ordinal); + Assert.Null(FacetOf(audit, "log_temp_files").ScopeNote); + } + + /// + /// log_lock_waits fires at deadlock_timeout, so its row quotes that setting's value from + /// the same snapshot — and says plainly when the snapshot does not have it, rather than quoting the + /// compiled-in default as if it were this server's. + /// + [Fact] + public void LockWaits_QuotesDeadlockTimeoutFromTheSnapshot_OrSaysItIsMissing() + { + var withTimeout = FacetOf(DarlingPgLoggingAudit.Audit(AllOff()), "log_lock_waits"); + Assert.Contains("deadlock_timeout (1000 ms)", withTimeout.Unlocks, StringComparison.Ordinal); + + var rows = AllOff(); + rows.RemoveAll(r => r.Name == "deadlock_timeout"); + var without = FacetOf(DarlingPgLoggingAudit.Audit(rows), "log_lock_waits"); + Assert.Contains("deadlock_timeout (not in the snapshot)", without.Unlocks, StringComparison.Ordinal); + } + + /* ───────────────────────── the cross-reference to readiness ───────────────────────── */ + + /// + /// Plan capture's own settings are LISTED, with the readiness facet that judges each, and never judged + /// here — no facet row carries them, so there is exactly one verdict on auto_explain in the + /// product and it is get_pg_plan_capture_readiness's. An auto_explain.* GUC missing from + /// the snapshot (the library is not loaded) is a null value, not an absent entry. + /// + [Fact] + public void PlanCaptureSettings_AreListedWithTheirReadinessFacet_AndNeverJudgedHere() + { + var audit = DarlingPgLoggingAudit.Audit(AllOff()); + + Assert.Equal( + new[] { "shared_preload_libraries", "auto_explain.log_min_duration", "log_line_prefix", "lc_messages" }, + audit.JudgedByReadiness.Select(s => s.Setting).ToArray()); + Assert.Equal( + new[] { "library_loaded", "capture_threshold", "plan_attribution", "message_locale" }, + audit.JudgedByReadiness.Select(s => s.ReadinessFacet).ToArray()); + + var autoExplain = Assert.Single(audit.JudgedByReadiness, s => s.Setting == "auto_explain.log_min_duration"); + Assert.Null(autoExplain.Value); + + var locale = Assert.Single(audit.JudgedByReadiness, s => s.Setting == "lc_messages"); + Assert.Equal("en_US.utf8", locale.Value); + Assert.Equal("configuration file", locale.Source); + + Assert.DoesNotContain(audit.Facets, f => f.Setting.StartsWith("auto_explain", StringComparison.Ordinal)); + Assert.DoesNotContain(audit.Facets, f => f.Setting is "shared_preload_libraries" or "log_line_prefix" or "lc_messages"); + + using var doc = JsonDocument.Parse(DarlingMcpPgLoggingAuditTools.BuildAuditJson("srv", audit)); + var block = doc.RootElement.GetProperty("judged_by_readiness"); + Assert.Equal("get_pg_plan_capture_readiness", block.GetProperty("tool").GetString()); + Assert.Contains("NOT judged here", block.GetProperty("note").GetString(), StringComparison.Ordinal); + Assert.Equal(4, block.GetProperty("settings").GetArrayLength()); + } + + /* ───────────────────────── the wire ───────────────────────── */ + + /// + /// The response: the snapshot's time as captured_at (the #3541 A10 spelling every stamped + /// latest read uses), counts that sum to the facet total, the off and unknown settings NAMED, and + /// every facet carrying every prose column — unlocks, consumer, recommended, + /// cost_note, remedy — because the readiness lesson (#3070) was that the remedy column is + /// the one that goes missing on the way to the wire. + /// + [Fact] + public void TheWire_CarriesCapturedAt_TheCounts_TheNamedOffSettings_AndEveryProseColumn() + { + var rows = Pg18Defaults(); + rows.RemoveAll(r => r.Name == "log_disconnections"); + var audit = DarlingPgLoggingAudit.Audit(rows); + + using var doc = JsonDocument.Parse(DarlingMcpPgLoggingAuditTools.BuildAuditJson("srv", audit)); + var root = doc.RootElement; + + Assert.Equal("srv", root.GetProperty("server").GetString()); + Assert.Equal("logging_audit", root.GetProperty("status").GetString()); + Assert.Equal(Stamp.ToString("o"), root.GetProperty("captured_at").GetString()); + Assert.False(root.TryGetProperty("as_of", out _)); + Assert.Contains("not the live server", root.GetProperty("source").GetString(), StringComparison.Ordinal); + Assert.Equal("self-hosted", root.GetProperty("hosting").GetString()); + Assert.False(root.TryGetProperty("hours_back", out _)); + + var total = root.GetProperty("total").GetInt32(); + Assert.Equal(7, total); + Assert.Equal(total, + root.GetProperty("instrumented_count").GetInt32() + root.GetProperty("partial_count").GetInt32() + + root.GetProperty("off_count").GetInt32() + root.GetProperty("unknown_count").GetInt32()); + + /* 18 defaults: checkpoints on (instrumented); autovacuum at ten minutes (partial); statements, lock + waits, temp files and connections off; disconnections removed from the fixture (unknown). */ + Assert.Equal(1, root.GetProperty("instrumented_count").GetInt32()); + Assert.Equal(1, root.GetProperty("partial_count").GetInt32()); + Assert.Equal( + new[] { "log_min_duration_statement", "log_lock_waits", "log_temp_files", "log_connections" }, + root.GetProperty("off_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + Assert.Equal(new[] { "log_disconnections" }, + root.GetProperty("unknown_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + + foreach (var facet in root.GetProperty("facets").EnumerateArray()) + { + foreach (var key in new[] { "setting", "verdict", "unlocks", "consumer", "recommended", "cost_note", "remedy" }) + { + Assert.False(string.IsNullOrWhiteSpace(facet.GetProperty(key).GetString()), + $"{facet.GetProperty("setting").GetString()}.{key} reached the wire empty"); + } + + Assert.True(facet.TryGetProperty("value", out _)); + Assert.True(facet.TryGetProperty("source", out _)); + Assert.True(facet.TryGetProperty("scope_note", out _)); + Assert.True(facet.TryGetProperty("pending_restart", out _)); + Assert.True(facet.TryGetProperty("restart_note", out _)); + } + + Assert.Equal(0, root.GetProperty("pending_restart_count").GetInt32()); + } + + /// + /// Every consumer says PLANNED today, and this pin is meant to go red. The issue's sequencing + /// note says the audit earns its keep once #3601's pipeline and its parser families (#3602, #3603) + /// consume the lines, and none of those ships. Each facet says so beside the read that exists today. + /// When a consumer lands, the facet whose lines it reads must stop saying planned — and this assertion + /// is what makes that a deliberate edit rather than stale prose an agent plans against. + /// + [Fact] + public void EveryConsumer_IsHonestlyPlanned_UntilTheLogPipelineShips() + { + var audit = DarlingPgLoggingAudit.Audit(AllOff()); + + Assert.All(audit.Facets, f => Assert.StartsWith("PLANNED", f.Consumer, StringComparison.Ordinal)); + + /* And each names its issue and the read that exists today, so "planned" is a pointer and not a shrug. */ + Assert.Contains("#3601", FacetOf(audit, "log_min_duration_statement").Consumer, StringComparison.Ordinal); + Assert.Contains("get_pg_top_queries", FacetOf(audit, "log_min_duration_statement").Consumer, StringComparison.Ordinal); + Assert.Contains("#3601", FacetOf(audit, "log_lock_waits").Consumer, StringComparison.Ordinal); + Assert.Contains("get_pg_blocking", FacetOf(audit, "log_lock_waits").Consumer, StringComparison.Ordinal); + Assert.Contains("#3602", FacetOf(audit, "log_temp_files").Consumer, StringComparison.Ordinal); + Assert.Contains("#3603", FacetOf(audit, "log_autovacuum_min_duration").Consumer, StringComparison.Ordinal); + Assert.Contains("get_pg_autovacuum_health", FacetOf(audit, "log_autovacuum_min_duration").Consumer, StringComparison.Ordinal); + Assert.Contains("get_pg_write_stats", FacetOf(audit, "log_checkpoints").Consumer, StringComparison.Ordinal); + } + + /// An empty snapshot is the tool's empty/not_collected path, never an audit of nothing. + [Fact] + public void AnEmptySnapshot_IsRefusedByTheJudgment() + { + Assert.Throws(() => DarlingPgLoggingAudit.Audit(Array.Empty())); + } + + /* ───────────────────────── the tool's contract ───────────────────────── */ + + /// + /// The description is what an agent plans against, so the claims it must keep making are pinned: it + /// reads the STORED snapshot and not the live server, it names all seven settings, it defers plan + /// capture's settings to the readiness read, it defines partial, and it says PostgreSQL-only. + /// + [Fact] + public void TheToolDescription_StatesWhatItJudges_AndThatItReadsStoredConfig() + { + var method = typeof(DarlingMcpPgLoggingAuditTools).GetMethod(nameof(DarlingMcpPgLoggingAuditTools.GetPgLoggingAudit))!; + + Assert.Equal("get_pg_logging_audit", method.GetCustomAttribute()!.Name); + var description = method.GetCustomAttribute()!.Description; + + foreach (var setting in DarlingPgLoggingAudit.JudgedSettings) + { + Assert.Contains(setting, description, StringComparison.Ordinal); + } + + Assert.Contains("STORED configuration snapshot", description, StringComparison.Ordinal); + Assert.Contains("never the live server", description, StringComparison.Ordinal); + Assert.Contains("LATEST IS A TIME", description, StringComparison.Ordinal); + Assert.Contains("captured_at", description, StringComparison.Ordinal); + Assert.Contains("get_pg_plan_capture_readiness", description, StringComparison.Ordinal); + Assert.Contains("partial means a THRESHOLD is filtering", description, StringComparison.Ordinal); + Assert.Contains("PLANNED", description, StringComparison.Ordinal); + Assert.Contains("parameter group on RDS/Aurora", description, StringComparison.Ordinal); + Assert.EndsWith("PostgreSQL-only.", description, StringComparison.Ordinal); + + /* A latest-snapshot read: no window and no anchor, the get_pg_server_config convention. The + AsOfWindowAnchorTests pins hold the catalog to the same fact from the other side. */ + var parameters = method.GetParameters().Select(p => p.Name).ToArray(); + Assert.DoesNotContain("hours_back", parameters); + Assert.DoesNotContain("as_of", parameters); + } + + /// + /// Reachable from the web, with a catalog entry that binds only the server — the same fact from the + /// dispatch side. Registration with the MCP host is McpToolTypeRegistrationTests' derived pin. + /// + [Fact] + public void TheRead_IsDispatched_AndItsCatalogEntryBindsOnlyTheServer() + { + Assert.Contains("get_pg_logging_audit", DarlingWebEndpoints.BuildReadDispatch().Keys); + + var descriptor = DarlingWebEndpoints.CatalogDescriptors["get_pg_logging_audit"]; + Assert.Equal(new[] { "server" }, descriptor.Params.Select(p => p.Name).ToArray()); + Assert.Contains("get_pg_plan_capture_readiness", descriptor.Description, StringComparison.Ordinal); + } +} + +/// +/// The audit end to end against a live store (#3607): two servers whose newest pg_server_config +/// snapshots model the two hosting flavours, an OLDER snapshot on each that must lose to the newest, a +/// client-sourced row that must not be read as the server's setting, and a third server with no +/// snapshot at all — the empty path. Executed on this lane's 18.4 rig through the mactest harness +/// before CI. +/// +[Collection("live-postgres")] +public sealed class DarlingMcpPgLoggingAuditLivePostgresTests +{ + private const string SelfHostedName = "darling-pg-logging-audit-self-hosted"; + private const string ManagedName = "darling-pg-logging-audit-managed"; + private const string EmptyName = "darling-pg-logging-audit-empty"; + private static readonly int SelfHostedId = ServerIdHelper.GetDeterministicHashCode(SelfHostedName); + private static readonly int ManagedId = ServerIdHelper.GetDeterministicHashCode(ManagedName); + private static readonly int EmptyId = ServerIdHelper.GetDeterministicHashCode(EmptyName); + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task TheAudit_JudgesTheNewestSnapshot_PerFlavour_AndReportsAnEmptyStoreHonestly() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live logging-audit test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + foreach (var (id, name) in new[] { (SelfHostedId, SelfHostedName), (ManagedId, ManagedName), (EmptyId, EmptyName) }) + { + await DarlingMcpTestData.RegisterServerAsync(connection, id, name, ct); + await DarlingMcpTestData.ExecAsync(connection, ct, + "UPDATE servers SET engine_kind = $2 WHERE server_id = $1", id, MonitoredEngineKind.Postgres); + } + + var older = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddHours(-2); + var newest = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-20); + + /* Self-hosted: the OLDER snapshot had lock waits on; the newest has it off. The audit must + report the newest, and the client-sourced row saying statements are logged at 0 is the + monitoring session's own SET and must not become the server's setting. */ + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, older, "log_lock_waits", "on", null, "superuser", "configuration file", "off"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_lock_waits", "off", null, "superuser", "default", "off"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_min_duration_statement", "-1", "ms", "superuser", "default", "-1"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_min_duration_statement", "0", "ms", "superuser", "client", "-1"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_temp_files", "0", "kB", "superuser", "configuration file", "-1"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_autovacuum_min_duration", "600000", "ms", "sighup", "default", "600000"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_checkpoints", "on", null, "sighup", "default", "on"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_connections", "", null, "superuser-backend", "default", ""); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_disconnections", "off", null, "superuser-backend", "default", "off"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "deadlock_timeout", "1000", "ms", "superuser", "default", "1000"); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "lc_messages", "C", null, "superuser", "configuration file", ""); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "log_line_prefix", "%m [%p] %q%u@%d %Q ", null, "sighup", "configuration file", "%m [%p] "); + await SeedAsync(connection, ct, SelfHostedId, SelfHostedName, newest, "shared_preload_libraries", "pg_stat_statements", null, "postmaster", "configuration file", ""); + + /* Managed: rds.* in the snapshot, a per-role override on temp files. */ + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "rds.extensions", "auto_explain,pg_stat_statements", null, "postmaster", "default", ""); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_lock_waits", "on", null, "superuser", "configuration file", "off"); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_min_duration_statement", "1000", "ms", "superuser", "configuration file", "-1"); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_temp_files", "10240", "kB", "superuser", "user", "-1"); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_autovacuum_min_duration", "-1", "ms", "sighup", "configuration file", "600000"); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_checkpoints", "on", null, "sighup", "default", "on"); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_connections", "on", null, "superuser-backend", "configuration file", ""); + await SeedAsync(connection, ct, ManagedId, ManagedName, newest, "log_disconnections", "on", null, "superuser-backend", "configuration file", "off", pendingRestart: true); + + /* ── self-hosted ── */ + var self = JsonDocument.Parse(await DarlingMcpPgLoggingAuditTools.GetPgLoggingAudit(postgres, SelfHostedName)).RootElement; + + Assert.Equal("logging_audit", self.GetProperty("status").GetString()); + Assert.Equal(SelfHostedName, self.GetProperty("server").GetString()); + Assert.Equal(DateTime.SpecifyKind(newest, DateTimeKind.Utc).ToString("o"), self.GetProperty("captured_at").GetString()); + Assert.Equal("self-hosted", self.GetProperty("hosting").GetString()); + + var selfFacets = self.GetProperty("facets").EnumerateArray().ToDictionary(f => f.GetProperty("setting").GetString()!); + + /* The newest snapshot won: off, not the older snapshot's on. */ + Assert.Equal("off", selfFacets["log_lock_waits"].GetProperty("verdict").GetString()); + Assert.StartsWith("ALTER SYSTEM SET log_lock_waits = on;", selfFacets["log_lock_waits"].GetProperty("remedy").GetString(), StringComparison.Ordinal); + + /* The client-sourced 0 was excluded: the server's own -1 is what was judged. */ + Assert.Equal("off", selfFacets["log_min_duration_statement"].GetProperty("verdict").GetString()); + Assert.Equal("-1", selfFacets["log_min_duration_statement"].GetProperty("value").GetString()); + + Assert.Equal("instrumented", selfFacets["log_temp_files"].GetProperty("verdict").GetString()); + Assert.Equal("partial", selfFacets["log_autovacuum_min_duration"].GetProperty("verdict").GetString()); + Assert.Equal("instrumented", selfFacets["log_checkpoints"].GetProperty("verdict").GetString()); + Assert.Equal("off", selfFacets["log_connections"].GetProperty("verdict").GetString()); + Assert.Equal("off", selfFacets["log_disconnections"].GetProperty("verdict").GetString()); + Assert.Equal(0, self.GetProperty("unknown_count").GetInt32()); + + var readiness = self.GetProperty("judged_by_readiness").GetProperty("settings").EnumerateArray() + .ToDictionary(s => s.GetProperty("setting").GetString()!); + Assert.Equal("C", readiness["lc_messages"].GetProperty("value").GetString()); + Assert.Equal("pg_stat_statements", readiness["shared_preload_libraries"].GetProperty("value").GetString()); + Assert.Equal(JsonValueKind.Null, readiness["auto_explain.log_min_duration"].GetProperty("value").ValueKind); + + /* ── managed ── */ + var managed = JsonDocument.Parse(await DarlingMcpPgLoggingAuditTools.GetPgLoggingAudit(postgres, ManagedName)).RootElement; + + Assert.Equal("managed (RDS/Aurora)", managed.GetProperty("hosting").GetString()); + var managedFacets = managed.GetProperty("facets").EnumerateArray().ToDictionary(f => f.GetProperty("setting").GetString()!); + + Assert.Equal("off", managedFacets["log_autovacuum_min_duration"].GetProperty("verdict").GetString()); + var remedy = managedFacets["log_autovacuum_min_duration"].GetProperty("remedy").GetString()!; + Assert.StartsWith("Set log_autovacuum_min_duration = 0 in the DB parameter group", remedy, StringComparison.Ordinal); + Assert.DoesNotContain("pg_reload_conf", remedy, StringComparison.Ordinal); + + Assert.Equal("partial", managedFacets["log_temp_files"].GetProperty("verdict").GetString()); + Assert.Contains("source is 'user'", managedFacets["log_temp_files"].GetProperty("scope_note").GetString(), StringComparison.Ordinal); + Assert.Equal("partial", managedFacets["log_min_duration_statement"].GetProperty("verdict").GetString()); + Assert.StartsWith("No change needed", managedFacets["log_min_duration_statement"].GetProperty("remedy").GetString(), StringComparison.Ordinal); + Assert.Equal(new[] { "log_autovacuum_min_duration" }, + managed.GetProperty("off_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + /* deadlock_timeout was not seeded for this server, and the row says so rather than quoting 1000. */ + Assert.Contains("not in the snapshot", managedFacets["log_lock_waits"].GetProperty("unlocks").GetString(), StringComparison.Ordinal); + /* The one pending_restart row travelled from the store to the wire and into the summary. */ + Assert.True(managedFacets["log_disconnections"].GetProperty("pending_restart").GetBoolean()); + Assert.Equal(new[] { "log_disconnections" }, + managed.GetProperty("pending_restart_settings").EnumerateArray().Select(e => e.GetString()).ToArray()); + Assert.False(managedFacets["log_lock_waits"].GetProperty("pending_restart").GetBoolean()); + + /* ── no snapshot at all ── */ + var empty = JsonDocument.Parse(await DarlingMcpPgLoggingAuditTools.GetPgLoggingAudit(postgres, EmptyName)).RootElement; + Assert.Equal("empty", empty.GetProperty("status").GetString()); + Assert.Contains("nothing to audit", empty.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Contains("not a verdict", empty.GetProperty("message").GetString(), StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, DeleteRowsAsync); + } + } + + private static Task SeedAsync( + NpgsqlConnection connection, CancellationToken ct, int serverId, string serverName, DateTime collectionTime, + string name, string? setting, string? unit, string context, string source, string? bootVal, bool pendingRestart = false) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_server_config + (collection_id, collection_time, server_id, server_name, name, setting, unit, category, context, vartype, + source, boot_val, reset_val, sourcefile, sourceline, pending_restart, short_desc) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)", + CollectionIdGenerator.Next(), collectionTime, serverId, serverName, name, setting, unit, + "Reporting and Logging / What to Log", context, unit is null ? "bool" : "integer", + source, bootVal, setting, null, 0, pendingRestart, null); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, + "DELETE FROM pg_server_config WHERE server_id IN ($1, $2, $3)", SelfHostedId, ManagedId, EmptyId); + await DarlingMcpTestData.ExecAsync(connection, ct, + "DELETE FROM servers WHERE server_id IN ($1, $2, $3)", SelfHostedId, ManagedId, EmptyId); + } +} diff --git a/Darling/Darling.Tests/DarlingMcpPgPercentDenominatorTests.cs b/Darling/Darling.Tests/DarlingMcpPgPercentDenominatorTests.cs new file mode 100644 index 000000000..fc3f02be1 --- /dev/null +++ b/Darling/Darling.Tests/DarlingMcpPgPercentDenominatorTests.cs @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3541 A7: five PostgreSQL MCP tools published a share of the PAGE under a name that promised a share of +/// the WINDOW. Each divided every row by the sum of the rows it had fetched, so at limit = 3 the three +/// pct_of_total_* figures summed to 100% by construction and read, to an agent holding only the JSON, +/// as "these three are everything". get_pg_top_queries compounded it with a hidden LIMIT 50 +/// under a Take(limit), so its denominator was whichever of the two caps bit — never the window. +/// +/// The fix is one rule: the denominator comes off the SAME statement as the rows, as a window +/// aggregate over the grouped result before LIMIT, and rides on a page record beside them. This file +/// pins the arithmetic through each tool's projection with no store: a window of three series whose +/// shares are 60 / 30 / 10, read at limit = 1, must report the top row at 60 and not 100, must +/// publish the window's total as total_* and the page's own sum as returned_*, and must observe +/// truncation from the sentinel row rather than infer it from the cap. +/// +/// Every case is a PAIR — the cut page and the whole page over the same three series — because +/// the whole-page case is where the old and new arithmetic AGREE (a page that is the window sums to 100 +/// either way) and only the cut page can tell them apart. Asserting the whole page alone would pass under +/// the defect. The SQL that produces the window total is exercised against live PostgreSQL by +/// ; here the page is constructed, so what is under test is +/// that the projection DIVIDES BY THE PAGE'S WINDOW TOTAL and not by anything it can compute from the rows. +/// +public sealed class DarlingMcpPgPercentDenominatorTests +{ + /* The window: three series at 600 / 300 / 100, so the whole window is 1,000 and the shares are + 60 / 30 / 10 whatever the unit. A cut page at limit = 1 carries the 600 row plus the 300 row as the + over-fetched sentinel; the projection must drop the sentinel and still divide by 1,000. */ + private const long WindowTotal = 1_000; + private static readonly long[] Series = [600, 300, 100]; + + private static JsonElement Parse(string json) => JsonDocument.Parse(json).RootElement.Clone(); + + /// + /// The envelope contract shared by all five: the named count is the array's length, truncated is + /// what the sentinel said, total_* is the WINDOW's figure, returned_* is the page's own sum, + /// and the headline ratio is page over window. Asserted by KEY NAME so a projection that quietly + /// repointed total_* at the page sum again fails on the number rather than on a missing property. + /// + private static void AssertEnvelope( + JsonElement root, string rowsKey, string returnedCountKey, int returned, bool truncated, + string totalKey, double total, string returnedKey, double returnedSum, string ratioKey) + { + Assert.Equal(returned, root.GetProperty(rowsKey).GetArrayLength()); + Assert.Equal(returned, root.GetProperty(returnedCountKey).GetInt32()); + Assert.Equal(truncated, root.GetProperty("truncated").GetBoolean()); + Assert.Equal(total, root.GetProperty(totalKey).GetDouble()); + Assert.Equal(returnedSum, root.GetProperty(returnedKey).GetDouble()); + Assert.Equal(Math.Round(returnedSum / total * 100, 1), root.GetProperty(ratioKey).GetDouble()); + Assert.False(string.IsNullOrEmpty(root.GetProperty("order").GetString())); + } + + /// The per-row shares divide by the window: the cut page's single row is 60, the whole page's + /// rows are 60 / 30 / 10 and sum to 100 ONLY because that page is the whole window. + private static void AssertShares(JsonElement rows, string shareKey, params double[] expected) + { + var actual = rows.EnumerateArray().Select(r => r.GetProperty(shareKey).GetDouble()).ToArray(); + Assert.Equal(expected, actual); + } + + /* ───────────────────────── get_pg_top_queries ───────────────────────── */ + + private static DarlingPgStatementReader.PgStatementRow Statement(int i) => new( + QueryId: 1_000 + i, DatabaseId: 16384, Calls: 10, TotalExecTimeMs: Series[i], RowsReturned: 100, + MaxExecTimeMs: 9.5, SharedBlocksHit: 1, SharedBlocksRead: 1, StorageBlocksRead: null, OrcacheBlocksHit: null, + TempBlocksRead: 0, TempBlocksWritten: 0, WalBytes: 0, MaxPeakMemBytes: null, QueryText: null); + + [Fact] + public void TopQueries_ShareIsOfTheWindow_NotOfThePage() + { + var cut = Parse(DarlingMcpPgStatementTools.BuildTopQueriesJson( + "srv", 24, new DarlingPgStatementReader.PgTopQueriesPage([Statement(0), Statement(1)], WindowTotal), limit: 1)); + AssertEnvelope(cut, "queries", "queries_returned", returned: 1, truncated: true, + "total_exec_time_ms", WindowTotal, "returned_exec_time_ms", 600, "returned_pct_of_total"); + AssertShares(cut.GetProperty("queries"), "pct_of_total_time", 60); + Assert.Equal("total_exec_time_ms_desc", cut.GetProperty("order").GetString()); + + var whole = Parse(DarlingMcpPgStatementTools.BuildTopQueriesJson( + "srv", 24, new DarlingPgStatementReader.PgTopQueriesPage([Statement(0), Statement(1), Statement(2)], WindowTotal), limit: 3)); + AssertEnvelope(whole, "queries", "queries_returned", returned: 3, truncated: false, + "total_exec_time_ms", WindowTotal, "returned_exec_time_ms", 1_000, "returned_pct_of_total"); + AssertShares(whole.GetProperty("queries"), "pct_of_total_time", 60, 30, 10); + } + + /// + /// The window total is NOT recomputed from the rows: a page whose rows sum to 600 against a window of + /// 1,000 must say 1,000, and a page handed a window total SMALLER than its rows (impossible from the + /// reader, but the discriminating input here) must still divide by what it was handed. A projection + /// that summed the rows would pass the first and fail the second. + /// + [Fact] + public void TopQueries_DividesByThePagesWindowTotal_NeverByASumOfTheRows() + { + var page = new DarlingPgStatementReader.PgTopQueriesPage([Statement(0)], WindowTotalExecTimeMs: 2_400); + var root = Parse(DarlingMcpPgStatementTools.BuildTopQueriesJson("srv", 24, page, limit: 5)); + + Assert.Equal(2_400, root.GetProperty("total_exec_time_ms").GetInt64()); + Assert.Equal(25.0, root.GetProperty("queries")[0].GetProperty("pct_of_total_time").GetDouble()); + Assert.Equal(25.0, root.GetProperty("returned_pct_of_total").GetDouble()); + Assert.False(root.GetProperty("truncated").GetBoolean()); + } + + /* ───────────────────────── get_pg_wait_stats ───────────────────────── */ + + private static DarlingPgWaitReader.PgWaitRow Wait(int i) => + new("IO", "DataFileRead" + i, TotalWaits: 10, TotalWaitTimeMs: Series[i], AvgWaitTimeMs: Series[i] / 10.0); + + [Fact] + public void WaitStats_ShareIsOfTheWindow_NotOfThePage() + { + var cut = Parse(DarlingMcpPgWaitTools.BuildWaitStatsJson( + "srv", 24, new DarlingPgWaitReader.PgWaitStatsPage([Wait(0), Wait(1)], WindowTotal), limit: 1)); + AssertEnvelope(cut, "waits", "wait_events_returned", returned: 1, truncated: true, + "total_wait_time_ms", WindowTotal, "returned_wait_time_ms", 600, "returned_pct_of_total"); + AssertShares(cut.GetProperty("waits"), "pct_of_total_wait", 60); + Assert.Equal("total_wait_time_ms_desc", cut.GetProperty("order").GetString()); + + var whole = Parse(DarlingMcpPgWaitTools.BuildWaitStatsJson( + "srv", 24, new DarlingPgWaitReader.PgWaitStatsPage([Wait(0), Wait(1), Wait(2)], WindowTotal), limit: 3)); + AssertEnvelope(whole, "waits", "wait_events_returned", returned: 3, truncated: false, + "total_wait_time_ms", WindowTotal, "returned_wait_time_ms", 1_000, "returned_pct_of_total"); + AssertShares(whole.GetProperty("waits"), "pct_of_total_wait", 60, 30, 10); + } + + /* ───────────────────────── get_pg_wait_sampling ───────────────────────── */ + + private static DarlingPgWaitSamplingReader.PgWaitSamplingRow Sample(int i) => new( + EventType: i == 2 ? "CPU" : "IO", Event: i == 2 ? "Running" : "DataFileRead", QueryId: 1_000 + i, + SampleCount: Series[i], EstimatedWaitMs: Series[i] * 10, BackendCount: 1, CounterReset: false, + CaptureTime: new DateTime(2026, 9, 18, 12, 0, 0, DateTimeKind.Utc)); + + [Fact] + public void WaitSampling_ShareIsOfTheWindow_NotOfThePage() + { + var cut = Parse(DarlingMcpPgWaitSamplingTools.BuildWaitSamplingJson( + "srv", 24, new DarlingPgWaitSamplingReader.PgWaitSamplingPage([Sample(0), Sample(1)], WindowTotal), limit: 1)); + AssertEnvelope(cut, "waits", "waits_returned", returned: 1, truncated: true, + "total_samples", WindowTotal, "returned_samples", 600, "returned_pct_of_total"); + AssertShares(cut.GetProperty("waits"), "pct_of_samples", 60); + Assert.Equal("samples_desc", cut.GetProperty("order").GetString()); + + var whole = Parse(DarlingMcpPgWaitSamplingTools.BuildWaitSamplingJson( + "srv", 24, new DarlingPgWaitSamplingReader.PgWaitSamplingPage([Sample(0), Sample(1), Sample(2)], WindowTotal), limit: 3)); + AssertEnvelope(whole, "waits", "waits_returned", returned: 3, truncated: false, + "total_samples", WindowTotal, "returned_samples", 1_000, "returned_pct_of_total"); + AssertShares(whole.GetProperty("waits"), "pct_of_samples", 60, 30, 10); + } + + /// + /// The sentinel row must not decide anything the page reports: a reset flagged ONLY on the over-fetched + /// row is not on the page and must not put the reset sentence in the page's note. + /// + [Fact] + public void WaitSampling_TheSentinelRow_DoesNotSpeakForThePage() + { + var sentinelReset = Sample(1) with { CounterReset = true }; + var root = Parse(DarlingMcpPgWaitSamplingTools.BuildWaitSamplingJson( + "srv", 24, new DarlingPgWaitSamplingReader.PgWaitSamplingPage([Sample(0), sentinelReset], WindowTotal), limit: 1)); + + Assert.True(root.GetProperty("truncated").GetBoolean()); + Assert.DoesNotContain("RESET", root.GetProperty("note").GetString(), StringComparison.Ordinal); + } + + /* ───────────────────────── get_pg_kernel_stats ───────────────────────── */ + + private static DarlingPgKernelStatsReader.PgKernelStatRow Kernel(int i) => new( + DatabaseName: "app", QueryId: 1_000 + i, TotalCpuMs: Series[i], ExecUserTimeMs: Series[i] * 0.7, ExecSystemTimeMs: Series[i] * 0.3, + ExecReadBytes: 8_192, ExecWriteBytes: 0, MajorFaults: 0, CounterReset: false, + CaptureTime: new DateTime(2026, 9, 18, 12, 0, 0, DateTimeKind.Utc)); + + [Fact] + public void KernelStats_ShareIsOfTheWindow_NotOfThePage() + { + var cut = Parse(DarlingMcpPgKernelStatsTools.BuildKernelStatsJson( + "srv", 24, new DarlingPgKernelStatsReader.PgKernelStatsPage([Kernel(0), Kernel(1)], WindowTotal), limit: 1)); + AssertEnvelope(cut, "queries", "queries_returned", returned: 1, truncated: true, + "total_cpu_ms", WindowTotal, "returned_cpu_ms", 600, "returned_pct_of_total"); + AssertShares(cut.GetProperty("queries"), "pct_of_total_cpu", 60); + Assert.Equal("cpu_ms_desc", cut.GetProperty("order").GetString()); + + var whole = Parse(DarlingMcpPgKernelStatsTools.BuildKernelStatsJson( + "srv", 24, new DarlingPgKernelStatsReader.PgKernelStatsPage([Kernel(0), Kernel(1), Kernel(2)], WindowTotal), limit: 3)); + AssertEnvelope(whole, "queries", "queries_returned", returned: 3, truncated: false, + "total_cpu_ms", WindowTotal, "returned_cpu_ms", 1_000, "returned_pct_of_total"); + AssertShares(whole.GetProperty("queries"), "pct_of_total_cpu", 60, 30, 10); + } + + /* ───────────────────────── get_pg_io_stats ───────────────────────── */ + + private static DarlingPgIoReader.PgIoRow Io(int i, bool timed) => new( + BackendType: "client backend", ObjectType: "relation", Context: i == 0 ? "normal" : i == 1 ? "vacuum" : "bulkread", + Reads: Series[i], ReadTimeMs: timed ? Series[i] / 10.0 : 0, Hits: 0, Extends: 0, ExtendTimeMs: 0, Evictions: 0, Reuses: 0, + Writes: 0, WriteTimeMs: 0, OpBytes: 8_192, WriteCountersTracked: true, StatsReset: null, + ReadBytes: 0, WriteBytes: 0, ExtendBytes: 0, ByteCountersTracked: false); + + [Fact] + public void IoStats_BothSharesAreOfTheWindow_NotOfThePage() + { + var cut = Parse(DarlingMcpPgIoTools.BuildIoJson( + "srv", 24, new DarlingPgIoReader.PgIoPage([Io(0, true), Io(1, true)], WindowTotal, WindowTotal / 10.0), limit: 1, timingSetting: true)); + AssertEnvelope(cut, "combinations", "combination_count", returned: 1, truncated: true, + "total_reads", WindowTotal, "returned_reads", 600, "returned_pct_of_total_reads"); + Assert.Equal(100.0, cut.GetProperty("total_read_time_ms").GetDouble()); + Assert.Equal(60.0, cut.GetProperty("returned_read_time_ms").GetDouble()); + Assert.Equal(60.0, cut.GetProperty("returned_pct_of_total_read_time").GetDouble()); + AssertShares(cut.GetProperty("combinations"), "pct_of_total_reads", 60); + AssertShares(cut.GetProperty("combinations"), "pct_of_total_read_time", 60); + Assert.Equal("read_time_ms_desc_then_reads_desc", cut.GetProperty("order").GetString()); + + var whole = Parse(DarlingMcpPgIoTools.BuildIoJson( + "srv", 24, new DarlingPgIoReader.PgIoPage([Io(0, true), Io(1, true), Io(2, true)], WindowTotal, WindowTotal / 10.0), limit: 3, timingSetting: true)); + AssertEnvelope(whole, "combinations", "combination_count", returned: 3, truncated: false, + "total_reads", WindowTotal, "returned_reads", 1_000, "returned_pct_of_total_reads"); + AssertShares(whole.GetProperty("combinations"), "pct_of_total_reads", 60, 30, 10); + AssertShares(whole.GetProperty("combinations"), "pct_of_total_read_time", 60, 30, 10); + } + + /// + /// The window's read-time total is a sum of stored zeros when track_io_timing is off, and it goes + /// null with every other time figure (#3536) — the read-count side is unaffected, because operation + /// counts are measured whatever the timing setting is. + /// + [Fact] + public void IoStats_UntrackedTiming_NullsTheReadTimeTotalsAndKeepsTheReadCounts() + { + var root = Parse(DarlingMcpPgIoTools.BuildIoJson( + "srv", 24, new DarlingPgIoReader.PgIoPage([Io(0, false), Io(1, false)], WindowTotal, 0), limit: 1, timingSetting: false)); + + Assert.Equal(JsonValueKind.Null, root.GetProperty("total_read_time_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, root.GetProperty("returned_read_time_ms").ValueKind); + Assert.Equal(JsonValueKind.Null, root.GetProperty("returned_pct_of_total_read_time").ValueKind); + Assert.Equal(JsonValueKind.Null, root.GetProperty("combinations")[0].GetProperty("pct_of_total_read_time").ValueKind); + + Assert.Equal(WindowTotal, root.GetProperty("total_reads").GetInt64()); + Assert.Equal(60.0, root.GetProperty("returned_pct_of_total_reads").GetDouble()); + Assert.Equal(60.0, root.GetProperty("combinations")[0].GetProperty("pct_of_total_reads").GetDouble()); + } + + /// + /// The timing INFERENCE (used only when the server's configuration was never collected) runs over the + /// cut page, not over the sentinel: a non-zero time that exists only on the over-fetched row is not + /// evidence the page can cite. + /// + [Fact] + public void IoStats_TheSentinelRow_DoesNotDecideTheTimingInference() + { + var root = Parse(DarlingMcpPgIoTools.BuildIoJson( + "srv", 24, new DarlingPgIoReader.PgIoPage([Io(0, false), Io(1, true)], WindowTotal, 30), limit: 1, timingSetting: null)); + + Assert.True(root.GetProperty("truncated").GetBoolean()); + Assert.False(root.GetProperty("io_timing_tracked").GetBoolean()); + } + + /* ───────────────────────── the boundary, once for the dialect ───────────────────────── */ + + /// + /// truncated is observed from the sentinel, never inferred from the cap: a page holding EXACTLY + /// limit rows is not truncated, and one holding limit + 1 is. count >= limit gets + /// the first case wrong, which is why every pair above seeds the whole window at limit = 3; this + /// states the rule once more at limit = 2 so it is not mistaken for a property of the number 3. + /// + [Fact] + public void Truncation_IsObservedFromTheSentinel_NotInferredFromTheCap() + { + var exactlyLimit = Parse(DarlingMcpPgWaitTools.BuildWaitStatsJson( + "srv", 24, new DarlingPgWaitReader.PgWaitStatsPage([Wait(0), Wait(1)], WindowTotal), limit: 2)); + Assert.False(exactlyLimit.GetProperty("truncated").GetBoolean()); + Assert.Equal(2, exactlyLimit.GetProperty("wait_events_returned").GetInt32()); + + var onePast = Parse(DarlingMcpPgWaitTools.BuildWaitStatsJson( + "srv", 24, new DarlingPgWaitReader.PgWaitStatsPage([Wait(0), Wait(1), Wait(2)], WindowTotal), limit: 2)); + Assert.True(onePast.GetProperty("truncated").GetBoolean()); + Assert.Equal(2, onePast.GetProperty("wait_events_returned").GetInt32()); + /* And the sentinel's own figure is not on the page: returned is 900, not 1,000. */ + Assert.Equal(900.0, onePast.GetProperty("returned_wait_time_ms").GetDouble()); + } +} diff --git a/Darling/Darling.Tests/DarlingMcpPgPlanToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPgPlanToolsTests.cs index 1dbf82747..f8a46e06e 100644 --- a/Darling/Darling.Tests/DarlingMcpPgPlanToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpPgPlanToolsTests.cs @@ -8,8 +8,14 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; using Xunit; @@ -128,6 +134,62 @@ public void AnUnparseablePlan_IsShownRatherThanDropped() Assert.Equal("{not valid json", plan.GetString()); } + /* ── the queryid filter and the empty answers (#3533) ── */ + + /// + /// The queryid filter runs IN the SQL, over every capture in the window. It used to be applied in C# + /// over a fetched top-duration page, which made any plan ranked below the page unfindable — the ranking + /// is by total duration, so a cheap-but-asked-about statement sits arbitrarily far down and no page + /// size reaches it. NULL must leave the read as the top page, which is what the OR arm is. + /// + [Fact] + public void TheQueryIdFilter_RunsInTheStore_NotOverAFetchedPage() + { + var sql = DarlingPgPlanCaptureReader.PgPlanCaptureSql; + + Assert.Contains("($4::bigint IS NULL OR query_id = $4)", sql, StringComparison.Ordinal); + Assert.Contains("LIMIT $5", sql, StringComparison.Ordinal); + } + + /// + /// The queryid miss says what was actually searched — the whole window, not a page — and names what a + /// miss can mean: the statement never ran (get_pg_top_queries confirms), it never crossed the capture + /// threshold, or capture was not working when it ran (get_pg_plan_capture_readiness has the facets). + /// The text this replaced declared the query "not the query to look at" over a fetched page the plan + /// could legitimately sit below, which is the confident wrong verdict #3533 exists to remove. + /// + [Fact] + public void TheQueryIdMiss_SaysTheWholeWindowWasSearched_AndWhatAMissCanMean() + { + var text = DarlingMcpPgPlanTools.NoPlanCapturedMessage(BigQueryId, 24); + + Assert.Contains("not a top-N page", text, StringComparison.Ordinal); + Assert.Contains("the last 24 hour(s)", text, StringComparison.Ordinal); + Assert.Contains("get_pg_top_queries", text, StringComparison.Ordinal); + Assert.Contains("auto_explain.log_min_duration", text, StringComparison.Ordinal); + Assert.Contains("get_pg_plan_capture_readiness", text, StringComparison.Ordinal); + Assert.Contains("plan_content_retention_days", text, StringComparison.Ordinal); + + Assert.DoesNotContain("not the query to look at", text, StringComparison.Ordinal); + } + + /// + /// The unfiltered miss is a statement about EVERY statement — the read has no filter, so zero rows + /// means the window is genuinely empty, and the message must not borrow the per-query verdict. + /// + [Fact] + public void TheUnfilteredMiss_IsAboutEveryStatement_AndNamesBothCauses() + { + var text = DarlingMcpPgPlanTools.NoPlanCapturedMessage(null, 48); + + Assert.Contains("every statement", text, StringComparison.Ordinal); + Assert.Contains("the last 48 hour(s)", text, StringComparison.Ordinal); + Assert.Contains("auto_explain.log_min_duration", text, StringComparison.Ordinal); + Assert.Contains("plan_content_retention_days", text, StringComparison.Ordinal); + + Assert.DoesNotContain("not the query to look at", text, StringComparison.Ordinal); + } + /* ── get_pg_plan_capture_readiness (#3070) ── */ /// @@ -264,3 +326,125 @@ public void ACappedResult_WithholdsTheUnsatisfiedSummary_RatherThanDescribingThe Assert.Contains("TRUNCATED", root.GetProperty("note").GetString(), StringComparison.Ordinal); } } + +/// +/// Gated (DARLING_TEST_PG) proof of #3533's mechanism, with the fixture the bug requires: a plan ranked +/// BELOW the page the old path fetched. The old code pulled the top limit * 10 shapes by total +/// duration and filtered them in C#, so with limit 2 a plan ranked 21st was unreachable at any +/// window size — and the miss was then reported as "capture is working, this plan was never captured". +/// The predicate now runs in the store, so the same call must return the plan; and a queryid that was +/// genuinely never captured must get the honest whole-window miss rather than the old verdict. +/// +[Collection("live-postgres")] +public sealed class DarlingMcpPgPlanQueryIdLiveTests +{ + private const string ServerName = "darling-pg-plans-queryid-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + + /// Past 2^53 and negative, the way real pg_stat_statements ids look (#2548). + private const long WantedQueryId = -8126435036642491494; + + /// Never seeded, so the filtered read over it must miss honestly. + private const long AbsentQueryId = -7000000000000000001; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task AQueryIdRankedBelowTheTopPage_IsFound_AndAGenuineMissIsReportedHonestly() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live get_pg_plans queryid test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await DarlingMcpTestData.ExecAsync(connection, ct, + "UPDATE servers SET engine_kind = $2 WHERE server_id = $1", + ServerId, MonitoredEngineKind.Postgres); + + var seen = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-30); + + /* 20 expensive shapes — exactly the limit * 10 page the OLD path fetched for limit 2 — and the + wanted plan 21st, cheaper than all of them. Ranked by total duration it sits one row below + everything the old path could ever see. */ + for (var i = 0; i < 20; i++) + { + await SeedCaptureAsync(connection, ct, seen, queryId: 9_100_000_000_000_000_001 + i, + planHash: $"EXPENSIVE{i:D2}", durationMs: 10_000 - (i * 100), + planJson: """{"Plan":{"Node Type":"Hash Join"}}"""); + } + + await SeedCaptureAsync(connection, ct, seen, WantedQueryId, + planHash: "WANTED", durationMs: 1.5, + planJson: """{"Plan":{"Node Type":"Index Scan","Relation Name":"orders"}}"""); + + /* The premise, demonstrated rather than assumed: the top page at this limit does not contain + the wanted plan. If this ever fails the fixture has stopped modeling the bug. */ + var topPage = JsonDocument.Parse( + await DarlingMcpPgPlanTools.GetPgPlans(postgres, ServerName, 24, 2)).RootElement; + var pageIds = topPage.GetProperty("plans").EnumerateArray() + .Select(p => p.GetProperty("queryid").GetString()) + .ToArray(); + Assert.Equal(2, pageIds.Length); + Assert.DoesNotContain(WantedQueryId.ToString(CultureInfo.InvariantCulture), pageIds); + + /* The fix: the same limit, pinned to the queryid, finds the plan the old path could not. */ + var found = JsonDocument.Parse(await DarlingMcpPgPlanTools.GetPgPlans( + postgres, ServerName, 24, 2, WantedQueryId.ToString(CultureInfo.InvariantCulture))).RootElement; + + var plan = Assert.Single(found.GetProperty("plans").EnumerateArray().ToArray()); + Assert.Equal(WantedQueryId.ToString(CultureInfo.InvariantCulture), + plan.GetProperty("queryid").GetString()); + Assert.Equal("WANTED", plan.GetProperty("plan_hash").GetString()); + Assert.Equal("Index Scan", + plan.GetProperty("plan").GetProperty("Plan").GetProperty("Node Type").GetString()); + + /* A queryid that was never captured now misses HONESTLY: the whole window was searched, and + the answer says what a miss can mean instead of declaring the query healthy. */ + var miss = JsonDocument.Parse(await DarlingMcpPgPlanTools.GetPgPlans( + postgres, ServerName, 24, 2, AbsentQueryId.ToString(CultureInfo.InvariantCulture))).RootElement; + + Assert.Equal("empty", miss.GetProperty("status").GetString()); + var message = miss.GetProperty("message").GetString()!; + Assert.Contains("not a top-N page", message, StringComparison.Ordinal); + Assert.Contains("get_pg_plan_capture_readiness", message, StringComparison.Ordinal); + Assert.DoesNotContain("not the query to look at", message, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static async Task SeedCaptureAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, + long queryId, string planHash, double durationMs, string planJson) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_plan_capture + (collection_id, collection_time, server_id, server_name, query_id, plan_hash, duration_ms, + node_count, top_node_type, plan_json) +VALUES ($1, $2, $3, $4, $5, $6, $7, 3, 'Seeded', $8)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(collectionTimeUtc), ServerId, ServerName, + queryId, planHash, durationMs, planJson); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM pg_plan_capture WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM pg_plan_capture_readiness WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM servers WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM config_monitored_servers WHERE server_id = $1", ServerId); + } +} diff --git a/Darling/Darling.Tests/DarlingMcpPlanCacheSchedulerToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPlanCacheSchedulerToolsTests.cs index d660da5e0..75be4d9f6 100644 --- a/Darling/Darling.Tests/DarlingMcpPlanCacheSchedulerToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpPlanCacheSchedulerToolsTests.cs @@ -73,12 +73,16 @@ public void ParamContract_PlanCacheBloat_ServerHours() Assert.All(p, x => Assert.True(x.Optional)); } + /// #3541 A10: the same (server_name, hours_back, as_of) surface Lite's twin has always had. The + /// tool took server_name alone here — two parameter surfaces under one tool name, on the one tool whose + /// answer is a CRITICAL/HIGH/MEDIUM/NORMAL verdict. McpLatestSnapshotStampTests pins the two SKUs' + /// descriptions equal; this pins the shape. [Fact] - public void ParamContract_CpuSchedulerPressure_ServerNameOnly() + public void ParamContract_CpuSchedulerPressure_ServerHoursAsOf_MatchesLite() { var p = McpParams("get_cpu_scheduler_pressure"); - Assert.Equal(new[] { "server_name" }, p.Select(x => x.Name).ToArray()); - Assert.True(p.Single().Optional); + Assert.Equal(new[] { "server_name", "hours_back", "as_of" }, p.Select(x => x.Name).ToArray()); + Assert.All(p, x => Assert.True(x.Optional)); } [Fact] @@ -103,6 +107,9 @@ public void CpuSchedulerPressureSql_LatestRow() Assert.Contains("worker_thread_exhaustion_warning", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY collection_time DESC", sql, StringComparison.Ordinal); Assert.Contains("LIMIT 1", sql, StringComparison.Ordinal); + /* #3541 A10: the newest row IN THE WINDOW, as Lite reads it — not the newest row the store ever held. */ + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); } [Theory] diff --git a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs index a2809ac0a..ed2a19c1f 100644 --- a/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpPlanCorrectionToolsTests.cs @@ -82,8 +82,11 @@ such database shows up as a phantom "recommendation" in the tool output. */ SqlTextPin.AssertExpresses("collection_time <= $3", sql, "the window's end no longer bounds the read"); SqlTextPin.AssertExpresses("ORDER BY collection_time DESC", sql, "the newest recommendations no longer come first"); - /* Mirrors the Viewer grid read's bound. */ - Assert.Contains("LIMIT 200", sql, StringComparison.Ordinal); + /* #3541 A3: the cap is the CALLER'S, bound as $4, not the Viewer grid's literal 200. The literal gave + every window the same ~16-hour reach over per-cycle re-captures and the tool published that page as + the window's count. */ + SqlTextPin.AssertExpresses("LIMIT $4", sql, "the row cap is no longer the caller's limit"); + Assert.DoesNotContain("LIMIT 200", sql, StringComparison.Ordinal); } [Fact] diff --git a/Darling/Darling.Tests/DarlingMcpServerAdminToolsTests.cs b/Darling/Darling.Tests/DarlingMcpServerAdminToolsTests.cs index f0b75e7af..5487bf951 100644 --- a/Darling/Darling.Tests/DarlingMcpServerAdminToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpServerAdminToolsTests.cs @@ -7,10 +7,13 @@ */ using System; +using System.Collections.Generic; using System.ComponentModel; +using System.IO; using System.Linq; using System.Reflection; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -161,10 +164,291 @@ never opened (the dead store would throw) and the throwing probe is never reache using var doc = JsonDocument.Parse(result); Assert.Equal(0, doc.RootElement.GetProperty("added").GetInt32()); Assert.Equal(0, doc.RootElement.GetProperty("skipped").GetInt32()); + Assert.Equal(0, doc.RootElement.GetProperty("collided").GetInt32()); Assert.Equal(1, doc.RootElement.GetProperty("failed").GetInt32()); + Assert.Equal(1, doc.RootElement.GetProperty("requested").GetInt32()); Assert.Equal("invalid", doc.RootElement.GetProperty("results")[0].GetProperty("status").GetString()); } + /* ---------------- #3541 A14: writes report what happened — every status lands in exactly one counter ---------------- */ + + /// Every per-row status the tool can produce, read off the constants class rather than restated + /// here, so a sixth status added there is in this census the moment it exists. + private static string[] AllAddStatuses() => typeof(DarlingMcpServerAdminTools.AddStatus) + .GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(f => f.IsLiteral && f.FieldType == typeof(string)) + .Select(f => (string)f.GetRawConstantValue()!) + .OrderBy(s => s, StringComparer.Ordinal) + .ToArray(); + + /// + /// The whole of the summary contract: the status set and the counter map's key set are the SAME set, and + /// every counter the map names is one the envelope carries. This is what makes "the four counters sum to + /// requested" a property of the code rather than of the batches anyone happened to test: a status without + /// a counter is what collides was for the months between #2280 and this — present in every result row, + /// absent from every summary number. + /// + [Fact] + public void EveryAddStatus_HasExactlyOneSummaryCounter_AndNoCounterIsUnnamed() + { + var statuses = AllAddStatuses(); + Assert.Equal(5, statuses.Length); + Assert.Contains("collides", statuses); + + var mapped = DarlingMcpServerAdminTools.CounterOfStatus.Keys.OrderBy(s => s, StringComparer.Ordinal).ToArray(); + Assert.Equal(statuses, mapped); + + var counters = new[] { "added", "skipped", "collided", "failed" }; + Assert.All(DarlingMcpServerAdminTools.CounterOfStatus.Values, c => Assert.Contains(c, counters)); + + /* And the one status that was missing has its OWN counter, not a seat inside failed: a collided entry + must not be retried as sent, which is exactly what a failed entry invites. */ + Assert.Equal("collided", DarlingMcpServerAdminTools.CounterOfStatus["collides"]); + Assert.Equal("skipped", DarlingMcpServerAdminTools.CounterOfStatus["duplicate"]); + } + + /// + /// The envelope over a batch that exercises EVERY status at once: requested is the input count, each + /// counter is the number of rows with the statuses mapped to it, and the four sum to requested. Two of + /// each so a counter that merely tested "any" would read wrong. + /// + [Fact] + public void Aggregate_CountsEveryResultOnce_AndTheCountersSumToRequested() + { + var results = new List + { + new(0, "a", DarlingMcpServerAdminTools.AddStatus.Added, "Connected"), + new(1, "b", DarlingMcpServerAdminTools.AddStatus.Collides, "lands elsewhere"), + new(2, "c", DarlingMcpServerAdminTools.AddStatus.Duplicate, "seen"), + new(3, "d", DarlingMcpServerAdminTools.AddStatus.ConnectionFailed, "no route"), + new(4, "e", DarlingMcpServerAdminTools.AddStatus.Invalid, "bad field"), + new(5, "f", DarlingMcpServerAdminTools.AddStatus.Added, "Connected"), + new(6, "g", DarlingMcpServerAdminTools.AddStatus.Collides, "lands elsewhere"), + new(7, "h", DarlingMcpServerAdminTools.AddStatus.Duplicate, "seen"), + new(8, "i", DarlingMcpServerAdminTools.AddStatus.ConnectionFailed, "no route"), + new(9, "j", DarlingMcpServerAdminTools.AddStatus.Invalid, "bad field"), + }; + + using var doc = JsonDocument.Parse(DarlingMcpServerAdminTools.Aggregate(results)); + var root = doc.RootElement; + + Assert.Equal(10, root.GetProperty("requested").GetInt32()); + Assert.Equal(2, root.GetProperty("added").GetInt32()); + Assert.Equal(2, root.GetProperty("skipped").GetInt32()); + Assert.Equal(2, root.GetProperty("collided").GetInt32()); + Assert.Equal(4, root.GetProperty("failed").GetInt32()); + + var sum = root.GetProperty("added").GetInt32() + root.GetProperty("skipped").GetInt32() + + root.GetProperty("collided").GetInt32() + root.GetProperty("failed").GetInt32(); + Assert.Equal(root.GetProperty("requested").GetInt32(), sum); + + /* Results echo input order and carry the per-row status the counters were derived from. */ + var statuses = root.GetProperty("results").EnumerateArray().Select(r => r.GetProperty("status").GetString()).ToArray(); + Assert.Equal(results.Select(r => r.Status).ToArray(), statuses); + } + + /// A status the map does not know is a LOUD failure, never a row that quietly counts toward + /// nothing — the failure shape the old three-filter summary had. + [Fact] + public void Aggregate_RefusesAStatusNoCounterAccountsFor() + { + var results = new List + { + new(0, "a", DarlingMcpServerAdminTools.AddStatus.Added, "Connected"), + new(1, "b", "quarantined", "a status nobody mapped"), + }; + + var ex = Assert.Throws(() => DarlingMcpServerAdminTools.Aggregate(results)); + Assert.Contains("quarantined", ex.Message, StringComparison.Ordinal); + } + + /// + /// Every result-construction site in the tool names its status through AddStatus, never as a bare + /// literal — the guard that keeps the census above complete. A literal at a new site would compile, be + /// absent from the constants class, and so be absent from this file's reflection; this pin is what turns + /// that into a red run instead of a green one over an incomplete set. + /// + [Fact] + public void EveryServerResultConstruction_NamesItsStatusThroughAddStatus() + { + var source = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpServerAdminTools.cs"); + var code = CSharpSourceWalker.StripCommentsAndStrings(source); + + /* Both construction spellings: the explicit `new ServerResult(` and the target-typed `=> new(` inside + the local Invalid() factory. The record DECLARATION is a `ServerResult(` too and is excluded by its + parameter list. */ + var sites = Regex.Matches(code, @"(?new)\s+ServerResult\s*\(|ServerResult\s+Invalid\s*\([^)]*\)\s*=>\s*(?new)\s*\(") + .Select(m => CSharpSourceWalker.ConstructionSpanFrom(code, m.Groups["new"].Index)) + .ToList(); + + Assert.True(sites.Count >= 5, $"expected the five result-construction sites, found {sites.Count} — the scan is broken"); + Assert.All(sites, span => Assert.Contains("AddStatus.", span, StringComparison.Ordinal)); + } + + /// The tool description promises the counters and the sum; a caller reads the description, not the + /// code. + [Fact] + public void AddServersDescription_NamesEveryCounter_TheSumRule_AndTheCollidesStatus() + { + var method = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "add_servers"); + var description = method.GetCustomAttribute()!.Description; + + foreach (var token in new[] { "requested:N", "added:N", "skipped:N", "collided:N", "failed:N", "SUM", "\"collides\"" }) + { + Assert.Contains(token, description, StringComparison.Ordinal); + } + + /* And the instruction table agrees with the description — the two surfaces an agent reads. */ + Assert.Contains("`{requested, added, skipped, collided, failed, results:[{server, status, detail}]}`", DarlingMcpInstructions.Text, StringComparison.Ordinal); + Assert.Contains("is `collides`", DarlingMcpInstructions.Text, StringComparison.Ordinal); + } + + /// + /// #3484 accepted the two non-interactive Entra modes; the instruction table went on saying + /// "Entra/MFA/Service-Principal/Managed-Identity auth is `invalid` (Windows/SQL only)" for a release. The + /// accepted set is read off the parser — the authority — and the table is held to it. + /// + [Fact] + public void InstructionsTable_MatchesTheAuthModesTheParserAccepts() + { + /* The parser is the authority: these two are accepted (they parse to an entry) ... */ + foreach (var accepted in new[] { "ServicePrincipal", "ManagedIdentity" }) + { + var json = accepted == "ManagedIdentity" + ? "[{\"host\":\"x\",\"auth\":\"ManagedIdentity\"}]" + : "[{\"host\":\"x\",\"auth\":\"ServicePrincipal\",\"username\":\"app\",\"password\":\"s\"}]"; + var (entries, invalid, wholeError) = DarlingMcpServerAdminTools.ParseRequest(json); + Assert.Null(wholeError); + Assert.Empty(invalid); + Assert.Single(entries); + } + + /* ... and these are refused. */ + foreach (var refused in new[] { "EntraMFA", "EntraDeviceCodeAuth", "EntraDefaultCredential" }) + { + var (entries, invalid, _) = DarlingMcpServerAdminTools.ParseRequest($"[{{\"host\":\"x\",\"auth\":\"{refused}\"}}]"); + Assert.Empty(entries); + Assert.Single(invalid); + } + + /* The table says the same: the accepted pair is named as accepted, the interactive trio as invalid, and + the release-old denial is gone in both of its spellings. */ + var row = DarlingMcpInstructions.Text.Split('\n').Single(l => l.Contains("| `add_servers` |", StringComparison.Ordinal)); + Assert.Contains("`ServicePrincipal` / `ManagedIdentity`", row, StringComparison.Ordinal); + Assert.Contains("INTERACTIVE Entra modes (MFA / device-code / default-credential) are `invalid`", row, StringComparison.Ordinal); + Assert.DoesNotContain("Windows/SQL only", row, StringComparison.Ordinal); + Assert.DoesNotContain("Service-Principal/Managed-Identity auth is `invalid`", row, StringComparison.Ordinal); + + /* And the tool description — the other surface an agent reads — agrees. */ + var method = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "add_servers"); + var description = method.GetCustomAttribute()!.Description; + Assert.Contains("\"ServicePrincipal\"", description, StringComparison.Ordinal); + Assert.Contains("\"ManagedIdentity\"", description, StringComparison.Ordinal); + } + + /* ---------------- #3541 A14: remove_server refuses what it cannot honor ---------------- */ + + private static DarlingServerResolver.RegisteredServer Row(int id, string name, string? display = null) => new(id, name, display); + + [Fact] + public void ResolveForRemoval_ExactStorageOrDisplayName_IsOneCandidate_MatchedExact() + { + var servers = new[] { Row(1, "sql-01", "Payments"), Row(2, "sql-02", "Ledger") }; + + var byStorage = DarlingMcpServerAdminTools.ResolveForRemoval(servers, "SQL-02"); + Assert.Equal("exact", byStorage.MatchedBy); + Assert.Equal(2, Assert.Single(byStorage.Candidates).ServerId); + + var byDisplay = DarlingMcpServerAdminTools.ResolveForRemoval(servers, " payments "); + Assert.Equal("exact", byDisplay.MatchedBy); + Assert.Equal(1, Assert.Single(byDisplay.Candidates).ServerId); + } + + /// THE defect: a fragment two siblings contain. The read resolver returns the first by storage-name + /// order; a delete must return both and choose neither. + [Fact] + public void ResolveForRemoval_FragmentSeveralServersContain_IsEveryCandidate_NotTheFirst() + { + var servers = new[] { Row(1, "sql-01"), Row(2, "sql-02"), Row(3, "pg-01") }; + + var target = DarlingMcpServerAdminTools.ResolveForRemoval(servers, "sql-"); + + Assert.Equal("partial", target.MatchedBy); + Assert.Equal(new[] { 1, 2 }, target.Candidates.Select(c => c.ServerId).OrderBy(i => i).ToArray()); + } + + /// A partial that only ONE server contains is honored — the documented convenience survives where it + /// is unambiguous. + [Fact] + public void ResolveForRemoval_UniquePartial_IsOneCandidate_MatchedPartial() + { + var servers = new[] { Row(1, "sql-01", "Payments"), Row(2, "sql-02", "Ledger") }; + + var target = DarlingMcpServerAdminTools.ResolveForRemoval(servers, "ledg"); + + Assert.Equal("partial", target.MatchedBy); + Assert.Equal(2, Assert.Single(target.Candidates).ServerId); + } + + /// An exact match wins over the partials that would otherwise also match it: "sql-01" against + /// "sql-01" and "sql-010" is one server, not two. + [Fact] + public void ResolveForRemoval_ExactBeatsPartial_EvenWhenThePartialWouldBeAmbiguous() + { + var servers = new[] { Row(1, "sql-01"), Row(2, "sql-010") }; + + var target = DarlingMcpServerAdminTools.ResolveForRemoval(servers, "sql-01"); + + Assert.Equal("exact", target.MatchedBy); + Assert.Equal(1, Assert.Single(target.Candidates).ServerId); + } + + /// display_name is not unique; two registrations sharing one exactly are ambiguous too, reported as + /// such rather than resolved to whichever the registry returned first. + [Fact] + public void ResolveForRemoval_SharedDisplayName_IsAmbiguousExact() + { + var servers = new[] { Row(1, "sql-01", "Prod"), Row(2, "sql-01:AppDb", "Prod") }; + + var target = DarlingMcpServerAdminTools.ResolveForRemoval(servers, "Prod"); + + Assert.Equal("exact", target.MatchedBy); + Assert.Equal(2, target.Candidates.Count); + } + + [Theory] + [InlineData("nothing-like-it")] + [InlineData("")] + [InlineData(" ")] + public void ResolveForRemoval_NoMatch_IsZeroCandidates(string name) + { + var servers = new[] { Row(1, "sql-01"), Row(2, "sql-02") }; + + var target = DarlingMcpServerAdminTools.ResolveForRemoval(servers, name); + + Assert.Empty(target.Candidates); + Assert.Equal("none", target.MatchedBy); + } + + /// The description tells the caller what an ambiguous name does (nothing) and what comes back; the + /// instruction table says the same. + [Fact] + public void RemoveServerDescription_PromisesTheAmbiguousRefusal() + { + var method = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "remove_server"); + var description = method.GetCustomAttribute()!.Description; + + Assert.Contains("status:\"ambiguous\"", description, StringComparison.Ordinal); + Assert.Contains("NOTHING is deleted", description, StringComparison.Ordinal); + Assert.Contains("ONLY when exactly one", description, StringComparison.Ordinal); + Assert.DoesNotContain("resolved the same way the read tools resolve", description, StringComparison.Ordinal); + + var row = DarlingMcpInstructions.Text.Split('\n').Single(l => l.Contains("| `remove_server` |", StringComparison.Ordinal)); + Assert.Contains("status:\"ambiguous\"", row, StringComparison.Ordinal); + Assert.Contains("deletes NOTHING", row, StringComparison.Ordinal); + } + [Theory] [InlineData("")] [InlineData(" ")] @@ -299,9 +583,11 @@ public void PartitionDuplicates_EntryMatchingAnExistingServer_IsSkipped() /// probe is STUBBED to success (no live SQL Server), so this exercises the STORE side: add_servers INSERTs the /// config_monitored_servers rows (SQL-auth password DPAPI-encrypted at rest, Windows-auth server secret-free), /// the case-folded duplicate is skipped, the write self-bumps config_version (the reload beacon) via the existing -/// trigger, and remove_server resolves + DELETEs. Own-scoped per the shared-store doctrine (GUID-suffixed hosts + -/// a finally cleanup). No live SQL Server connection is made in CI — the real end-to-end probe is what the human -/// dogfoods (remove sql2016, re-add via MCP). +/// trigger, a #2280 collision is refused and COUNTED (#3541 A14: collided, with the four counters summing +/// to requested), and remove_server refuses an ambiguous fragment with both candidates named, honors a unique +/// partial, and DELETEs on an exact name. Own-scoped per the shared-store doctrine (GUID-suffixed hosts + a finally +/// cleanup). No live SQL Server connection is made in CI — the real end-to-end probe is what the human dogfoods +/// (remove sql2016, re-add via MCP). /// [Collection("live-postgres")] public sealed class DarlingMcpServerAdminToolsLivePostgresTests @@ -314,6 +600,15 @@ public sealed class DarlingMcpServerAdminToolsLivePostgresTests Success: true, MajorVersion: 15, EngineEdition: 3, EngineEditionDescription: "Enterprise", IsAzureSqlDb: false, IsAzureManagedInstance: false, IsAwsRds: false, HasMsdbAccess: true, Error: null)); + /// The same healthy box, but the probe REPORTS which database the connection reached — the #2280 + /// input. Every entry in a batch probed through this lands in , whatever + /// database it declared. + private static DarlingMcpServerAdminTools.ServerProbe ProbeReaching(string connectedDatabase) => + (_, _) => Task.FromResult(new ConnectionProbeResult( + Success: true, MajorVersion: 15, EngineEdition: 3, EngineEditionDescription: "Enterprise", + IsAzureSqlDb: false, IsAzureManagedInstance: false, IsAwsRds: false, HasMsdbAccess: true, Error: null, + ConnectedDatabase: connectedDatabase)); + [Fact] public async Task AddServers_InsertsEncryptsDedupesBumpsVersion_ThenRemoveServer_AgainstDevPostgres() { @@ -337,11 +632,16 @@ public async Task AddServers_InsertsEncryptsDedupesBumpsVersion_ThenRemoveServer var suffix = Guid.NewGuid().ToString("N")[..12]; var sqlHost = "mcp-add-sql-" + suffix; var winHost = "mcp-add-win-" + suffix; + var dbHost = "mcp-add-db-" + suffix; var sqlId = ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(sqlHost, null, false)); var winId = ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(winHost, null, false)); + /* #3541 A14: the collision batch's two identities — the one that lands (dbHost:AppDb) and the one that + must NOT (dbHost:Decoy), listed for cleanup so a regression that inserted it does not leak a row. */ + var dbAppId = ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(dbHost, "AppDb", false)); + var dbDecoyId = ServerIdHelper.GetDeterministicHashCode(ServerIdHelper.BuildStorageName(dbHost, "Decoy", false)); var password = "P@ss-" + Guid.NewGuid().ToString("N"); - await CleanupAsync(connection, ct, sqlId, winId); + await CleanupAsync(connection, ct, sqlId, winId, dbAppId, dbDecoyId); await DarlingMcpTestData.ExecAsync(connection, ct, "INSERT INTO config_service (id) VALUES (1) ON CONFLICT (id) DO NOTHING"); var bodySucceeded = false; @@ -358,8 +658,10 @@ public async Task AddServers_InsertsEncryptsDedupesBumpsVersion_ThenRemoveServer var added = await DarlingMcpServerAdminTools.AddServersAsync(postgres, json, SuccessProbe, ct); using (var doc = JsonDocument.Parse(added)) { + Assert.Equal(3, doc.RootElement.GetProperty("requested").GetInt32()); Assert.Equal(2, doc.RootElement.GetProperty("added").GetInt32()); Assert.Equal(1, doc.RootElement.GetProperty("skipped").GetInt32()); + Assert.Equal(0, doc.RootElement.GetProperty("collided").GetInt32()); Assert.Equal(0, doc.RootElement.GetProperty("failed").GetInt32()); } @@ -389,12 +691,71 @@ existing trg_bump_monitored_servers trigger — proving the mcp beacon column-gr { Assert.Equal(0, doc.RootElement.GetProperty("added").GetInt32()); Assert.Equal(2, doc.RootElement.GetProperty("skipped").GetInt32()); + Assert.Equal(2, doc.RootElement.GetProperty("requested").GetInt32()); } - /* remove_server resolves against the servers registry, so register the SQL host there (same server_id - config_monitored_servers keys on), then remove — the config row is deleted. */ + /* #3541 A14 — the batch that used to summarise as a clean run. Three entries on one host, probed + through a stub that reports every connection landing in AppDb: the first names AppDb and is added; + the second names Decoy but lands in AppDb, which the first now claims → collides, NOT inserted; + the third is a case-variant duplicate of the first → skipped. The old envelope read + {added: 1, skipped: 1, failed: 0} for this — one entry unaccounted for, and it was the one not + being monitored. */ + var collisionBatch = await DarlingMcpServerAdminTools.AddServersAsync( + postgres, + $"[{{\"host\":\"{dbHost}\",\"database\":\"AppDb\"}}," + + $"{{\"host\":\"{dbHost}\",\"database\":\"Decoy\"}}," + + $"{{\"host\":\"{dbHost}\",\"database\":\"appdb\"}}]", + ProbeReaching("AppDb"), ct); + using (var doc = JsonDocument.Parse(collisionBatch)) + { + var root = doc.RootElement; + Assert.Equal(3, root.GetProperty("requested").GetInt32()); + Assert.Equal(1, root.GetProperty("added").GetInt32()); + Assert.Equal(1, root.GetProperty("skipped").GetInt32()); + Assert.Equal(1, root.GetProperty("collided").GetInt32()); + Assert.Equal(0, root.GetProperty("failed").GetInt32()); + + var statuses = root.GetProperty("results").EnumerateArray().Select(r => r.GetProperty("status").GetString()).ToArray(); + Assert.Equal(new[] { "added", "collides", "duplicate" }, statuses); + } + + /* And the store agrees with the counters: the AppDb identity exists, the Decoy identity does not. */ + Assert.Equal(1L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {dbAppId}"))); + Assert.Equal(0L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {dbDecoyId}"))); + + /* remove_server resolves against the servers registry, so register BOTH hosts there (same server_id + config_monitored_servers keys on). */ await DarlingMcpTestData.RegisterServerAsync(connection, sqlId, sqlHost, ct); - Assert.Equal("removed", DarlingMcpTestData.StatusOf(await DarlingMcpServerAdminTools.RemoveServer(postgres, sqlHost))); + await DarlingMcpTestData.RegisterServerAsync(connection, winId, winHost, ct); + + /* #3541 A14: the GUID suffix is a fragment BOTH registered names contain. The read resolver would + hand back whichever sorts first; the delete must refuse, name both, and remove neither. */ + using (var doc = JsonDocument.Parse(await DarlingMcpServerAdminTools.RemoveServer(postgres, suffix))) + { + Assert.Equal("ambiguous", doc.RootElement.GetProperty("status").GetString()); + Assert.Equal("partial", doc.RootElement.GetProperty("matched_by").GetString()); + var candidates = doc.RootElement.GetProperty("candidates").EnumerateArray() + .Select(c => c.GetProperty("server").GetString()).OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Assert.Equal(new[] { sqlHost, winHost }.OrderBy(n => n, StringComparer.Ordinal).ToArray(), candidates); + } + Assert.Equal(1L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {sqlId}"))); + Assert.Equal(1L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {winId}"))); + + /* A partial that only ONE registered name contains is honored, and says so. */ + using (var doc = JsonDocument.Parse(await DarlingMcpServerAdminTools.RemoveServer(postgres, "add-win-" + suffix))) + { + Assert.Equal("removed", doc.RootElement.GetProperty("status").GetString()); + Assert.Equal(winHost, doc.RootElement.GetProperty("server").GetString()); + Assert.Equal("partial", doc.RootElement.GetProperty("matched_by").GetString()); + } + Assert.Equal(0L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {winId}"))); + + /* The exact name removes the SQL host — the config row is deleted. */ + using (var doc = JsonDocument.Parse(await DarlingMcpServerAdminTools.RemoveServer(postgres, sqlHost))) + { + Assert.Equal("removed", doc.RootElement.GetProperty("status").GetString()); + Assert.Equal("exact", doc.RootElement.GetProperty("matched_by").GetString()); + } Assert.Equal(0L, Convert.ToInt64(await ScalarAsync(connection, ct, $"SELECT count(*) FROM config_monitored_servers WHERE server_id = {sqlId}"))); /* Remove again: the servers-registry row still resolves, but the config row is gone → not_found. */ @@ -408,7 +769,7 @@ existing trg_bump_monitored_servers trigger — proving the mcp beacon column-gr finally { await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => - await CleanupAsync(cleanup, cleanupCt, sqlId, winId)); + await CleanupAsync(cleanup, cleanupCt, sqlId, winId, dbAppId, dbDecoyId)); } } @@ -418,9 +779,10 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = return await command.ExecuteScalarAsync(ct); } - private static async Task CleanupAsync(NpgsqlConnection connection, CancellationToken ct, int sqlId, int winId) + private static async Task CleanupAsync(NpgsqlConnection connection, CancellationToken ct, params int[] serverIds) { - await DarlingMcpTestData.ExecAsync(connection, ct, $"DELETE FROM config_monitored_servers WHERE server_id IN ({sqlId}, {winId})"); - await DarlingMcpTestData.ExecAsync(connection, ct, $"DELETE FROM servers WHERE server_id IN ({sqlId}, {winId})"); + var list = string.Join(", ", serverIds); + await DarlingMcpTestData.ExecAsync(connection, ct, $"DELETE FROM config_monitored_servers WHERE server_id IN ({list})"); + await DarlingMcpTestData.ExecAsync(connection, ct, $"DELETE FROM servers WHERE server_id IN ({list})"); } } diff --git a/Darling/Darling.Tests/DarlingMcpSessionToolsTests.cs b/Darling/Darling.Tests/DarlingMcpSessionToolsTests.cs index 9f4850119..b77b656ab 100644 --- a/Darling/Darling.Tests/DarlingMcpSessionToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpSessionToolsTests.cs @@ -115,6 +115,10 @@ public void WaitingTasksSql_ReadsBaseTable_NoView_Windowed() Assert.Contains("wait_duration_ms", sql, StringComparison.Ordinal); Assert.Contains("resource_description", sql, StringComparison.Ordinal); Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + /* #3541 A3: the cap is the caller's ($4), not the 500 the reader hid under a tool that advertised + `limit` and then published a bare envelope. */ + Assert.Contains("LIMIT $4", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIMIT 500", sql, StringComparison.Ordinal); } [Theory] diff --git a/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs b/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs index 03e368fdc..6f46ba491 100644 --- a/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpStoreMetricsToolsTests.cs @@ -8,8 +8,11 @@ using System; using System.ComponentModel; +using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using ModelContextProtocol.Server; using Npgsql; @@ -102,6 +105,10 @@ grain a growth question wants. */ [Theory] [InlineData(nameof(DarlingStoreMetricsReader.StoreMetricsLatestSql))] [InlineData(nameof(DarlingStoreMetricsReader.StoreMetricsDailySql))] + [InlineData(nameof(DarlingStoreMetricsReader.JobHistoryEvidenceSql))] + [InlineData(nameof(DarlingStoreMetricsReader.ContinuousAggregateStateSql))] + [InlineData(nameof(DarlingStoreMetricsReader.LargestUnenumeratedSql))] + [InlineData(nameof(DarlingStoreMetricsReader.LargestUnenumeratedPlainSql))] public void Reads_ArePostgresDialect_NoTsqlIsms_NoBareNow(string sqlName) { var sql = (string)typeof(DarlingStoreMetricsReader).GetField(sqlName, BindingFlags.Public | BindingFlags.Static)!.GetValue(null)!; @@ -216,9 +223,13 @@ public void TheJobHistoryNote_IsDifferentForEveryState_AndSplitsOffOnWhoSetIt() vacuous, and the whole point of four states is that there are four. */ Assert.Equal(4, statuses.Length); + /* NotApplicable evidence on purpose: it contributes NO text (pinned by its own test below), so what + is compared here is the GUC half alone — the #3175 arms, byte-for-byte what they were before the + evidence half was appended to them. */ var notes = statuses .Select(s => DarlingMcpStoreMetricsTools.JobHistoryNote( - new DarlingStoreMetricsReader.JobExecutionLoggingReading(s, null, null, null))) + new DarlingStoreMetricsReader.JobExecutionLoggingReading(s, null, null, null), + DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable)) .ToArray(); Assert.Equal(notes.Length, notes.Distinct(StringComparer.Ordinal).Count()); @@ -235,8 +246,8 @@ conf append at all (postgresql.auto.conf is read last) and needs the override re Assert.False(offByDefault.OffByExplicitOverride); Assert.True(offByOverride.OffByExplicitOverride); Assert.NotEqual( - DarlingMcpStoreMetricsTools.JobHistoryNote(offByDefault), - DarlingMcpStoreMetricsTools.JobHistoryNote(offByOverride)); + DarlingMcpStoreMetricsTools.JobHistoryNote(offByDefault, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable), + DarlingMcpStoreMetricsTools.JobHistoryNote(offByOverride, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable)); /* Only On is Recording. A precondition check whose "yes" leaked into any other state would put a maximum question back on an instrument that is off. */ @@ -253,6 +264,943 @@ maximum question back on an instrument that is off. */ .OffByExplicitOverride); } + /* ---------------- #3574: the evidence behind the flag, and who the rows are visible to ---------------- */ + + private static DarlingStoreMetricsReader.JobExecutionLoggingReading On() => new( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.On, "on", "configuration file", null); + + private static DarlingStoreMetricsReader.JobExecutionLoggingReading OffByDefault() => new( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.Off, "off", "default", null); + + /// An Observed reading with every fact stated, so a test flips exactly the one it is + /// about. The defaults are the OWNER's reading on a live store: database-owner member, 110 jobs, 12 + /// rows in the window, 9 jobs started a run in it. + private static DarlingStoreMetricsReader.JobHistoryEvidence Observed( + string role = "darling", + bool dbOwnerMember = true, + long jobs = 110, + long ownerMemberJobs = 110, + long rows = 12, + DateTime? newestRow = null, + long ran = 9, + DateTime? newestRun = null) => new( + DarlingStoreMetricsReader.JobHistoryEvidenceStatus.Observed, + role, dbOwnerMember, jobs, ownerMemberJobs, rows, + newestRow ?? (rows > 0 ? new DateTime(2026, 9, 18, 10, 0, 0, DateTimeKind.Utc) : null), + ran, + newestRun ?? (ran > 0 ? new DateTime(2026, 9, 18, 10, 30, 0, DateTimeKind.Utc) : null)); + + /// The managed-mode mcp role's reading on the same store: a member of nothing, and the + /// view shows it nothing — zero rows, while the unfiltered job_stats still counts the runs. + private static DarlingStoreMetricsReader.JobHistoryEvidence FilteredReader() => + Observed(role: "mcp", dbOwnerMember: false, ownerMemberJobs: 0, rows: 0, newestRow: null); + + /// + /// #3574: the evidence read evaluates the view's OWN predicate for the connection doing the reading, + /// and takes its two counts from the two views that disagree about visibility. + /// + /// Both pg_has_role tests, in the view's own terms. The membership in the database + /// owner (resolved through pg_get_userbyid(datdba), as the view does) and the per-job membership + /// in j.owner. Pinned because a read that counted rows without asking whether it was allowed to + /// see any would report zero on every managed store — the MCP host connects as the mcp role, which + /// the view filters out — and manufacture the contradiction the issue exists to prevent. + /// + /// The population half comes from the UNFILTERED view. job_stats has no ownership + /// clause, so "jobs started a run in the window" holds whatever the reader's standing; taken from the + /// filtered view it would be zero exactly when the count it was meant to qualify is zero, and the pair + /// would agree for the wrong reason. + /// + [Fact] + public void TheEvidenceSql_EvaluatesTheViewsOwnPredicate_AndCountsFromBothViews() + { + var sql = DarlingStoreMetricsReader.JobHistoryEvidenceSql; + + /* The view's two tests, evaluated for this reader. IS TRUE mirrors the view, whose own second test + can meet a NULL owner (a history row whose job was deleted) and must read it as "not a member". */ + Assert.Contains("current_user::text", sql, StringComparison.Ordinal); + Assert.Matches(@"pg_has_role\(\s*current_user,\s*\(SELECT pg_get_userbyid\(datdba\) FROM pg_database WHERE datname = current_database\(\)\),\s*'MEMBER'\) IS TRUE", sql); + Assert.Matches(@"pg_has_role\(current_user, j\.owner, 'MEMBER'\) IS TRUE", sql); + + /* Three views: the filtered one being counted, and the two unfiltered ones the reader checks first. */ + Assert.Contains("FROM timescaledb_information.job_history", sql, StringComparison.Ordinal); + Assert.Contains("FROM timescaledb_information.jobs", sql, StringComparison.Ordinal); + Assert.Contains("FROM timescaledb_information.job_stats", sql, StringComparison.Ordinal); + + /* One window, bound once, applied to both halves — so the count and its population share a + denominator. */ + Assert.Contains("h.start_time >= $1", sql, StringComparison.Ordinal); + Assert.Contains("js.last_run_started_at >= $1", sql, StringComparison.Ordinal); + Assert.DoesNotContain("$2", sql, StringComparison.Ordinal); + + /* The never-ran sentinel is -infinity, not NULL (#1760); the newest start must NULLIF it away or a + store whose jobs have never run reports a start in 4714 BC. */ + Assert.Contains("NULLIF(js.last_run_started_at, '-infinity'::timestamptz)", sql, StringComparison.Ordinal); + + /* The window is fixed and published beside the count. 24 hours: every job this product schedules + runs at least daily, so a live store always has starts inside it, and it sits inside the history + view's own one-month default retention. */ + Assert.Equal(24, DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours); + } + + /// + /// #3574: the bind is timestamptz with Kind = Utc, stated explicitly — the INVERSE of + /// the naive-UTC discipline every other store read follows, because these are TimescaleDB's own + /// TIMESTAMPTZ catalog columns and here the naive bind would be the bug. A source pin, since the + /// only server that would catch the wrong Kind is one running off UTC, which no test store does. + /// + [Fact] + public void TheEvidenceRead_BindsAnExplicitTimestampTz_BecauseTheColumnsAreTimestampTz() + { + var source = File.ReadAllText(ReaderSourcePath()); + var method = source[source.IndexOf("GetJobHistoryEvidenceAsync(", StringComparison.Ordinal)..]; + method = method[..method.IndexOf("/// One object's newest self-metrics row", StringComparison.Ordinal)]; + + Assert.Contains("NpgsqlDbType.TimestampTz", method, StringComparison.Ordinal); + Assert.Contains("DateTimeKind.Utc", method, StringComparison.Ordinal); + /* And NOT the naive idiom, which is correct one method up and wrong here. */ + Assert.DoesNotContain("DateTimeKind.Unspecified", method, StringComparison.Ordinal); + } + + /// + /// #3574: is derived from the + /// view's two membership facts exactly as the view combines them — database-owner membership sees + /// everything and short-circuits the per-job test; otherwise the per-job count decides — and is + /// Unknown wherever a verdict would be vacuous. + /// + [Fact] + public void TheVisibility_IsDerivedFromTheViewsTwoTests_TheWayTheViewCombinesThem() + { + /* The owner: a member of the database owner, so All — regardless of the per-job count, which the + view never reaches for such a reader. */ + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.All, Observed().Visibility); + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.All, Observed(ownerMemberJobs: 0).Visibility); + Assert.Equal(110L, Observed(ownerMemberJobs: 0).HistoryVisibleJobCount); + + /* Not the database owner, but a member of every job's owner: still All. */ + Assert.Equal( + DarlingStoreMetricsReader.JobHistoryVisibility.All, + Observed(dbOwnerMember: false, ownerMemberJobs: 110).Visibility); + + /* The managed-mode mcp role: a member of neither. None, and the visible count is zero. */ + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.None, FilteredReader().Visibility); + Assert.Equal(0L, FilteredReader().HistoryVisibleJobCount); + + /* Some jobs' owner but not all: Partial, with the fraction preserved for the note. */ + var partial = Observed(dbOwnerMember: false, ownerMemberJobs: 3); + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.Partial, partial.Visibility); + Assert.Equal(3L, partial.HistoryVisibleJobCount); + + /* No jobs at all: None and All are both vacuously true, so neither is claimed. */ + Assert.Equal( + DarlingStoreMetricsReader.JobHistoryVisibility.Unknown, + Observed(jobs: 0, ownerMemberJobs: 0, ran: 0, rows: 0).Visibility); + + /* And nothing was observed: nothing is derived. Every derived field is null, never a plausible zero. */ + foreach (var blank in new[] + { + DarlingStoreMetricsReader.JobHistoryEvidence.Unreadable, + DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable, + }) + { + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.Unknown, blank.Visibility); + Assert.Null(blank.HistoryVisibleJobCount); + Assert.Null(blank.RowsObserved); + Assert.Null(blank.NewestRowAt); + Assert.Null(blank.ReaderRole); + } + } + + /// + /// #3574: the contradiction is the conjunction of FOUR conditions, and each one alone is a different, + /// non-finding shape. A zero read through a filtered role is the filter; a zero with no runs in the + /// window is an absence of information; a zero with the GUC off is #3175's arm; rows seen is recording + /// proven. Only all four together say the instrument is not writing what the GUC says it is. Each row of + /// the table flips exactly one condition off the positive control, so the pin cannot pass by a + /// predicate that is simply always false. + /// + [Fact] + public void TheContradiction_NeedsAllFourConditions_AndEachAloneIsNotOne() + { + var positive = Observed(rows: 0, newestRow: null); + Assert.True(positive.ContradictsRecording(recording: true)); + + /* 1. Not recording: the GUC-off arm, not this one. */ + Assert.False(positive.ContradictsRecording(recording: false)); + + /* 2. Reader filtered: the mcp role's zero is the view's doing. Partial is not enough either — the + jobs that ran may be the ones this reader cannot see. */ + Assert.False(FilteredReader().ContradictsRecording(recording: true)); + Assert.False(Observed(dbOwnerMember: false, ownerMemberJobs: 3, rows: 0, newestRow: null).ContradictsRecording(recording: true)); + + /* 3. Rows seen: recording is proven, whatever the run count. */ + Assert.False(Observed(rows: 1).ContradictsRecording(recording: true)); + + /* 4. Nothing ran: nothing to record, so nothing is contradicted. */ + Assert.False(Observed(rows: 0, newestRow: null, ran: 0, newestRun: null).ContradictsRecording(recording: true)); + + /* And not-observed evidence never contradicts anything: a read that did not complete has no + standing to declare a finding. */ + Assert.False(DarlingStoreMetricsReader.JobHistoryEvidence.Unreadable.ContradictsRecording(recording: true)); + Assert.False(DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable.ContradictsRecording(recording: true)); + } + + /// + /// #3574: the note names the visibility rule and the reader on every arm where the view exists, and + /// says a DIFFERENT thing for each thing the evidence established — including the new contradiction + /// arm, in so many words. + /// + /// Distinctness across evidence shapes with the GUC held constant, the same claim the + /// #3175 pin makes across GUC states with the evidence held constant. The defect class is one absence + /// being read as another; two evidence shapes sharing a sentence would reproduce it. + /// + [Fact] + public void TheVisibilityNote_NamesTheRuleAndTheReader_AndSaysADifferentThingPerShape() + { + var on = On(); + + var filtered = DarlingMcpStoreMetricsTools.JobHistoryNote(on, FilteredReader()); + var contradiction = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(rows: 0, newestRow: null)); + var proven = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed()); + var nothingRan = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(rows: 0, newestRow: null, ran: 0, newestRun: null)); + var partial = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(dbOwnerMember: false, ownerMemberJobs: 3)); + var noJobs = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(jobs: 0, ownerMemberJobs: 0, rows: 0, ran: 0)); + var unreadable = DarlingMcpStoreMetricsTools.JobHistoryNote(on, DarlingStoreMetricsReader.JobHistoryEvidence.Unreadable); + + var all = new[] { filtered, contradiction, proven, nothingRan, partial, noJobs, unreadable }; + Assert.Equal(all.Length, all.Distinct(StringComparer.Ordinal).Count()); + + /* The rule, stated with the view's own predicate, and the trap named, on every one of them. */ + foreach (var note in all) + { + Assert.Contains("pg_has_role(current_user, , 'MEMBER') OR pg_has_role(current_user, , 'MEMBER')", note, StringComparison.Ordinal); + Assert.Contains("jobs and job_stats views show every role every job", note, StringComparison.Ordinal); + Assert.Contains("contradicts nothing", note, StringComparison.Ordinal); + /* And the #3175 half is still in front of it, untouched. */ + Assert.StartsWith( + DarlingMcpStoreMetricsTools.JobHistoryNote(on, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable), + note, + StringComparison.Ordinal); + } + + /* The reader is named wherever there was one. */ + Assert.Contains("read as 'mcp'", filtered, StringComparison.Ordinal); + Assert.Contains("read as 'darling'", contradiction, StringComparison.Ordinal); + + /* The filtered arm: the zero is the filter, the role that can see is named, and the unfiltered + population is reported so the store does not read as idle. */ + Assert.Contains("NOTHING by construction", filtered, StringComparison.Ordinal); + Assert.Contains("the filter, not the table", filtered, StringComparison.Ordinal); + Assert.Contains("owner role", filtered, StringComparison.Ordinal); + Assert.Contains("9 job(s) started a run", filtered, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", filtered, StringComparison.Ordinal); + + /* The contradiction arm: named as such, with the benign cause and how to settle it, and never + 'quiet' or 'clean'. */ + Assert.Contains("CONTRADICTION", contradiction, StringComparison.Ordinal); + Assert.Contains("do not read this zero as quiet", contradiction, StringComparison.Ordinal); + Assert.Contains("switched on AFTER", contradiction, StringComparison.Ordinal); + Assert.Contains("re-read after", contradiction, StringComparison.Ordinal); + Assert.Contains("a finding", contradiction, StringComparison.Ordinal); + + /* Rows seen: the flag is a measurement. */ + Assert.Contains("12 row(s)", proven, StringComparison.Ordinal); + Assert.Contains("a measurement here, not a GUC echo", proven, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", proven, StringComparison.Ordinal); + + /* Zero rows, zero runs: not clean — an absence of information, said so. */ + Assert.Contains("proves nothing either way", nothingRan, StringComparison.Ordinal); + Assert.Contains("absence of information", nothingRan, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", nothingRan, StringComparison.Ordinal); + + /* Partial: the fraction, and no verdict. */ + Assert.Contains("3 of 110 jobs", partial, StringComparison.Ordinal); + Assert.Contains("THOSE jobs only", partial, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", partial, StringComparison.Ordinal); + + /* Unreadable: the flag is a GUC echo and the note says so, instead of a zero. */ + Assert.Contains("UNKNOWN", unreadable, StringComparison.Ordinal); + Assert.Contains("GUC's word alone", unreadable, StringComparison.Ordinal); + Assert.DoesNotContain("rows_observed = ", unreadable, StringComparison.Ordinal); + + /* The GUC-off arm with an admitted reader: says what it will see once on, and that anything seen + now is failures — never that logging is secretly on. */ + var offAdmitted = DarlingMcpStoreMetricsTools.JobHistoryNote(OffByDefault(), Observed(rows: 2)); + Assert.Contains("FAILED runs", offAdmitted, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", offAdmitted, StringComparison.Ordinal); + Assert.DoesNotContain("not a GUC echo", offAdmitted, StringComparison.Ordinal); + + /* The GUC unreadable but the view readable: the rows are counted and NOT classified — neither + proof of recording nor a failure census, because that split is the setting's to make. */ + var gucUnknownAdmitted = DarlingMcpStoreMetricsTools.JobHistoryNote( + new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.Unreadable, null, null, null), + Observed(rows: 2)); + Assert.Contains("not classified", gucUnknownAdmitted, StringComparison.Ordinal); + Assert.Contains("2 row(s)", gucUnknownAdmitted, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", gucUnknownAdmitted, StringComparison.Ordinal); + Assert.DoesNotContain("not a GUC echo", gucUnknownAdmitted, StringComparison.Ordinal); + Assert.DoesNotContain("any it sees now are FAILED", gucUnknownAdmitted, StringComparison.Ordinal); + } + + /// + /// #3574: on a plain-PostgreSQL connection there is no view to be filtered, so the evidence half adds + /// NOTHING and the #3175 NotRegistered note is byte-for-byte what it was. The one arm where a sentence + /// about who may read the view would be noise. + /// + [Fact] + public void TheNotApplicableEvidence_AddsNoText_SoTheNotRegisteredNoteIsUnchanged() + { + var notRegistered = new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.NotRegistered, null, null, null); + + Assert.Equal( + "", + DarlingMcpStoreMetricsTools.JobHistoryVisibilityNote(notRegistered, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable)); + + var note = DarlingMcpStoreMetricsTools.JobHistoryNote(notRegistered, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable); + Assert.DoesNotContain("WHO CAN SEE", note, StringComparison.Ordinal); + Assert.DoesNotContain("pg_has_role", note, StringComparison.Ordinal); + Assert.Contains("does not exist on this connection", note, StringComparison.Ordinal); + } + + /// + /// #3574: the two reads fail SEPARATELY, and the evidence read is not attempted where there is no view. + /// + /// No store is needed. The data source points at a port nothing listens on, so any read that + /// reaches the network fails at connect — which is exactly the failure the isolation has to absorb. The + /// NotRegistered half goes further: it hands the read an ALREADY-CANCELLED token, and the method's own + /// contract is that cancellation is never isolated, so a NotApplicable coming back (rather than an + /// ) proves the read returned before touching the connection + /// at all. + /// + [Fact] + public async Task TheEvidenceRead_FailsSeparatelyFromTheGucRead_AndIsSkippedWhereThereIsNoView() + { + await using var nowhere = NpgsqlDataSource.Create( + "Host=127.0.0.1;Port=1;Username=nobody;Password=nobody;Database=nowhere;Timeout=1;Command Timeout=1"); + + /* A registered GUC reading, handed in from outside: the evidence read fails at connect and reports + Unreadable, and the GUC reading it was given is untouched — the count timing out cannot make the + GUC unknown. */ + var on = On(); + var evidence = await DarlingStoreMetricsReader.GetJobHistoryEvidenceAsync(nowhere, on, TestContext.Current.CancellationToken); + Assert.Equal(DarlingStoreMetricsReader.JobHistoryEvidenceStatus.Unreadable, evidence.Status); + Assert.Null(evidence.RowsObserved); + Assert.Null(evidence.ReaderRole); + Assert.True(on.Recording); + + /* And the note on that pair still carries the GUC's answer, qualified as an echo. */ + var note = DarlingMcpStoreMetricsTools.JobHistoryNote(on, evidence); + Assert.Contains("is ON", note, StringComparison.Ordinal); + Assert.Contains("GUC's word alone", note, StringComparison.Ordinal); + + /* NotRegistered: not attempted. A fired token would throw out of any read that started. */ + var notRegistered = new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.NotRegistered, null, null, null); + var skipped = await DarlingStoreMetricsReader.GetJobHistoryEvidenceAsync( + nowhere, notRegistered, new CancellationToken(canceled: true)); + Assert.Same(DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable, skipped); + + /* The GUC read's own isolation, for the pairing: it too becomes Unreadable at connect rather than + throwing or inventing an Off. */ + var guc = await DarlingStoreMetricsReader.GetJobExecutionLoggingAsync(nowhere, TestContext.Current.CancellationToken); + Assert.Equal(DarlingStoreMetricsReader.JobExecutionLoggingStatus.Unreadable, guc.Status); + } + + /// + /// #3574: the description names the ownership filter and the fields the block now carries for it, + /// asserted TOGETHER with the shipped SQL evaluating the predicate — the sibling pins' reason: a + /// sentence about a read that does not exist is advice to look somewhere this tool declines to look, + /// and a read with no sentence sits in the JSON unexplained. + /// + [Fact] + public void TheDescription_NamesTheOwnershipFilter_AndTheEvidenceFieldsBehindIt() + { + var description = ToolMethods().Single().GetCustomAttribute()?.Description; + Assert.NotNull(description); + + /* The rule, the trap, and the fact that the tool's own managed-mode reader is on the wrong side of + it — in one ordered match each, so a rewording that kept the words but dropped the claim goes + red. */ + Assert.Matches(@"job_history is ownership-filtered[^.]*members of the job's owner role or of the database owner[^.]*jobs and job_stats views show every role every job", description!); + Assert.Matches(@"managed mode[^.]*mcp role[^.]*filters out", description!); + + /* Every evidence field the block publishes is named where a caller reads about the block. */ + foreach (var field in new[] { "reader_role", "visibility", "rows_observed", "newest_row_at", "jobs_run_in_window", "contradiction" }) + { + Assert.Contains(field, description!, StringComparison.Ordinal); + } + + /* And the window the counts are over, so the number never travels without its denominator. */ + Assert.Contains($"{DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours}-hour window", description!, StringComparison.Ordinal); + + /* The read behind the sentence. */ + Assert.Contains("pg_has_role", DarlingStoreMetricsReader.JobHistoryEvidenceSql, StringComparison.Ordinal); + } + + /* ---------------- #3574, managed-mode self-proof: the owner's persisted reading ---------------- */ + + /// + /// The evidence SELECT is ONE string with two consumers: the reader's constant IS the sweep's, the + /// window constants agree, and the sweep's INSERT embeds the reader's text verbatim. Two copies of a + /// nine-column predicate would drift without erroring; this pins that there is one. + /// + [Fact] + public void TheEvidenceSql_IsSharedWithTheSweep_NotCopied() + { + Assert.Equal(StoreSelfMetrics.JobHistoryEvidenceSql, DarlingStoreMetricsReader.JobHistoryEvidenceSql); + Assert.Equal(StoreSelfMetrics.JobHistoryEvidenceWindowHours, DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours); + Assert.Contains(DarlingStoreMetricsReader.JobHistoryEvidenceSql, StoreSelfMetrics.JobHistoryInsertSql, StringComparison.Ordinal); + } + + private static readonly DateTime SweepAt = new(2026, 9, 18, 15, 0, 0, DateTimeKind.Unspecified); + + /// A job_history row as the sweep writes it: role in the name, count in row_count (null = + /// filtered), population in total_runs, window in schedule_interval_ms, newest-row AGE in + /// last_run_duration_ms. + private static DarlingStoreMetricsReader.StoreMetricRow OwnerRow( + DateTime? at = null, long? rows = 48, long? ran = 110, long? ageMs = 780_000, string role = "darling", long? windowMs = 86_400_000) => new( + StoreSelfMetrics.JobHistoryObjectKind, role, at ?? SweepAt, + null, null, null, null, rows, null, + LastRunDurationMs: ageMs, ScheduleIntervalMs: windowMs, TotalRuns: ran, TotalFailures: null); + + private static DarlingStoreMetricsReader.StoreMetricRow StoreRow(DateTime? at = null, long bytes = 1_000) => new( + StoreSelfMetrics.StoreObjectKind, "darling", at ?? SweepAt, bytes, null, null, null, null, 52); + + private static DarlingStoreMetricsReader.StoreMetricRow Row(string kind, string name, long? bytes, DateTime? at = null, int? chunks = null) => new( + kind, name, at ?? SweepAt, bytes, null, null, chunks, null, null); + + /// + /// #3574: the decoder turns the sweep's overloaded columns back into what they mean — count, population, + /// window, and the newest row as an INSTANT (metric_time minus the persisted age) — and judges + /// freshness against the sweep's stamp, not against the newest row. The four statuses each come from + /// one fact: no row, a NULL count, an old stamp, or none of those. + /// + [Fact] + public void OwnerEvidence_DecodesTheColumnMapping_AndJudgesFreshnessFromTheSweepStamp() + { + var now = new DateTime(2026, 9, 18, 15, 30, 0, DateTimeKind.Utc); + + var observed = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(new[] { StoreRow(), OwnerRow() }, now); + Assert.Equal(DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Observed, observed.Status); + Assert.Equal("darling", observed.ReaderRole); + Assert.Equal(DateTime.SpecifyKind(SweepAt, DateTimeKind.Utc), observed.ObservedAt); + Assert.Equal(DateTimeKind.Utc, observed.ObservedAt!.Value.Kind); + Assert.Equal(0.5, observed.AgeHours); + Assert.Equal(24.0, observed.WindowHours); + Assert.Equal(48L, observed.RowsObserved); + Assert.Equal(110L, observed.JobsRunInWindow); + /* The age decodes to an instant 13 minutes before the sweep — never surfaced as a duration. */ + Assert.Equal(new DateTime(2026, 9, 18, 14, 47, 0, DateTimeKind.Utc), observed.NewestRowAt); + + /* No row ever seen: the age is NULL and so is the instant — a count of zero and a missing newest + row are two facts, and the whole issue is one being read as the other. */ + var noneEver = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(new[] { OwnerRow(rows: 0, ageMs: null) }, now); + Assert.Equal(0L, noneEver.RowsObserved); + Assert.Null(noneEver.NewestRowAt); + + /* Filtered: the sweep's role could not see, so it wrote no count — and that is a status, not a zero. */ + var filtered = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(new[] { OwnerRow(rows: null, ageMs: null, role: "svc") }, now); + Assert.Equal(DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Filtered, filtered.Status); + Assert.Equal("svc", filtered.ReaderRole); + Assert.Null(filtered.RowsObserved); + Assert.Equal(110L, filtered.JobsRunInWindow); + + /* Stale: judged on the SWEEP's age, past the fresh bar. The count is still carried. */ + var stale = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest( + new[] { OwnerRow(at: SweepAt.AddHours(-(DarlingStoreMetricsReader.OwnerEvidenceFreshHours + 1))) }, now); + Assert.Equal(DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Stale, stale.Status); + Assert.Equal(48L, stale.RowsObserved); + Assert.True(stale.AgeHours > DarlingStoreMetricsReader.OwnerEvidenceFreshHours); + + /* Exactly at the bar is fresh; one second past it is not. */ + var atBar = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest( + new[] { OwnerRow(at: now.AddHours(-DarlingStoreMetricsReader.OwnerEvidenceFreshHours)) }, now); + Assert.Equal(DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Observed, atBar.Status); + + /* Absent: no row of the kind at all — every field null, never a plausible zero. */ + var absent = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(new[] { StoreRow() }, now); + Assert.Same(DarlingStoreMetricsReader.OwnerJobHistoryEvidence.Absent, absent); + Assert.Null(absent.RowsObserved); + Assert.Null(absent.ReaderRole); + + /* Two rows (a renamed owner role): the newest sweep's speaks. */ + var renamed = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest( + new[] { OwnerRow(at: SweepAt.AddHours(-2), role: "old_owner", rows: 5), OwnerRow(role: "new_owner", rows: 7) }, now); + Assert.Equal("new_owner", renamed.ReaderRole); + Assert.Equal(7L, renamed.RowsObserved); + + Assert.Equal(3, DarlingStoreMetricsReader.OwnerEvidenceFreshHours); + } + + /// + /// #3574: the owner's contradiction is the same conjunction as the connection's own, judged on a FRESH + /// reading only. Each flip off the positive control is a different non-finding: not recording, stale, + /// filtered, rows seen, nothing ran. + /// + [Fact] + public void OwnerContradiction_NeedsAFreshAdmittedZeroWithRuns_AndEachAloneIsNotOne() + { + var now = new DateTime(2026, 9, 18, 15, 30, 0, DateTimeKind.Utc); + DarlingStoreMetricsReader.OwnerJobHistoryEvidence Decode(params DarlingStoreMetricsReader.StoreMetricRow[] rows) + => DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(rows, now); + + var positive = Decode(OwnerRow(rows: 0, ageMs: null)); + Assert.True(positive.ContradictsRecording(recording: true)); + + Assert.False(positive.ContradictsRecording(recording: false)); + Assert.False(Decode(OwnerRow(rows: 0, ageMs: null, at: SweepAt.AddHours(-9))).ContradictsRecording(recording: true)); + Assert.False(Decode(OwnerRow(rows: null, ageMs: null)).ContradictsRecording(recording: true)); + Assert.False(Decode(OwnerRow(rows: 1)).ContradictsRecording(recording: true)); + Assert.False(Decode(OwnerRow(rows: 0, ageMs: null, ran: 0)).ContradictsRecording(recording: true)); + Assert.False(DarlingStoreMetricsReader.OwnerJobHistoryEvidence.Absent.ContradictsRecording(recording: true)); + } + + /// + /// #3574: the owner's half of the note is appended exactly where this connection is NOT itself an + /// admitted reader, names its source (the service's hourly sweep) on every arm, and says a different + /// thing per owner shape — including the contradiction from the owner's numbers, in so many words. On + /// the All arm and the NotApplicable arm it adds NOTHING: there the connection's own count + /// is the census, or there is no view. + /// + [Fact] + public void TheOwnerNote_IsAppendedOnlyWhereThisConnectionCannotSee_AndSaysADifferentThingPerShape() + { + var now = new DateTime(2026, 9, 18, 15, 30, 0, DateTimeKind.Utc); + DarlingStoreMetricsReader.OwnerJobHistoryEvidence Decode(params DarlingStoreMetricsReader.StoreMetricRow[] rows) + => DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(rows, now); + var on = On(); + var mcp = FilteredReader(); + + var proven = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, Decode(OwnerRow())); + var contradiction = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, Decode(OwnerRow(rows: 0, ageMs: null))); + var nothingRan = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, Decode(OwnerRow(rows: 0, ageMs: null, ran: 0))); + var stale = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, Decode(OwnerRow(rows: 0, ageMs: null, at: SweepAt.AddHours(-9)))); + var filtered = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, Decode(OwnerRow(rows: null, ageMs: null, role: "svc"))); + var absent = DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp, DarlingStoreMetricsReader.OwnerJobHistoryEvidence.Absent); + var offFailures = DarlingMcpStoreMetricsTools.JobHistoryNote(OffByDefault(), mcp, Decode(OwnerRow(rows: 2))); + + var all = new[] { proven, contradiction, nothingRan, stale, filtered, absent, offFailures }; + Assert.Equal(all.Length, all.Distinct(StringComparer.Ordinal).Count()); + + /* Every arm: the connection's own None verdict first, untouched, then the source of the owner's numbers. */ + foreach (var note in all) + { + Assert.Contains("NOTHING by construction", note, StringComparison.Ordinal); + Assert.Contains("THE OWNER'S OWN COUNT", note, StringComparison.Ordinal); + Assert.Contains("the service's sweep, not from this connection", note, StringComparison.Ordinal); + } + + Assert.Contains("Reading as 'darling' at 2026-09-18T15:00:00.0000000Z", proven, StringComparison.Ordinal); + Assert.Contains("48 row(s)", proven, StringComparison.Ordinal); + Assert.Contains("newest 2026-09-18T14:47:00.0000000Z", proven, StringComparison.Ordinal); + Assert.Contains("110 job(s) started a run", proven, StringComparison.Ordinal); + Assert.Contains("a measurement on this store after all", proven, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", proven, StringComparison.Ordinal); + + Assert.Contains("CONTRADICTION (from the owner's numbers)", contradiction, StringComparison.Ordinal); + Assert.Contains("do not read this zero as quiet", contradiction, StringComparison.Ordinal); + Assert.Contains("switched on AFTER", contradiction, StringComparison.Ordinal); + Assert.Contains("none ever", contradiction, StringComparison.Ordinal); + + Assert.Contains("absence of information", nothingRan, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", nothingRan, StringComparison.Ordinal); + + /* Stale: shown with its age, explicitly not read as current, and NO verdict even though the numbers + alone would be the contradiction's. */ + Assert.Contains("hours old", stale, StringComparison.Ordinal); + Assert.Contains("not read as current evidence", stale, StringComparison.Ordinal); + Assert.DoesNotContain("CONTRADICTION", stale, StringComparison.Ordinal); + + Assert.Contains("'svc' was itself NOT admitted", filtered, StringComparison.Ordinal); + Assert.Contains("recorded no count", filtered, StringComparison.Ordinal); + + Assert.Contains("holds no such row yet", absent, StringComparison.Ordinal); + Assert.Contains("nothing in this block is a measurement", absent, StringComparison.Ordinal); + + Assert.Contains("FAILED runs, which TimescaleDB writes regardless", offFailures, StringComparison.Ordinal); + Assert.DoesNotContain("after all", offFailures, StringComparison.Ordinal); + + /* NOT appended where this connection already sees everything, and where there is no view. */ + var ownReader = DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(), Decode(OwnerRow())); + Assert.DoesNotContain("THE OWNER'S OWN COUNT", ownReader, StringComparison.Ordinal); + Assert.Equal(DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed()), ownReader); + + var notRegistered = new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.NotRegistered, null, null, null); + Assert.DoesNotContain("THE OWNER'S OWN COUNT", + DarlingMcpStoreMetricsTools.JobHistoryNote(notRegistered, DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable, Decode(OwnerRow())), + StringComparison.Ordinal); + + /* Appended on Partial and on Unreadable too — anywhere this connection's count is not a census. */ + Assert.Contains("THE OWNER'S OWN COUNT", + DarlingMcpStoreMetricsTools.JobHistoryNote(on, Observed(dbOwnerMember: false, ownerMemberJobs: 3), Decode(OwnerRow())), + StringComparison.Ordinal); + Assert.Contains("THE OWNER'S OWN COUNT", + DarlingMcpStoreMetricsTools.JobHistoryNote(on, DarlingStoreMetricsReader.JobHistoryEvidence.Unreadable, Decode(OwnerRow())), + StringComparison.Ordinal); + + /* And the two-argument form is the three-argument form with Absent, so every older pin still + describes a note the tool can produce. */ + Assert.Equal(absent, DarlingMcpStoreMetricsTools.JobHistoryNote(on, mcp)); + } + + /// + /// #3175 corrected (#3582 follow-up): an OFF setting does not empty the view. TimescaleDB writes a + /// FAILED run's row regardless of the GUC, so both Off arms now say the view is a census of failures + /// and neither claims it "returns zero rows" — which contradicted the visibility arm beside it that + /// (correctly) classifies rows an admitted reader sees with the GUC off as failures. Pinned with a + /// positive control on the On arm, which is the one allowed to call the rows a census of runs. + /// + [Fact] + public void TheGucOffNotes_SayFailuresOnly_AndNoLongerClaimZeroRows() + { + var none = DarlingStoreMetricsReader.JobHistoryEvidence.NotApplicable; + var offDefault = DarlingMcpStoreMetricsTools.JobHistoryNote(OffByDefault(), none); + var offOverride = DarlingMcpStoreMetricsTools.JobHistoryNote( + new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.Off, "off", "configuration file", "postgresql.auto.conf"), + none); + + foreach (var note in new[] { offDefault, offOverride }) + { + Assert.Contains("recording FAILED runs only", note, StringComparison.Ordinal); + Assert.Contains("regardless of this setting", note, StringComparison.Ordinal); + Assert.Contains("census of failures", note, StringComparison.Ordinal); + Assert.Contains("secretly on", note, StringComparison.Ordinal); + Assert.DoesNotContain("returns zero rows", note, StringComparison.Ordinal); + Assert.DoesNotContain("is NOT recording", note, StringComparison.Ordinal); + } + + /* The two Off arms still differ on what to DO. */ + Assert.Contains("ALTER SYSTEM RESET", offOverride, StringComparison.Ordinal); + Assert.DoesNotContain("ALTER SYSTEM RESET", offDefault, StringComparison.Ordinal); + + /* Positive control: only the On arm calls it a census of the runs it covers. */ + var onNote = DarlingMcpStoreMetricsTools.JobHistoryNote(On(), none); + Assert.Contains("census of the runs it covers", onNote, StringComparison.Ordinal); + Assert.DoesNotContain("census of failures", onNote, StringComparison.Ordinal); + } + + /// + /// #3574 + #3582: the description names the owner fields and where their numbers come from, asserted + /// TOGETHER with the sweep arm that writes them (the sibling pins' reason). + /// + [Fact] + public void TheDescription_NamesTheOwnerEvidence_AndTheSweepThatProducesIt() + { + var description = ToolMethods().Single().GetCustomAttribute()?.Description; + Assert.NotNull(description); + + Assert.Matches(@"hourly self-metrics sweep runs as it[^.]*persists the owner's own count[^.]*same 24-hour window", description!); + foreach (var field in new[] { "owner_evidence", "owner_role", "owner_observed_at", "owner_rows_observed", "owner_newest_row_at", "owner_jobs_run_in_window" }) + { + Assert.Contains(field, description!, StringComparison.Ordinal); + } + + Assert.Contains("taken by the service's sweep rather than by this connection", description!, StringComparison.Ordinal); + Assert.Contains($"'{StoreSelfMetrics.JobHistoryObjectKind}'", StoreSelfMetrics.JobHistoryInsertSql, StringComparison.Ordinal); + + /* And the corrected GUC claim: successes need it on, failures are written regardless. */ + Assert.Matches(@"records a SUCCESSFUL run only while timescaledb\.enable_job_execution_logging is on[^.]*defaults OFF[^.]*FAILED run's row is written regardless", description!); + } + + /* ---------------- #3582: the inventory reconciled ---------------- */ + + /// + /// #3582: the reconciliation is computed over ONE sweep — the rows sharing the store row's stamp — and + /// states the two coverages separately: enumerated (named objects) and attributed (any row). Rows from + /// an older sweep are counted and excluded, not summed against a newer database figure; a missing + /// catch-all row makes the verdict unjudgeable rather than false-by-arithmetic; a residual past the bar + /// is NOT reconciled; the bar is the larger of the percent and the floor. + /// + [Fact] + public void ComputeInventory_ReconcilesOneSweep_SplitsEnumeratedFromAttributed_AndCountsStaleRows() + { + const long gib = 1L << 30; + var older = SweepAt.AddHours(-1); + + var rows = new[] + { + StoreRow(bytes: 415 * gib), + Row(StoreSelfMetrics.HypertableObjectKind, "wait_stats", 100 * gib), + Row(StoreSelfMetrics.HypertableObjectKind, "query_stats", 20 * gib), + Row(StoreSelfMetrics.ContinuousAggregateObjectKind, "query_store_stats_hourly", 235 * gib), + Row(StoreSelfMetrics.DimensionObjectKind, PayloadDimensions.QueryPlanDimTable, 39 * gib), + Row(StoreSelfMetrics.TableObjectKind, QueryStoreTextStore.TableName, 15 * gib), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 3 * gib, chunks: 14), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, 2 * gib, chunks: 161), + /* Kinds with no bytes contribute nothing and are not stale. */ + new DarlingStoreMetricsReader.StoreMetricRow(StoreSelfMetrics.BackgroundJobObjectKind, "policy_compression x [1]", SweepAt, null, null, null, null, null, null, 100, 3_600_000, 5, 0), + OwnerRow(), + /* From the previous sweep: excluded and counted. */ + Row(StoreSelfMetrics.HypertableObjectKind, "dropped_since", 7 * gib, at: older), + }; + + var inventory = DarlingStoreMetricsReader.ComputeInventory(rows); + Assert.NotNull(inventory); + Assert.Equal(SweepAt, inventory!.SweepAt); + Assert.Equal(415 * gib, inventory.DatabaseBytes); + Assert.Equal((100 + 20 + 235 + 39 + 15) * gib, inventory.EnumeratedBytes); + Assert.Equal((100 + 20 + 235 + 39 + 15 + 3 + 2) * gib, inventory.AttributedBytes); + Assert.Equal(gib, inventory.ResidualBytes); + Assert.Equal(1, inventory.StaleRowCount); + Assert.Equal(3 * gib, inventory.UnenumeratedBytes); + Assert.Equal(14, inventory.UnenumeratedRelationCount); + Assert.Equal(2 * gib, inventory.SystemBytes); + Assert.Equal(161, inventory.SystemRelationCount); + Assert.True(inventory.CatchAllPresent); + + Assert.Equal(Math.Round(100.0 * 409 / 415, 2), inventory.EnumeratedPercent); + Assert.Equal(Math.Round(100.0 * 414 / 415, 2), inventory.AttributedPercent); + Assert.Equal(120 * gib, inventory.BytesByKind[StoreSelfMetrics.HypertableObjectKind]); + Assert.Equal(235 * gib, inventory.BytesByKind[StoreSelfMetrics.ContinuousAggregateObjectKind]); + Assert.False(inventory.BytesByKind.ContainsKey(StoreSelfMetrics.BackgroundJobObjectKind)); + + /* The bar: 1% of 415 GiB is 4.15 GiB, well above the 64 MiB floor; a 1 GiB residual reconciles. */ + Assert.Equal((long)Math.Ceiling(415 * gib * 0.01), inventory.ToleranceBytes); + Assert.True(inventory.Reconciled); + + /* Ten GiB attributed to no row: a finding. */ + var gap = DarlingStoreMetricsReader.ComputeInventory(new[] + { + StoreRow(bytes: 415 * gib), + Row(StoreSelfMetrics.HypertableObjectKind, "a", 400 * gib), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 3 * gib, chunks: 1), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, 2 * gib, chunks: 1), + })!; + Assert.Equal(10 * gib, gap.ResidualBytes); + Assert.False(gap.Reconciled); + + /* The floor: on a 17 MiB store, 1% is 170 KiB and a 160 KiB residual would sit under it — but the + floor is what carries a small store, and a residual under 64 MiB reconciles regardless. */ + var small = DarlingStoreMetricsReader.ComputeInventory(new[] + { + StoreRow(bytes: 17_192_639), + Row(StoreSelfMetrics.HypertableObjectKind, "a", 1_720_320), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 827_392, chunks: 40), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, 13_352_960, chunks: 161), + })!; + Assert.Equal(DarlingStoreMetricsReader.ReconciliationToleranceFloorBytes, small.ToleranceBytes); + Assert.True(small.Reconciled); + + /* No catch-all rows: not judgeable, so not reconciled — however small the arithmetic residual. */ + var noCatchAll = DarlingStoreMetricsReader.ComputeInventory(new[] + { + StoreRow(bytes: 1_000), + Row(StoreSelfMetrics.HypertableObjectKind, "a", 1_000), + })!; + Assert.False(noCatchAll.CatchAllPresent); + Assert.False(noCatchAll.Reconciled); + Assert.Null(noCatchAll.UnenumeratedBytes); + + /* No store row: nothing to reconcile against, and no percentage of nothing. */ + Assert.Null(DarlingStoreMetricsReader.ComputeInventory(new[] { Row(StoreSelfMetrics.HypertableObjectKind, "a", 1) })); + Assert.Null(DarlingStoreMetricsReader.ComputeInventory(Array.Empty())); + + Assert.Equal(1.0, DarlingStoreMetricsReader.ReconciliationTolerancePercent); + Assert.Equal(64L * 1024 * 1024, DarlingStoreMetricsReader.ReconciliationToleranceFloorBytes); + } + + /// + /// #3582: the inventory note states coverage in the issue's words, names the largest un-enumerated + /// relations, calls a residual past the bar a FINDING, calls missing catch-all rows a sweep failure, + /// explains a low enumerated share on a store with no TimescaleDB rows, and reports aggregates holding + /// bytes with compression off. Each shape is distinct from the others. + /// + [Fact] + public void TheInventoryNote_StatesCoverage_NamesTheLargest_AndCallsAGapAFinding() + { + const long gib = 1L << 30; + var rows = new[] + { + StoreRow(bytes: 415 * gib), + Row(StoreSelfMetrics.HypertableObjectKind, "wait_stats", 120 * gib), + Row(StoreSelfMetrics.ContinuousAggregateObjectKind, "query_store_stats_hourly", 200 * gib), + Row(StoreSelfMetrics.ContinuousAggregateObjectKind, "query_store_stats_daily", 35 * gib), + Row(StoreSelfMetrics.DimensionObjectKind, PayloadDimensions.QueryPlanDimTable, 39 * gib), + Row(StoreSelfMetrics.TableObjectKind, QueryStoreTextStore.TableName, 15 * gib), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 3 * gib, chunks: 14), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, 2 * gib, chunks: 161), + }; + var states = new[] + { + new DarlingStoreMetricsReader.ContinuousAggregateState("query_store_stats_hourly", false, true, "query_store_stats", 1001, null, null), + new DarlingStoreMetricsReader.ContinuousAggregateState("query_store_stats_daily", true, true, "query_store_stats_hourly", 1002, 1003, 1004), + }; + var largest = new[] + { + new DarlingStoreMetricsReader.UnenumeratedRelation("collect.store_log_events", "r", 2 * gib), + new DarlingStoreMetricsReader.UnenumeratedRelation("collect.store_metrics", "r", gib / 2), + }; + + var inventory = DarlingStoreMetricsReader.ComputeInventory(rows)!; + var reconciled = DarlingMcpStoreMetricsTools.InventoryNote(inventory, rows, states, largest); + + Assert.Contains("account for 409.0 GiB of the 415.0 GiB database (98.55%)", reconciled, StringComparison.Ordinal); + Assert.Contains("3.0 GiB sits in 14 un-enumerated user-schema relation(s) (object_kind other)", reconciled, StringComparison.Ordinal); + Assert.Contains("the largest being collect.store_log_events (2.0 GiB), collect.store_metrics (512.0 MiB)", reconciled, StringComparison.Ordinal); + Assert.Contains("2.0 GiB is PostgreSQL catalog and TimescaleDB bookkeeping in 161 relation(s)", reconciled, StringComparison.Ordinal); + Assert.Contains("RECONCILED: every row together accounts for 99.76%", reconciled, StringComparison.Ordinal); + Assert.Contains("1 of 2 continuous aggregate(s) have compression DISABLED and hold 200.0 GiB", reconciled, StringComparison.Ordinal); + Assert.DoesNotContain("NOT RECONCILED", reconciled, StringComparison.Ordinal); + Assert.DoesNotContain("OLDER sweep", reconciled, StringComparison.Ordinal); + Assert.DoesNotContain("plain-PostgreSQL", reconciled, StringComparison.Ordinal); + + /* The gap: a finding, with the direction stated. */ + var gapRows = new[] + { + StoreRow(bytes: 415 * gib), + Row(StoreSelfMetrics.HypertableObjectKind, "a", 400 * gib), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 3 * gib, chunks: 1), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, 2 * gib, chunks: 1), + }; + var gap = DarlingMcpStoreMetricsTools.InventoryNote(DarlingStoreMetricsReader.ComputeInventory(gapRows)!, gapRows, states, largest); + Assert.Contains("NOT RECONCILED — a finding: 10.0 GiB of pg_database_size is attributed to NO row", gap, StringComparison.Ordinal); + Assert.DoesNotContain("RECONCILED: every", gap, StringComparison.Ordinal); + + /* Missing catch-all rows: a sweep failure, said so, and no coverage verdict dressed up as arithmetic. */ + var partialRows = new[] { StoreRow(bytes: 415 * gib), Row(StoreSelfMetrics.HypertableObjectKind, "a", 400 * gib) }; + var partial = DarlingMcpStoreMetricsTools.InventoryNote(DarlingStoreMetricsReader.ComputeInventory(partialRows)!, partialRows, null, null); + Assert.Contains("'other' catch-all row is MISSING", partial, StringComparison.Ordinal); + Assert.Contains("'system' catch-all row is MISSING", partial, StringComparison.Ordinal); + Assert.Contains("NOT RECONCILED: without both catch-all rows", partial, StringComparison.Ordinal); + Assert.Contains("live read of each aggregate's compression and policy state did not complete", partial, StringComparison.Ordinal); + + /* Stale rows and the live census failing are each named. */ + var staleRows = rows.Append(Row(StoreSelfMetrics.HypertableObjectKind, "old", gib, at: SweepAt.AddHours(-1))).ToArray(); + var stale = DarlingMcpStoreMetricsTools.InventoryNote(DarlingStoreMetricsReader.ComputeInventory(staleRows)!, staleRows, states, null); + Assert.Contains("1 object row(s) in objects[] are from an OLDER sweep", stale, StringComparison.Ordinal); + Assert.Contains("live census naming them did not complete", stale, StringComparison.Ordinal); + + /* No TimescaleDB rows at all: the low share is explained, not flagged. */ + var plainRows = new[] + { + StoreRow(bytes: 10 * gib), + Row(StoreSelfMetrics.DimensionObjectKind, PayloadDimensions.QueryPlanDimTable, gib), + Row(StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.OtherObjectName, 8 * gib, chunks: 70), + Row(StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.SystemObjectName, gib, chunks: 60), + }; + var plain = DarlingMcpStoreMetricsTools.InventoryNote(DarlingStoreMetricsReader.ComputeInventory(plainRows)!, plainRows, Array.Empty(), Array.Empty()); + Assert.Contains("a plain-PostgreSQL store, or TimescaleDB unavailable to the sweep", plain, StringComparison.Ordinal); + Assert.Contains("not a fault", plain, StringComparison.Ordinal); + Assert.DoesNotContain("the largest being", plain, StringComparison.Ordinal); + + Assert.Equal(5, new[] { reconciled, gap, partial, stale, plain }.Distinct(StringComparer.Ordinal).Count()); + } + + /// The byte formatter picks the unit that gives a whole-number part, so a registry table is + /// not "0.0 GiB" beside a 235 GiB aggregate family. + [Fact] + public void TheByteFormatter_PicksTheUnitWithAWholeNumberPart() + { + Assert.Equal("235.0 GiB", DarlingMcpStoreMetricsTools.Gib(235L << 30)); + Assert.Equal("1.5 GiB", DarlingMcpStoreMetricsTools.Gib(3L << 29)); + Assert.Equal("512.0 MiB", DarlingMcpStoreMetricsTools.Gib(1L << 29)); + Assert.Equal("72.0 KiB", DarlingMcpStoreMetricsTools.Gib(73_728)); + Assert.Equal("161 bytes", DarlingMcpStoreMetricsTools.Gib(161)); + Assert.Equal("0 bytes", DarlingMcpStoreMetricsTools.Gib(0)); + } + + /// + /// #3582: the aggregate-state read takes its three policy facts from the jobs view by the + /// aggregate's VIEW name (the view reports a policy on an aggregate under user_view_name), hedges + /// the compression proc's 2.18+ rebrand the way the rest of the codebase does, and resolves a + /// hierarchical aggregate's source back to its parent's view name. The live top-N composes the SAME + /// census fragments the sweep sums with, so the list and the number cannot disagree. + /// + [Fact] + public void TheAggregateStateAndTopNReads_UseTheCatalogTheWayTheSweepDoes() + { + var state = DarlingStoreMetricsReader.ContinuousAggregateStateSql; + Assert.Contains("FROM timescaledb_information.continuous_aggregates ca", state, StringComparison.Ordinal); + Assert.Contains("ca.compression_enabled", state, StringComparison.Ordinal); + Assert.Contains("j.proc_name = 'policy_refresh_continuous_aggregate'", state, StringComparison.Ordinal); + Assert.Contains("(j.proc_name LIKE '%compression%' OR j.proc_name LIKE '%columnstore%')", state, StringComparison.Ordinal); + Assert.Contains("j.proc_name = 'policy_retention'", state, StringComparison.Ordinal); + Assert.Equal(3, System.Text.RegularExpressions.Regex.Matches(state, @"j\.hypertable_schema = ca\.view_schema AND j\.hypertable_name = ca\.view_name").Count); + Assert.Contains("coalesce(parent.view_name, ca.hypertable_name) AS source_name", state, StringComparison.Ordinal); + Assert.Contains("parent.materialization_hypertable_name = ca.hypertable_name", state, StringComparison.Ordinal); + + foreach (var (sql, timescale) in new[] { (DarlingStoreMetricsReader.LargestUnenumeratedSql, true), (DarlingStoreMetricsReader.LargestUnenumeratedPlainSql, false) }) + { + Assert.Contains(StoreSelfMetrics.CensusRelationPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains("NOT " + StoreSelfMetrics.SystemSchemaPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains("NOT " + StoreSelfMetrics.NamedRelationPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains("n.nspname || '.' || c.relname AS relation", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY pg_total_relation_size(c.oid) DESC", sql, StringComparison.Ordinal); + Assert.Contains("LIMIT $1", sql, StringComparison.Ordinal); + Assert.Equal(timescale, sql.Contains(StoreSelfMetrics.TimescaleInventoriedPredicateSql, StringComparison.Ordinal)); + } + + Assert.Equal(10, DarlingStoreMetricsReader.LargestUnenumeratedLimit); + } + + /// + /// #3582: the two live reads fail to NULL, not to an empty list — an empty list reads as "no aggregates" + /// / "nothing un-enumerated" and would drop a section without a word — and the aggregate read is not + /// attempted where the GUC probe said the TimescaleDB catalogs do not exist. The port-1 data source + /// and the pre-cancelled token are the sibling test's devices. + /// + [Fact] + public async Task TheLiveInventoryReads_FailToNull_AndSkipTheCatalogsThatDoNotExist() + { + await using var nowhere = NpgsqlDataSource.Create( + "Host=127.0.0.1;Port=1;Username=nobody;Password=nobody;Database=nowhere;Timeout=1;Command Timeout=1"); + var ct = TestContext.Current.CancellationToken; + + Assert.Null(await DarlingStoreMetricsReader.GetContinuousAggregateStatesAsync(nowhere, On(), ct)); + Assert.Null(await DarlingStoreMetricsReader.GetLargestUnenumeratedAsync(nowhere, On(), ct)); + + var notRegistered = new DarlingStoreMetricsReader.JobExecutionLoggingReading( + DarlingStoreMetricsReader.JobExecutionLoggingStatus.NotRegistered, null, null, null); + var skipped = await DarlingStoreMetricsReader.GetContinuousAggregateStatesAsync(nowhere, notRegistered, new CancellationToken(canceled: true)); + Assert.NotNull(skipped); + Assert.Empty(skipped!); + + /* The top-N is attempted on a plain store (its plain variant), so it fails to null there too. */ + Assert.Null(await DarlingStoreMetricsReader.GetLargestUnenumeratedAsync(nowhere, notRegistered, ct)); + } + + /// + /// #3582: the description names every new kind, the coverage fields and the reconciliation bar, + /// asserted TOGETHER with the sweep arms that write the kinds and the constants that set the bar. + /// + [Fact] + public void TheDescription_StatesTheInventorysCoverage_AndNamesTheNewKinds() + { + var description = ToolMethods().Single().GetCustomAttribute()?.Description; + Assert.NotNull(description); + + foreach (var kind in new[] { StoreSelfMetrics.ContinuousAggregateObjectKind, StoreSelfMetrics.TableObjectKind, StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.SystemObjectKind }) + { + Assert.Contains($"object_kind {kind}", description!, StringComparison.Ordinal); + } + + Assert.Matches(@"RECONCILES the newest sweep against its own pg_database_size", description!); + foreach (var field in new[] { "enumerated_percent", "attributed_percent", "residual_bytes", "reconciled", "bytes_by_kind", "largest_unenumerated", "compression_enabled" }) + { + Assert.Contains(field, description!, StringComparison.Ordinal); + } + + /* The bar, in the description's words, agrees with the constants. */ + Assert.Contains("the larger of 1% and 64 MiB", description!, StringComparison.Ordinal); + Assert.Equal(1.0, DarlingStoreMetricsReader.ReconciliationTolerancePercent); + Assert.Equal(64L << 20, DarlingStoreMetricsReader.ReconciliationToleranceFloorBytes); + + /* The named tables, by the names the sweep writes. */ + Assert.Contains(QueryStoreTextStore.TableName, description!, StringComparison.Ordinal); + Assert.Contains(StoreSelfMetrics.AlertLogTable, description!, StringComparison.Ordinal); + + /* And the mechanism claim is the one the rig verified: the hypertables view never lists a + materialization — not "lists it under an internal name". */ + Assert.Contains("timescaledb_information.hypertables never lists a materialization", description!, StringComparison.Ordinal); + } + + private static string ReaderSourcePath([CallerFilePath] string thisFile = "") + => Path.GetFullPath(Path.Combine( + Path.GetDirectoryName(thisFile)!, "..", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingStoreMetricsReader.cs")); + [Fact] public void GetStoreMetrics_IsInTheServerInstructions() { @@ -338,9 +1286,10 @@ public void ComputeDailyGrowth_EmptyAndSingleDay_YieldNothing() } /// -/// The job_history precondition probe against a LIVE server (#3175). Two things no text assertion -/// can reach: that the shipped SQL parses and binds its one positional parameter, and that the GUC name the -/// product writes into postgresql.conf is a name PostgreSQL actually knows. +/// The job_history precondition probe against a LIVE server (#3175), and the evidence probe behind it +/// (#3574). Things no text assertion can reach: that the shipped SQL parses and binds its positional +/// parameter, that the GUC name the product writes into postgresql.conf is a name PostgreSQL actually knows, +/// and that the view's ownership predicate evaluates for the connection the way the record derives it. /// /// A PAIRED control, because a zero-row result is the whole subject. The reader maps "no /// pg_settings row" to NotRegistered, so a probe that only ever saw zero rows — because the name was @@ -354,10 +1303,11 @@ public void ComputeDailyGrowth_EmptyAndSingleDay_YieldNothing() /// change look like a defect. What is pinned is that the reading is INTERNALLY CONSISTENT — a registered /// state, a value PostgreSQL renders for a bool, and Recording true for exactly on. /// -/* #1776 own-store: this class reads only pg_settings — a server-scoped catalog view, no store tables, no - DDL, no rows written — but it takes [Collection("live-postgres")] anyway because it shares the cluster - whose GUCs it reads with every other class that has it, and a class reading the shared store must either - carry the attribute or record why not. */ +/* #1776 own-store: this class reads only pg_settings and TimescaleDB's own information views (#3574) — + server- and catalog-scoped, no store tables, no DDL, no rows written — but it takes + [Collection("live-postgres")] anyway because it shares the cluster whose GUCs and jobs it reads with every + other class that has it, and a class reading the shared store must either carry the attribute or record + why not. */ [Collection("live-postgres")] public sealed class DarlingStoreMetricsJobLoggingLivePostgresTests { @@ -390,4 +1340,108 @@ name resolves — not which way the setting happens to be pointing on this clust await using var reader = await command.ExecuteReaderAsync(ct); Assert.False(await reader.ReadAsync(ct)); } + + /// + /// #3574: the evidence read against a live TimescaleDB — the shipped SQL parses, binds its one + /// timestamptz parameter, evaluates the view's predicate for the connection, and its fields are + /// INTERNALLY CONSISTENT with each other and with direct reads of the same views. + /// + /// Inserts nothing and pins no count. How many jobs exist, whether any ran in the last day + /// and whether the GUC is on are properties of whatever cluster DARLING_TEST_PG points at. What + /// is pinned is the null-versus-zero contract: RowsObserved is a COUNT and never null once the + /// read completes (a zero is a zero), while NewestRowAt is null exactly when the view showed this + /// connection no row at all — the two are different facts, and the whole issue is one being read as the + /// other. The all-time total is read DIRECTLY first, then the shipped read; rows are only ever added + /// between two reads (the history retention job runs monthly), so a direct total above zero must be + /// matched by a non-null newest row, and a direct total of zero by a null one and a zero count. + /// + /// The predicate is cross-checked, not trusted. The database-owner test is re-evaluated + /// directly on the same connection and must agree with the record; when it holds and jobs exist, the + /// derived Visibility must be All and the visible count the job count — which is the + /// managed-mode OWNER's reading, and the shape under which a zero would be a real contradiction. + /// + [Fact] + public async Task JobHistoryEvidenceProbe_EvaluatesThePredicateForThisConnection_AndItsFieldsAgree() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live job-history evidence probe test."); + + var ct = TestContext.Current.CancellationToken; + await using var postgres = NpgsqlDataSource.Create(cs!); + + /* Direct facts first, through the same views the shipped read uses. */ + long directTotal; + bool directDbOwnerMember; + string directRole; + await using (var direct = postgres.CreateCommand( + "SELECT (SELECT count(*) FROM timescaledb_information.job_history), " + + "pg_has_role(current_user, (SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()), 'MEMBER') IS TRUE, " + + "current_user::text")) + await using (var directReader = await direct.ExecuteReaderAsync(ct)) + { + Assert.True(await directReader.ReadAsync(ct)); + directTotal = directReader.GetInt64(0); + directDbOwnerMember = directReader.GetBoolean(1); + directRole = directReader.GetString(2); + } + + var logging = await DarlingStoreMetricsReader.GetJobExecutionLoggingAsync(postgres, ct); + Assert.NotEqual(DarlingStoreMetricsReader.JobExecutionLoggingStatus.NotRegistered, logging.Status); + + var evidence = await DarlingStoreMetricsReader.GetJobHistoryEvidenceAsync(postgres, logging, ct); + + Assert.Equal(DarlingStoreMetricsReader.JobHistoryEvidenceStatus.Observed, evidence.Status); + Assert.Equal(directRole, evidence.ReaderRole); + Assert.Equal(directDbOwnerMember, evidence.ReaderIsDatabaseOwnerMember); + + /* Counts, never nulls, once observed. */ + Assert.NotNull(evidence.JobCount); + Assert.NotNull(evidence.OwnerMemberJobCount); + Assert.NotNull(evidence.RowsObserved); + Assert.NotNull(evidence.JobsRunInWindow); + Assert.InRange(evidence.OwnerMemberJobCount!.Value, 0, evidence.JobCount!.Value); + Assert.InRange(evidence.HistoryVisibleJobCount!.Value, 0, evidence.JobCount.Value); + + /* The null-versus-zero contract. */ + if (directTotal == 0) + { + Assert.Null(evidence.NewestRowAt); + Assert.Equal(0L, evidence.RowsObserved); + } + else + { + Assert.NotNull(evidence.NewestRowAt); + Assert.Equal(DateTimeKind.Utc, evidence.NewestRowAt!.Value.Kind); + } + + /* A row inside the window implies a newest row; a run inside the window implies a newest run. */ + if (evidence.RowsObserved > 0) + { + Assert.NotNull(evidence.NewestRowAt); + } + + if (evidence.JobsRunInWindow > 0) + { + Assert.NotNull(evidence.NewestRunStartedAt); + Assert.Equal(DateTimeKind.Utc, evidence.NewestRunStartedAt!.Value.Kind); + } + + /* The predicate's verdict, derived as the view combines its two tests. */ + if (evidence.JobCount > 0 && directDbOwnerMember) + { + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.All, evidence.Visibility); + Assert.Equal(evidence.JobCount, evidence.HistoryVisibleJobCount); + } + + if (evidence.JobCount == 0) + { + Assert.Equal(DarlingStoreMetricsReader.JobHistoryVisibility.Unknown, evidence.Visibility); + } + + /* And the note built on a live reading names the role that read. */ + Assert.Contains( + $"read as '{directRole}'", + DarlingMcpStoreMetricsTools.JobHistoryNote(logging, evidence), + StringComparison.Ordinal); + } } diff --git a/Darling/Darling.Tests/DarlingMcpToolsTests.cs b/Darling/Darling.Tests/DarlingMcpToolsTests.cs index dd274a4c1..e77921dcb 100644 --- a/Darling/Darling.Tests/DarlingMcpToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpToolsTests.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.ComponentModel; using System.IO; using System.Linq; using System.Reflection; @@ -42,7 +43,8 @@ namespace Darling.Tests; /// directly (not over the wire) — get_analysis_findings round-trips the finding through Lite's /// envelope, the empty server returns the #1224 "empty" Status envelope, partial-name and /// unknown-name resolution behave, mute_analysis_finding writes the mute row and returns the -/// muted envelope, and the mute registry then filters the same story from a subsequent +/// muted envelope (with #3541 A14's registered / matched_now disclosure, and the muted_unmatched +/// status for a hash no stored finding carries), and the mute registry then filters the same story from a subsequent /// analysis-run save (the exact mechanism analyze_server runs through). /// [Collection("live-postgres")] @@ -85,6 +87,64 @@ tool returning the serialized-string envelope both apps' tools return. */ Assert.All(toolMethods, m => Assert.Equal(typeof(Task), m.ReturnType)); } + /// + /// #3541 A14: the mute verb's description promises the disclosure the payload now carries — on Darling in + /// the same words as Lite (its twin pin is McpMuteReportsWhatItMatchedTests), and the instruction + /// table row agrees. The live round-trip below is what proves the numbers; this is what a caller reads + /// before deciding to trust them. + /// + [Fact] + public void MuteAnalysisFinding_Description_NamesRegistered_MatchedNow_AndTheUnmatchedStatus() + { + var method = typeof(DarlingMcpTools).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == "mute_analysis_finding"); + var description = method.GetCustomAttribute()!.Description; + + foreach (var token in new[] { "registered", "matched_now", "\"muted_unmatched\"", "mistyped hash", "\"error\"" }) + { + Assert.Contains(token, description, StringComparison.Ordinal); + } + + var row = DarlingMcpInstructions.Text.Split('\n').Single(l => l.Contains("| `mute_analysis_finding` |", StringComparison.Ordinal)); + Assert.Contains("`matched_now`", row, StringComparison.Ordinal); + Assert.Contains("`muted_unmatched`", row, StringComparison.Ordinal); + } + + /// + /// #3538 A3: compare_analysis's description promises the verdict shape the payload now carries — + /// value-banded rows with band_source / delta_sigma, the rules in band_rules, + /// physical-cause families, plan_cache_churn, coverage_caveat — and says what "worse" does + /// NOT mean (one window against one window is not an experiment). Same words as Lite (its twin pin is + /// CompareAnalysisDispersionTests; the shared sentences are in McpMissMessageParityPinTests), and + /// the instruction table row agrees. The banding arithmetic is pinned on the shared + /// ComparisonBanding in Lite.Tests; this is what a caller reads before trusting a verdict. + /// + [Fact] + public void CompareAnalysis_Description_SaysWhatWorseMeans_AndWhatItDoesNot() + { + var method = typeof(DarlingMcpTools).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == "compare_analysis"); + var description = method.GetCustomAttribute()!.Description; + + foreach (var token in new[] { "delta_sigma", "band_source", "band_rules", "families", "plan_cache_churn", "coverage_caveat", "N=1 vs N=1", "cannot show that a change CAUSED anything" }) + { + Assert.Contains(token, description, StringComparison.Ordinal); + } + + var row = DarlingMcpInstructions.Text.Split('\n').Single(l => l.Contains("| `compare_analysis` |", StringComparison.Ordinal)); + foreach (var token in new[] { "`band_source`", "`families`", "`plan_cache_churn`", "N=1 vs N=1" }) + { + Assert.Contains(token, row, StringComparison.Ordinal); + } + + /* The tool's ComparePeriodsAsync seam returns the dispersion the banding needs — a 5-tuple whose + last item is the per-metric BaselineBucket map. Pinned so a twin that forgot the item would fail + here rather than silently band everything by the absolute rule. */ + var compare = typeof(DarlingAnalysisService).GetMethod(nameof(DarlingAnalysisService.ComparePeriodsAsync))!; + var tuple = compare.ReturnType.GetGenericArguments()[0]; + Assert.Contains(typeof(IReadOnlyDictionary), tuple.GetGenericArguments()); + } + /* ---------------- ungated: config + hosting pins ---------------- */ [Fact] @@ -325,7 +385,7 @@ nowhere near the window-covering cap carries a present-but-null truncation_note. including the #2000 occurrence stats. */ foreach (var field in new[] { - "finding_id", "analysis_time", "severity", "confidence", "category", + "finding_id", "analysis_time", "severity", "confidence", "confidence_basis", "category", "root_fact", "leaf_fact", "story_path", "story_path_hash", "fact_count", "incident_id", "occurrences", "first_seen", "last_seen", "peak_severity", "co_fired", "time_range", "advice", "remediation_command", "structured_remediation" @@ -343,6 +403,9 @@ including the #2000 occurrence stats. */ Assert.Equal(TestStoryHash, finding.GetProperty("story_path_hash").GetString()); Assert.Equal(2.5, finding.GetProperty("severity").GetDouble()); Assert.Equal(0.9, finding.GetProperty("confidence").GetDouble()); + /* #3538 A6: 0.9 on a two-node path is not the legacy (n-1)/n = 0.5, so the basis reads as + corroboration-derived; the legacy label is pinned on Lite's twin with a real 1.0/1 row. */ + Assert.StartsWith("corroboration (#3538)", finding.GetProperty("confidence_basis").GetString(), StringComparison.Ordinal); Assert.Equal("cpu", finding.GetProperty("category").GetString()); Assert.Equal("an4-incident-1", finding.GetProperty("incident_id").GetString()); Assert.Equal("SOS_SCHEDULER_YIELD", finding.GetProperty("root_fact").GetProperty("key").GetString()); @@ -493,7 +556,10 @@ the shared "empty" Status envelope. */ Assert.Contains(TestServerName, unknown, StringComparison.Ordinal); /* ---- mute via the tool: the muted envelope comes back and the row lands in - analysis_muted under the resolved server id. */ + analysis_muted under the resolved server id. #3541 A14: the envelope now says what the + write DID — registered, and matched_now counted in the mute's scope. TestStoryHash sits on + exactly two persisted rows for this server (the planted chain and its second cycle), so + the number is a fact of the rows above, not of the analyser. */ var muteJson = await DarlingMcpTools.MuteAnalysisFinding( analysisService, postgres, TestStoryHash, TestServerName, "an4 e2e mute"); @@ -503,6 +569,30 @@ analysis_muted under the resolved server id. */ Assert.Equal(TestStoryHash, doc.RootElement.GetProperty("story_path_hash").GetString()); Assert.Equal(TestServerName, doc.RootElement.GetProperty("server").GetString()); Assert.Equal("an4 e2e mute", doc.RootElement.GetProperty("reason").GetString()); + Assert.True(doc.RootElement.GetProperty("registered").GetBoolean()); + Assert.Equal(2, doc.RootElement.GetProperty("matched_now").GetInt64()); + } + + /* ---- #3541 A14, THE case: a hash no stored finding carries. Registered (pattern registry — it + bites if the pattern ever appears) but reported as muted_unmatched with matched_now 0, + where the old envelope said "muted" and nothing else. */ + var unmatchedJson = await DarlingMcpTools.MuteAnalysisFinding( + analysisService, postgres, "an4-mcp-e2e-never-seen-hash", TestServerName, "an4 e2e unmatched mute"); + + using (var doc = JsonDocument.Parse(unmatchedJson)) + { + Assert.Equal("muted_unmatched", doc.RootElement.GetProperty("status").GetString()); + Assert.True(doc.RootElement.GetProperty("registered").GetBoolean()); + Assert.Equal(0, doc.RootElement.GetProperty("matched_now").GetInt64()); + Assert.Contains("no stored finding", doc.RootElement.GetProperty("note").GetString(), StringComparison.Ordinal); + } + + using (var unmatchedCount = new NpgsqlCommand( + "SELECT COUNT(*) FROM analysis_muted WHERE server_id = $1 AND story_path_hash = $2", connection)) + { + unmatchedCount.Parameters.AddWithValue(TestServerId); + unmatchedCount.Parameters.AddWithValue("an4-mcp-e2e-never-seen-hash"); + Assert.Equal(1L, await unmatchedCount.ExecuteScalarAsync(ct)); } using (var muteCount = new NpgsqlCommand( @@ -521,8 +611,12 @@ that used to make the all-servers mute match no real server. */ using (var doc = JsonDocument.Parse(allServersJson)) { - Assert.Equal("muted", doc.RootElement.GetProperty("status").GetString()); + /* No persisted row carries AllServersStoryHash anywhere in the store, so fleet-wide it is + unmatched too — the scope of the count follows the scope of the mute. */ + Assert.Equal("muted_unmatched", doc.RootElement.GetProperty("status").GetString()); Assert.Equal("(all servers)", doc.RootElement.GetProperty("server").GetString()); + Assert.True(doc.RootElement.GetProperty("registered").GetBoolean()); + Assert.Equal(0, doc.RootElement.GetProperty("matched_now").GetInt64()); } using (var nullCount = new NpgsqlCommand( diff --git a/Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs b/Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs index 1a4a6c46a..dbfbcdfaa 100644 --- a/Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpTrendToolsTests.cs @@ -20,6 +20,7 @@ using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; using Xunit; namespace Darling.Tests; @@ -101,6 +102,21 @@ public void ParamContract_ServerNameOptional_RequiredKeysAreNot() Assert.False(McpParams("get_query_trend").Single(x => x.Name == "database_name").Optional); } + /// #3529's description half, superseded by the #3548 join: the tool now DELIVERS granted + /// memory (joined per point from the grants series), so the description may promise it again — but it + /// must name the null gap rather than promising an always-filled field, and still point at + /// get_memory_grants as the series' own tool. + [Fact] + public void MemoryTrend_Description_PromisesTheJoinedGrantSeries_AndNamesTheNullGap() + { + var method = ToolMethods().Single(m => m.GetCustomAttribute()!.Name == "get_memory_trend"); + var description = method.GetCustomAttribute()!.Description; + + Assert.Contains("granted memory joined per point", description, StringComparison.Ordinal); + Assert.Contains("total_granted_mb is null", description, StringComparison.Ordinal); + Assert.Contains("get_memory_grants", description, StringComparison.Ordinal); + } + [Fact] public void MemoryTrendSql_WindowedBothSides_CastsNumericToDouble() { @@ -113,6 +129,24 @@ public void MemoryTrendSql_WindowedBothSides_CastsNumericToDouble() Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); } + /// + /// #3548: the grants-series read the get_memory_trend join rides on — byte-identical to the viewer's + /// proven overlay read (the reader's doctrine), so the MCP payload and the Memory Overview overlay can + /// never disagree about what the grants series says. + /// + [Fact] + public void MemoryGrantTrendSql_IsTheViewersOverlayRead_ByteForByte() + { + Assert.Equal(ViewerDataService.MemoryGrantTrendSql, DarlingTrendReader.MemoryGrantTrendSql); + + var sql = DarlingTrendReader.MemoryGrantTrendSql; + Assert.Contains("FROM v_memory_grant_stats", sql, StringComparison.Ordinal); + Assert.Contains("CAST(SUM(granted_memory_mb) AS double precision)", sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY collection_time", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); + } + [Fact] public void PerfmonTrendSql_SingleCounter_SumsInstances_CastsBigint() { @@ -170,6 +204,45 @@ public void ProcedureDurationTrendSql_SameRate_OverProcedureStats() Assert.Contains("executions_per_second", sql, StringComparison.Ordinal); } + /// + /// #3540 (V128): the procedure trend reads the collection's STORED interval — MAX over the collection's + /// rows, 0 → NULL through NULLIF so a restart's marker collection has no rate rather than plotting 0.00 — + /// and falls back to the LAG derivation only for a pre-V128 collection. No ELSE 0. Byte-identical to the + /// viewer's copy apart from the database filter, as the pair always were. The C# half: since #3541 A12 + /// the MCP reader KEEPS the NULL-rate row as an unrated point (QueryDurationTrendPoint.HasRate + /// false) rather than dropping it — a lone collection must not become an empty series the empty ladder + /// mislabels as quiet, and effective_start must be the first collection the store held. The viewer's + /// chart reader is the one that drops, because a chart has nowhere to draw "unknown". + /// + [Fact] + public void ProcedureDurationTrendSql_PrefersTheStoredInterval_NeverFabricatesZero_AndMirrorsTheViewer() + { + var sql = DarlingTrendReader.ProcedureDurationTrendSql; + Assert.Contains("CASE WHEN MAX(sample_interval_seconds) IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(MAX(sample_interval_seconds), 0)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", sql, StringComparison.Ordinal); + Assert.Contains("CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds END AS elapsed_ms_per_second", sql, StringComparison.Ordinal); + + /* The viewer's copy minus its database-filter line is this string, whitespace aside. */ + var viewer = string.Join('\n', ViewerDataService.ProcedureDurationTrendSql + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Split('\n') + .Where(l => !l.Contains("$4::text[]", StringComparison.Ordinal)) + .Select(l => l.Trim())); + var mcp = string.Join('\n', sql.Replace("\r\n", "\n", StringComparison.Ordinal).Split('\n').Select(l => l.Trim())); + Assert.Equal(viewer, mcp); + + /* And the shared reader KEEPS a NULL-rate row as an unrated point rather than reading it as 0 or + dropping it — the C# half of the idiom (#3541 A12). */ + var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingTrendReader.cs"); + var reader = source[source.IndexOf("private static async Task> ReadDurationPointsAsync(", StringComparison.Ordinal)..]; + reader = reader[..reader.IndexOf("return items;", StringComparison.Ordinal)]; + Assert.Contains("reader.IsDBNull(1) ? null", reader, StringComparison.Ordinal); + Assert.DoesNotContain("continue;", reader, StringComparison.Ordinal); + Assert.DoesNotContain("reader.IsDBNull(1) ? 0", reader, StringComparison.Ordinal); + Assert.Equal(typeof(double?), typeof(DarlingTrendReader.QueryDurationTrendPoint).GetProperty("Value")!.PropertyType); + } + /// /// #2484: the Query Store trend carries the #1841 tier-2 interval placement, copied from the viewer's /// read rather than rewritten. Both arms are pinned because losing either one changes the numbers: drop @@ -199,6 +272,95 @@ public void QueryStoreDurationTrendSql_KeepsBothIntervalArms_AndPlacesWorkWhenIt Assert.Contains("executions_per_second", sql, StringComparison.Ordinal); } + /// + /// #3541 A2: the hourly-tier twins the two plan-cache trends fall to past the raw horizon read the + /// ROLLUP and bucket by it — asserted on the shipped SQL so a later edit that quietly repoints either at + /// its raw table (which would reintroduce the four-days-labelled-seven defect while every other test + /// still passed) has to argue with this. The view name is bound to the + /// constant rather than restated, so a rollup rename cannot leave the read naming a relation that no + /// longer exists. + /// + [Theory] + [InlineData(nameof(DarlingTrendReader.QueryDurationTrendHourlySql), TimescaleSupport.QueryStatsHourlyView, "query_stats")] + [InlineData(nameof(DarlingTrendReader.ProcedureDurationTrendHourlySql), TimescaleSupport.ProcedureStatsHourlyView, "procedure_stats")] + public void DurationTrendHourlySql_ReadsTheRollup_BucketsByIt_ProjectsTheSharedShape(string sqlName, string view, string rawTable) + { + var sql = SqlByName(sqlName); + + Assert.Contains("FROM " + view, sql, StringComparison.Ordinal); + Assert.DoesNotContain("FROM " + rawTable + "\n", sql, StringComparison.Ordinal); + Assert.DoesNotContain("collection_time >=", sql, StringComparison.Ordinal); + Assert.Contains("bucket >= $2", sql, StringComparison.Ordinal); + Assert.Contains("bucket <= $3", sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY bucket", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY bucket", sql, StringComparison.Ordinal); + + /* Same three columns, same aliases, as the raw read — one mapper serves both tiers. */ + Assert.Contains("bucket AS collection_time", sql, StringComparison.Ordinal); + Assert.Contains("AS elapsed_ms_per_second", sql, StringComparison.Ordinal); + Assert.Contains("AS executions_per_second", sql, StringComparison.Ordinal); + + /* The rollup's own summed columns, which the CAGG definition must still carry under these names. */ + Assert.Contains("SUM(elapsed_time_sum)", sql, StringComparison.Ordinal); + Assert.Contains("SUM(execution_count_sum)", sql, StringComparison.Ordinal); + var createSql = view == TimescaleSupport.QueryStatsHourlyView + ? TimescaleSupport.CreateQueryStatsHourlySql + : TimescaleSupport.CreateProcedureStatsHourlySql; + Assert.Contains("AS elapsed_time_sum", createSql, StringComparison.Ordinal); + Assert.Contains("AS execution_count_sum", createSql, StringComparison.Ordinal); + Assert.Contains("AS bucket", createSql, StringComparison.Ordinal); + } + + /// + /// The hourly tier divides by the bucket WIDTH, never by a LAG over neighbouring points (the measurement + /// lane's A11a, closed where the routing rewrite made it free). Two things ride on this: every bucket + /// has a real denominator, so the raw idiom's fabricated first-point zero does not exist on this tier; + /// and the literal the SQL divides by is pinned to the rollup's declared bucket so the two cannot drift + /// — a rollup moved to 30-minute buckets with this still saying 3,600 would halve every rate. + /// + [Fact] + public void DurationTrendHourlySql_DividesByTheBucketWidth_NotALag() + { + Assert.Equal(TimescaleSupport.HourlyBucket.TotalSeconds, + double.Parse(DarlingTrendReader.HourlyBucketSecondsSql, System.Globalization.CultureInfo.InvariantCulture)); + + foreach (var sql in new[] { DarlingTrendReader.QueryDurationTrendHourlySql, DarlingTrendReader.ProcedureDurationTrendHourlySql }) + { + Assert.DoesNotContain("LAG(", sql, StringComparison.Ordinal); + Assert.DoesNotContain("interval_seconds", sql, StringComparison.Ordinal); + Assert.Contains("/ " + DarlingTrendReader.HourlyBucketSecondsSql + " AS elapsed_ms_per_second", sql, StringComparison.Ordinal); + Assert.Contains("/ " + DarlingTrendReader.HourlyBucketSecondsSql + " AS executions_per_second", sql, StringComparison.Ordinal); + } + + /* And the raw reads still LAG — the A11a residual is reported, not silently rewritten here. */ + Assert.Contains("LAG(collection_time)", DarlingTrendReader.QueryDurationTrendSql, StringComparison.Ordinal); + Assert.Contains("LAG(collection_time)", DarlingTrendReader.ProcedureDurationTrendSql, StringComparison.Ordinal); + } + + /// + /// The truncation boundary the four tiered reads share, pinned to the value Lite's twin + /// (McpQueryTools.TruncationSlack) carries: the two SKUs' payloads are one contract, and a window + /// one SKU calls truncated and the other does not is a divergence about the same data. Lite.Tests pins + /// its side to the same ninety minutes; neither project can reference the other's assembly, so the + /// value is pinned twice rather than compared once. + /// + [Fact] + public void TruncationSlack_IsNinetyMinutes_AndDescribeCoverageAppliesIt() + { + Assert.Equal(TimeSpan.FromMinutes(90), DarlingTrendReader.TruncationSlack); + + var start = new DateTime(2026, 3, 4, 6, 0, 0, DateTimeKind.Unspecified); + + /* Empty: the requested start stands and nothing is called truncated — the empty branch's message + carries the coverage story instead. */ + Assert.Equal((start, false), DarlingTrendReader.DescribeCoverage(null, start)); + + /* A head inside the slack is not truncated; one past it is, and effective_start is the head. */ + Assert.Equal((start.AddMinutes(90), false), DarlingTrendReader.DescribeCoverage(start.AddMinutes(90), start)); + Assert.Equal((start.AddMinutes(91), true), DarlingTrendReader.DescribeCoverage(start.AddMinutes(91), start)); + Assert.Equal((start.AddHours(4), true), DarlingTrendReader.DescribeCoverage(start.AddHours(4), start)); + } + /// /// #2484: each probe must read the SAME table its trend reads. A probe on a different source could /// report a server as sampled for rows the trend can never see — the wrong branch in exactly the case @@ -239,7 +401,9 @@ public void QueryHistorySql_OneQuery_CarriesDeltas_ReadsBaseTable() [InlineData(nameof(DarlingTrendReader.DistinctPerfmonCountersSql))] [InlineData(nameof(DarlingTrendReader.FileIoLatencyTrendSql))] [InlineData(nameof(DarlingTrendReader.QueryDurationTrendSql))] + [InlineData(nameof(DarlingTrendReader.QueryDurationTrendHourlySql))] [InlineData(nameof(DarlingTrendReader.ProcedureDurationTrendSql))] + [InlineData(nameof(DarlingTrendReader.ProcedureDurationTrendHourlySql))] [InlineData(nameof(DarlingTrendReader.QueryStoreDurationTrendSql))] [InlineData(nameof(DarlingTrendReader.QueryStoreDurationTrendRollupSql))] [InlineData(nameof(DarlingTrendReader.HasAnyQueryStatSql))] @@ -265,7 +429,9 @@ public void Reads_ArePostgresDialect_NoTsqlIsms(string sqlName) nameof(DarlingTrendReader.DistinctPerfmonCountersSql) => DarlingTrendReader.DistinctPerfmonCountersSql, nameof(DarlingTrendReader.FileIoLatencyTrendSql) => DarlingTrendReader.FileIoLatencyTrendSql, nameof(DarlingTrendReader.QueryDurationTrendSql) => DarlingTrendReader.QueryDurationTrendSql, + nameof(DarlingTrendReader.QueryDurationTrendHourlySql) => DarlingTrendReader.QueryDurationTrendHourlySql, nameof(DarlingTrendReader.ProcedureDurationTrendSql) => DarlingTrendReader.ProcedureDurationTrendSql, + nameof(DarlingTrendReader.ProcedureDurationTrendHourlySql) => DarlingTrendReader.ProcedureDurationTrendHourlySql, nameof(DarlingTrendReader.QueryStoreDurationTrendSql) => DarlingTrendReader.QueryStoreDurationTrendSql, nameof(DarlingTrendReader.QueryStoreDurationTrendRollupSql) => DarlingTrendReader.QueryStoreDurationTrendRollupSql, nameof(DarlingTrendReader.HasAnyQueryStatSql) => DarlingTrendReader.HasAnyQueryStatSql, @@ -440,3 +606,136 @@ private static async Task DeleteRowsAsync(NpgsqlConnection connection, System.Th await cleanup.ExecuteNonQueryAsync(ct); } } + +/// +/// #3541 A2: the tier decision the duration-trend trio shares with get_query_trend, walked as a table +/// without a store. The defect was three reads over ROLLED tables going to raw only — whose rows a +/// TimescaleDB store drops at four days — while accepting a 168-hour window, so a 7-day request returned +/// 4 days under a label saying 7. The complete fix already existed one read over (#2353); what this pins +/// is that the trio now makes the SAME decision for the same inputs, and that the decision degrades to +/// what the store has (#1664) and to what it has materialized (#1759) instead of naming a relation that +/// does not exist or reading an empty rollup while raw still held the rows. +/// +public sealed class DurationTrendTierRoutingTests +{ + private static readonly DateTime Now = new(2026, 8, 19, 12, 0, 0, DateTimeKind.Utc); + + private static readonly RollupCoverage NoCoverage = RollupCoverage.Unknown; + + /// + /// The census: for every hours_back the tools accept, on a fully-built store with no coverage evidence, + /// both plan-cache siblings route exactly where get_query_trend's age rule routes. The trio's + /// route-builders pass their own availability flag and coverage pair, so a wrong flag (the procedure + /// trend reading the query grain's availability, say) fails here even though all three call one + /// resolver. + /// + [Fact] + public void TheTrio_RoutesWhereGetQueryTrendRoutes_ForEveryAcceptedWindow() + { + for (var hoursBack = 1; hoursBack <= McpHelpers.MaxHoursBack; hoursBack++) + { + var start = Now.AddHours(-hoursBack); + var expected = DarlingTrendReader.ShouldUseRawTier(start, Now) ? RetentionTier.Raw : RetentionTier.Hourly; + + Assert.Equal(expected, DarlingTrendReader.ResolveTier(start, Now, hourlyAvailable: true, TierCoverage.Unknown)); + Assert.Equal(expected, DarlingTrendReader.ResolveQueryDurationTrendRoute(start, RollupAvailability.All, NoCoverage, Now).Tier); + Assert.Equal(expected, DarlingTrendReader.ResolveProcedureDurationTrendRoute(start, RollupAvailability.All, NoCoverage, Now).Tier); + } + + /* The two ends of the table, named, so the census cannot pass vacuously on a rule that answers one + tier for everything. */ + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveQueryDurationTrendRoute(Now.AddHours(-24), RollupAvailability.All, NoCoverage, Now).Tier); + Assert.Equal(RetentionTier.Hourly, DarlingTrendReader.ResolveQueryDurationTrendRoute(Now.AddHours(-168), RollupAvailability.All, NoCoverage, Now).Tier); + } + + /// + /// Availability (#1664): a store with no rollups routes every window to raw, because a relation named in + /// a statement is resolved at parse time and because nothing drops raw on such a store anyway. The + /// per-grain flag is the one consulted — a store whose PROCEDURE rollup failed its ensure sweep keeps + /// the query trend on the hourly tier and drops only the procedure trend to raw. + /// + /// And RawRetentionApplies follows the SAME grain, not the store: the #1680 arming gate arms + /// each raw table's purge only once that table's own rollup covers it, so on that partially-built store + /// procedure_stats keeps every row while query_stats is being dropped. A store-wide "any + /// rollup exists" answer would have told the procedure trend's caller that rows were dropped and widening + /// cannot help — the false-and-harmful narrative this route removes, reintroduced (review finding on the + /// first cut of this change). + /// + [Fact] + public void AStoreWithoutTheRollup_RoutesToRaw_AndKeepsRawComplete_PerGrain() + { + var start = Now.AddHours(-168); + + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveQueryDurationTrendRoute(start, RollupAvailability.None, NoCoverage, Now).Tier); + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveProcedureDurationTrendRoute(start, RollupAvailability.None, NoCoverage, Now).Tier); + Assert.False(DarlingTrendReader.ResolveQueryDurationTrendRoute(start, RollupAvailability.None, NoCoverage, Now).RawRetentionApplies); + + var noProcedureRollup = RollupAvailability.All with { ProcedureGrainHourly = false }; + var queryRoute = DarlingTrendReader.ResolveQueryDurationTrendRoute(start, noProcedureRollup, NoCoverage, Now); + var procedureRoute = DarlingTrendReader.ResolveProcedureDurationTrendRoute(start, noProcedureRollup, NoCoverage, Now); + + Assert.Equal(RetentionTier.Hourly, queryRoute.Tier); + Assert.True(queryRoute.RawRetentionApplies); + + Assert.Equal(RetentionTier.Raw, procedureRoute.Tier); + Assert.False(procedureRoute.RawRetentionApplies); + + /* Fully built: both grains' purges can be armed, so both routes carry the flag. */ + Assert.True(DarlingTrendReader.ResolveProcedureDurationTrendRoute(start, RollupAvailability.All, NoCoverage, Now).RawRetentionApplies); + } + + /// + /// Coverage (#1759), the comparative rule: hourly is abandoned for raw ONLY when raw is measured to reach + /// further back than the rollup's floor. A floor that covers the start keeps hourly; a floor above the + /// start with raw no deeper keeps hourly too (on a healthy store raw holds four days against the + /// rollup's ninety, and dropping would return LESS — the head is the payload's to disclose, not + /// routing's to hide); nulls are inert. + /// + [Fact] + public void Coverage_MovesToRaw_OnlyWhenRawIsMeasuredDeeperThanTheRollup() + { + var start = Now.AddHours(-168); + + /* Floor covers the start: hourly. */ + Assert.Equal(RetentionTier.Hourly, DarlingTrendReader.ResolveTier(start, Now, true, new TierCoverage(start.AddDays(-30), null, Now.AddDays(-4)))); + + /* Floor above the start, raw measured DEEPER than the floor: the #1759 held-purge shape — raw. */ + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveTier(start, Now, true, new TierCoverage(Now.AddDays(-2), null, Now.AddDays(-60)))); + + /* Floor above the start, raw NOT deeper: hourly, with the head left for the payload to disclose. */ + Assert.Equal(RetentionTier.Hourly, DarlingTrendReader.ResolveTier(start, Now, true, new TierCoverage(Now.AddDays(-5), null, Now.AddDays(-4)))); + + /* Rollup has materialized nothing (null floor) and raw is measured: raw beats a tier holding nothing. */ + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveTier(start, Now, true, new TierCoverage(null, null, Now.AddDays(-4)))); + + /* Nothing measured at all: inert, the age + availability answer stands. */ + Assert.Equal(RetentionTier.Hourly, DarlingTrendReader.ResolveTier(start, Now, true, TierCoverage.Unknown)); + + /* Coverage never promotes a raw-age window off raw. */ + Assert.Equal(RetentionTier.Raw, DarlingTrendReader.ResolveTier(Now.AddHours(-24), Now, true, new TierCoverage(Now.AddDays(-30), null, null))); + } + + /// The route carries the pair it reads and the word the payload publishes, per grain. + [Fact] + public void TheRoute_NamesItsOwnPair_AndTheSourceWord() + { + var hourly = DarlingTrendReader.ResolveQueryDurationTrendRoute(Now.AddHours(-168), RollupAvailability.All, NoCoverage, Now); + Assert.Equal("hourly", hourly.Source); + Assert.Equal(TimescaleSupport.QueryStatsHourlyView, hourly.Relation); + Assert.Equal("query_stats", hourly.RawTable); + + var raw = DarlingTrendReader.ResolveProcedureDurationTrendRoute(Now.AddHours(-1), RollupAvailability.All, NoCoverage, Now); + Assert.Equal("raw", raw.Source); + Assert.Equal("procedure_stats", raw.Relation); + Assert.Equal(TimescaleSupport.ProcedureStatsHourlyView, raw.HourlyView); + + /* RawReaches: measured against the window start, null when unmeasured. */ + var measured = new DarlingTrendReader.DurationTrendRoute( + RetentionTier.Raw, "query_stats", TimescaleSupport.QueryStatsHourlyView, true, + new TierCoverage(null, null, Now.AddDays(-3)), true, Now); + Assert.True(measured.RawReaches(Now.AddDays(-2))); + Assert.Equal(Now, hourly.ResolvedAtUtc); + Assert.False(measured.RawReaches(Now.AddDays(-4))); + Assert.Null(raw.RawReaches(Now.AddDays(-1))); + } +} diff --git a/Darling/Darling.Tests/DarlingMemoryTrendGrantJoinTests.cs b/Darling/Darling.Tests/DarlingMemoryTrendGrantJoinTests.cs new file mode 100644 index 000000000..94903ba30 --- /dev/null +++ b/Darling/Darling.Tests/DarlingMemoryTrendGrantJoinTests.cs @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3548: get_memory_trend joins the memory-grant series so total_granted_mb carries real data — the +/// complete fix #3529's null was the honest placeholder for. Lite's twin coverage is +/// MemoryTrendGrantJoinToolTests; the two pin the same three claims so the SKUs cannot drift: a +/// matched point carries the pool-summed measurement, a matched point measuring NOTHING granted is a +/// genuine 0.0 (a snapshot existed — zero is a measurement, not a fabrication), and an unmatched point is +/// null with the envelope's granted_note explaining the gap — a note that vanishes entirely when every +/// point matched. The join is nearest-match within 30 seconds because each collector stamps its own +/// DateTime.UtcNow per run: same-cycle rows sit seconds apart, so equality returns nothing, while a wider +/// match would smear a slower grants cadence across points it never measured. +/// +/// Gated on DARLING_TEST_PG like every other live class. +/// +[Collection("live-postgres")] +public sealed class DarlingMemoryTrendGrantJoinTests +{ + private const int ServerId = -949583; + private const string ServerName = "grant-join"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task JoinedPoints_CarryThePoolSum_AGenuineZero_ANullForTheUncovered_AndTheNoteOnlyWithAGap() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live grant-join test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + + /* ── fully covered: one memory point, one grants snapshot 3s later — value, and NO note ── */ + var t0 = MinutesAgo(30); + await SeedMemoryAsync(connection, ct, t0); + await SeedGrantAsync(connection, ct, t0.AddSeconds(3), poolId: 2, grantedMb: 50m); + + var covered = JsonDocument.Parse(await DarlingMcpTrendTools.GetMemoryTrend(postgres, ServerName, 4)).RootElement; + Assert.Equal(50.0, covered.GetProperty("trend")[0].GetProperty("total_granted_mb").GetDouble(), precision: 6); + Assert.False(covered.TryGetProperty("granted_note", out _), + "a window the grants series fully covers must not be captioned with a gap note"); + + /* ── the gap shapes: a two-pool sum, a genuine zero, and an uncovered point ── */ + var t1 = t0.AddMinutes(1); + var t2 = t0.AddMinutes(2); + var t3 = t0.AddMinutes(3); + await SeedMemoryAsync(connection, ct, t1); + await SeedMemoryAsync(connection, ct, t2); + await SeedMemoryAsync(connection, ct, t3); + + var snap1 = t1.AddSeconds(4); + await SeedGrantAsync(connection, ct, snap1, poolId: 1, grantedMb: 25m); + await SeedGrantAsync(connection, ct, snap1, poolId: 2, grantedMb: 100m); + await SeedGrantAsync(connection, ct, t2.AddSeconds(6), poolId: 2, grantedMb: 0m); + /* nothing anywhere near t3 */ + + var root = JsonDocument.Parse(await DarlingMcpTrendTools.GetMemoryTrend(postgres, ServerName, 4)).RootElement; + var trend = root.GetProperty("trend"); + Assert.Equal(4, trend.GetArrayLength()); + Assert.Equal(125.0, trend[1].GetProperty("total_granted_mb").GetDouble(), precision: 6); + Assert.Equal(JsonValueKind.Number, trend[2].GetProperty("total_granted_mb").ValueKind); + Assert.Equal(0.0, trend[2].GetProperty("total_granted_mb").GetDouble(), precision: 6); + Assert.Equal(JsonValueKind.Null, trend[3].GetProperty("total_granted_mb").ValueKind); + + var note = root.GetProperty("granted_note").GetString()!; + Assert.Contains("get_memory_grants", note, StringComparison.Ordinal); + Assert.Contains("30 seconds", note, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static DateTime MinutesAgo(int minutes) => + DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); + + private static async Task SeedMemoryAsync(NpgsqlConnection connection, CancellationToken ct, DateTime t) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO memory_stats + (collection_id, collection_time, server_id, server_name, + total_server_memory_mb, target_server_memory_mb, buffer_pool_mb, plan_cache_mb) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(t), ServerId, ServerName, + 40000m, 49152m, 35000m, 5000m); + + private static async Task SeedGrantAsync(NpgsqlConnection connection, CancellationToken ct, DateTime t, int poolId, decimal grantedMb) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO memory_grant_stats + (collection_id, collection_time, server_id, server_name, + resource_semaphore_id, pool_id, granted_memory_mb) +VALUES ($1, $2, $3, $4, $5, $6, $7)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(t), ServerId, ServerName, + (short)0, poolId, grantedMb); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM memory_grant_stats WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM memory_stats WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM servers WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM config_monitored_servers WHERE server_id = $1", ServerId); + } +} diff --git a/Darling/Darling.Tests/DarlingObservabilityTests.cs b/Darling/Darling.Tests/DarlingObservabilityTests.cs index aa7bedc76..26a640511 100644 --- a/Darling/Darling.Tests/DarlingObservabilityTests.cs +++ b/Darling/Darling.Tests/DarlingObservabilityTests.cs @@ -14,6 +14,7 @@ using PerformanceMonitor.Darling.Service; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; using Xunit; namespace Darling.Tests; @@ -1000,6 +1001,55 @@ await DarlingObservability.WriteAnalysisStateAsync( await DeleteTestRowsAsync(connection); } + /// + /// #3551: the marker's window-empty encoding round-trips through the REAL viewer read and then + /// self-heals. A window-empty pass persists (false, message) — the shape nothing else writes — + /// and must read it as window-empty, not + /// insufficient; the next facts-bearing pass writes (false, null) over the same row, and the + /// viewer read must no longer say window-empty, so a recovered server sheds the notice on its + /// next pass without any extra clearing mechanism. + /// + [Fact] + public async Task WriteAnalysisState_WindowEmptyThenFactsBearingPass_SelfHealsThroughTheViewerRead_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the window-empty marker test."); + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await PgMigrations.MigrateAsync(connection, TestContext.Current.CancellationToken); + await DeleteTestRowsAsync(connection); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + await using var viewer = new ViewerDataService(connectionString!); + + /* A window-empty pass (the worker's #3551 arm): insufficient_data = false WITH the engine's + message. */ + await DarlingObservability.WriteAnalysisStateAsync( + postgres, TestServerId, insufficientData: false, "No facts were collected in the analysis window.", + null, TestContext.Current.CancellationToken); + + var windowEmptyMarker = await viewer.GetAnalysisStateAsync(TestServerId); + Assert.NotNull(windowEmptyMarker); + Assert.True(windowEmptyMarker!.WindowEmpty); + Assert.False(windowEmptyMarker.InsufficientData); + Assert.Equal("No facts were collected in the analysis window.", windowEmptyMarker.Message); + + /* The next facts-bearing pass (false, null) upserts the SAME row — the window-empty marker + self-heals rather than sticking to a recovered server. */ + await DarlingObservability.WriteAnalysisStateAsync( + postgres, TestServerId, insufficientData: false, null, null, TestContext.Current.CancellationToken); + + var healedMarker = await viewer.GetAnalysisStateAsync(TestServerId); + Assert.NotNull(healedMarker); + Assert.False(healedMarker!.WindowEmpty); + Assert.False(healedMarker.InsufficientData); + Assert.Null(healedMarker.Message); + + await DeleteTestRowsAsync(connection); + } + private static async Task<(bool Insufficient, string? Message)> ReadAnalysisStateAsync(NpgsqlConnection connection) { using var read = new NpgsqlCommand("SELECT insufficient_data, message FROM analysis_state WHERE server_id = $1", connection); diff --git a/Darling/Darling.Tests/DarlingPerformanceTrendsReadTests.cs b/Darling/Darling.Tests/DarlingPerformanceTrendsReadTests.cs index 7dc1c43b7..492d3c70d 100644 --- a/Darling/Darling.Tests/DarlingPerformanceTrendsReadTests.cs +++ b/Darling/Darling.Tests/DarlingPerformanceTrendsReadTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Linq; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -101,11 +102,34 @@ database. Naming the collector first would send someone to the wrong place. Assert.Equal("empty", quietQueries.GetProperty("status").GetString()); Assert.Contains("widen", quietQueries.GetProperty("message").GetString()!, StringComparison.Ordinal); + /* #3541 A2: every branch carries the disclosure block — the never-sampled and quiet envelopes + above included — so a caller reads `source` without first checking whether it got data. On + an empty raw answer the requested start stands and the served span is the whole window. */ + foreach (var envelope in new[] { neverProcs, neverStore, neverQueries, quietProcs, quietQueries }) + { + AssertDisclosureBlock(envelope); + Assert.Equal("raw", envelope.GetProperty("source").GetString()); + Assert.False(envelope.GetProperty("truncated").GetBoolean()); + } + + /* Not asserted: effective_hours_back on the empty raw answers. It is derived from the store's + GLOBAL oldest raw row, and on the shared fixture that is whatever another live test left + behind — the scratch-store class below pins it against a store it owns. */ + Assert.Equal("per-interval", neverStore.GetProperty("bucket").GetString()); + Assert.Equal("per-collection", quietQueries.GetProperty("bucket").GetString()); + /* ── the procedure series, at a rate BELOW one execution per second ── Two snapshots five minutes apart, two executions between them: 0.0067/sec. The shipped integer field truncates that to 0, which reads as an idle server; the double does not. This is the whole reason executions_per_second exists. + + These rows carry no sample_interval_seconds (the pre-V128 shape), so the read LAG-derives + the interval — and the FIRST snapshot, which has nothing to LAG against, is UNRATED: two + points come back, the first with null rates (#3541 A12 — kept, not dropped, so a lone + collection is never an empty series and effective_start is the first collection the store + held; never the fabricated 0.0 it was before #3540), the envelope counting it and saying + why, and the second carrying the rate under test. */ await SeedProcedureAsync(connection, ct, MinutesAgo(20), executions: 0, elapsedUs: 0); await SeedProcedureAsync(connection, ct, MinutesAgo(15), executions: 2, elapsedUs: 600_000); @@ -115,6 +139,14 @@ This is the whole reason executions_per_second exists. var procTrend = procs.GetProperty("trend"); Assert.Equal(2, procTrend.GetArrayLength()); + var first = procTrend[0]; + Assert.Equal(JsonValueKind.Null, first.GetProperty("value").ValueKind); + Assert.Equal(JsonValueKind.Null, first.GetProperty("elapsed_ms_per_second").ValueKind); + Assert.Equal(JsonValueKind.Null, first.GetProperty("execution_count").ValueKind); + Assert.Equal(JsonValueKind.Null, first.GetProperty("executions_per_second").ValueKind); + Assert.Equal(1, procs.GetProperty("unrated_points").GetInt32()); + Assert.Contains("no previous one inside the window", procs.GetProperty("unrated_note").GetString()!, StringComparison.Ordinal); + var second = procTrend[1]; Assert.True(second.GetProperty("value").GetDouble() > 0, "elapsed ms/sec must be a real rate"); Assert.Equal(0, second.GetProperty("execution_count").GetInt64()); @@ -122,6 +154,28 @@ This is the whole reason executions_per_second exists. second.GetProperty("executions_per_second").GetDouble() > 0, "executions_per_second must survive a rate below 1/sec that execution_count truncates to zero"); + /* #3541 A2: the unit is in the field name now. Same quantity as `value`, kept beside it on the + execution_count precedent. */ + Assert.Equal(second.GetProperty("value").GetDouble(), second.GetProperty("elapsed_ms_per_second").GetDouble()); + + /* + #3541 A2: the disclosure block. A 4-hour window anchored at now sits inside the raw horizon + (the shared fixture carries no continuous aggregates — every test that builds them mints a + ScratchPostgres — so raw is also the only tier here), and the series the store held begins + at the 20-minutes-ago seed — the unrated first collection, kept since #3541 A12 exactly so + effective_start can say so — and the head sits three-plus hours past the requested start, + which is what `truncated` means. The point is that the label matches the data rather than + the request. + */ + Assert.Equal("raw", procs.GetProperty("source").GetString()); + Assert.Equal("per-collection", procs.GetProperty("bucket").GetString()); + Assert.Equal(JsonValueKind.Null, procs.GetProperty("aggregate_note").ValueKind); + Assert.False(procs.TryGetProperty("routing", out _)); + Assert.Equal(procTrend[0].GetProperty("time").GetString(), procs.GetProperty("effective_start").GetString()); + Assert.True(procs.GetProperty("truncated").GetBoolean()); + var effectiveHours = procs.GetProperty("effective_hours_back").GetDouble(); + Assert.InRange(effectiveHours, 0.2, 0.5); + /* ── the Query Store series: each interval counted ONCE, at the hour the work ran ── Two runtime intervals, each fetched twice while it was open, the second fetch carrying the @@ -143,10 +197,13 @@ separate UtcNow reads truncated to the second can land 3599 apart and quietly br await DarlingMcpTrendTools.GetQueryStoreDurationTrend(postgres, ServerName, 6)).RootElement; var storeTrend = store.GetProperty("trend"); - /* Four rows in, two points out — one per interval, not one per fetch. */ + /* Four rows in, two points out — one per interval, not one per fetch. The first interval has no + predecessor to difference against, so its rates are null, not 0 (#3541 A12). */ Assert.Equal(2, storeTrend.GetArrayLength()); Assert.StartsWith(intervalA.ToString("o")[..16], storeTrend[0].GetProperty("time").GetString()!, StringComparison.Ordinal); + Assert.Equal(JsonValueKind.Null, storeTrend[0].GetProperty("executions_per_second").ValueKind); Assert.StartsWith(intervalB.ToString("o")[..16], storeTrend[1].GetProperty("time").GetString()!, StringComparison.Ordinal); + Assert.Equal(1, store.GetProperty("unrated_points").GetInt32()); /* The surviving snapshot is the FINAL one (25 executions over the 3600 seconds between the two @@ -158,6 +215,14 @@ The surviving snapshot is the FINAL one (25 executions over the 3600 seconds bet storeTrend[1].GetProperty("executions_per_second").GetDouble(), 6); + /* #3541 A2: the Query Store sibling speaks the same disclosure — raw-only here (no corrected + rollup on the shared fixture), its grain named as the interval placement it uses. */ + AssertDisclosureBlock(store); + Assert.Equal("raw", store.GetProperty("source").GetString()); + Assert.Equal("per-interval", store.GetProperty("bucket").GetString()); + Assert.False(store.TryGetProperty("routing", out _)); + Assert.Equal(storeTrend[0].GetProperty("time").GetString(), store.GetProperty("effective_start").GetString()); + bodySucceeded = true; } finally @@ -167,6 +232,17 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = } } + /// The six keys every Performance-Trends envelope carries since #3541 A2, in the order they are + /// written — the same order on the data path, the empty path, and on Lite. + internal static void AssertDisclosureBlock(JsonElement envelope) + { + var keys = envelope.EnumerateObject().Select(p => p.Name).ToArray(); + var block = new[] { "source", "effective_start", "effective_hours_back", "truncated", "bucket", "aggregate_note" }; + var at = Array.IndexOf(keys, "source"); + Assert.True(at >= 0, "the envelope has no `source`"); + Assert.Equal(block, keys.Skip(at).Take(block.Length).ToArray()); + } + private static DateTime MinutesAgo(int minutes) => DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); @@ -214,3 +290,252 @@ private static async Task DeleteRowsAsync(NpgsqlConnection connection, Cancellat await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM config_monitored_servers WHERE server_id = $1", ServerId); } } + +/// +/// #3541 A2 end-to-end against a REAL TimescaleDB: get_query_duration_trend and +/// get_procedure_duration_trend serve a window the raw tier cannot hold from the hourly rollup, and say so. +/// +/// Live rather than a string pin because the load-bearing claims are about which RELATION answered +/// and what it computed: that the rollup-served point is the hour's summed work over the bucket width +/// (no LAG, so no first-point question at all), that the payload's source / effective_start / +/// truncated describe the served series rather than the request, and that the empty branch on this +/// route names an unserved head instead of a quiet window. The raw-only read of the SAME fixture is +/// asserted beside it as the revert-proof: put the raw-only read back and the rates, the point count and +/// the source word all go red, not just the routing. +/// +/// Fixed instants, not now-relative: measures the +/// window's age against the WALL CLOCK, so a window anchored in March 2026 is past the raw horizon on every +/// day this test can run, and every rate below is exact arithmetic off the seed. +/// +/// #1776 own-store — mints a scratch database (it creates continuous aggregates the shared +/// fixture must never inherit), so it is deliberately NOT in the live-postgres collection. +/// +public sealed class DarlingPerformanceTrendsTierRoutingLiveTests +{ + /// Distinctive fake ids — a real server_id is a storage-name hash, never these. + private const int ServerId = -935411; + + private const int NeverSampledServerId = -935412; + + private const string ServerName = "duration-trend-tier-e2e"; + + private const string NeverSampledServerName = "duration-trend-tier-never"; + + [Fact] + public async Task PastTheRawHorizon_TheTrio_ServesTheHourlyRollup_AndSaysSo() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #3541 A2 tier-routing test (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await DarlingMcpTestData.RegisterServerAsync(connection, NeverSampledServerId, NeverSampledServerName, ct); + + var hour10 = new DateTime(2026, 3, 4, 10, 0, 0, DateTimeKind.Unspecified); + var hour11 = hour10.AddHours(1); + var hour12 = hour10.AddHours(2); + + /* ── query_stats: two collections in the 10:00 hour (3.6 s + 7.2 s elapsed, 36 + 72 executions), + one in the 11:00 hour (1.8 s, 18). The hourly answer is the hour's sum over 3,600 s: + 10:00 → 10,800 ms / 3,600 = 3.0 ms/s and 108 / 3,600 = 0.03 exec/s; 11:00 → 0.5 ms/s, 0.005. ── */ + await SeedQueryAsync(connection, ct, hour10.AddMinutes(5), executions: 36, elapsedUs: 3_600_000); + await SeedQueryAsync(connection, ct, hour10.AddMinutes(20), executions: 72, elapsedUs: 7_200_000); + await SeedQueryAsync(connection, ct, hour11.AddMinutes(15), executions: 18, elapsedUs: 1_800_000); + + /* ── procedure_stats: 10:00 → 3.6 s / 18 (1.0 ms/s, 0.005 exec/s); 11:00 → 0.36 s / 2. ── */ + await SeedProcedureAsync(connection, ct, hour10.AddMinutes(5), executions: 9, elapsedUs: 1_800_000); + await SeedProcedureAsync(connection, ct, hour10.AddMinutes(20), executions: 9, elapsedUs: 1_800_000); + await SeedProcedureAsync(connection, ct, hour11.AddMinutes(15), executions: 2, elapsedUs: 360_000); + + await EnsureAggregatesWithoutRefreshPoliciesAsync(connection, ct); + + /* Materialize the 10:00 and 11:00 buckets only. */ + await RefreshRangeAsync(connection, TimescaleSupport.QueryStatsHourlyView, hour10, hour12, ct); + await RefreshRangeAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, hour10, hour12, ct); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + + /* ── the route resolves from what the store has and has materialized: past the horizon by age, + rollup present, floor at 10:00, raw no deeper than the floor → hourly. ── */ + var rollups = await TimescaleSupport.DetectRollupsAsync(postgres, ct); + var coverage = await TimescaleSupport.DetectRollupCoverageAsync(postgres, rollups, ct); + var route = DarlingTrendReader.ResolveQueryDurationTrendRoute(hour10.AddHours(-4), rollups, coverage); + Assert.Equal(RetentionTier.Hourly, route.Tier); + Assert.Equal(hour10, route.Coverage.HourlyFloorUtc); + Assert.Equal(hour10.AddMinutes(5), route.Coverage.RawOldestUtc); + + /* ── the query trend, 06:00–12:00: two hourly points, exact rates, the disclosure describing them ── */ + var queries = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryDurationTrend( + postgres, ServerName, hours_back: 6, as_of: "2026-03-04T12:00:00Z")).RootElement; + + DarlingPerformanceTrendsReadTests.AssertDisclosureBlock(queries); + Assert.Equal("hourly", queries.GetProperty("source").GetString()); + Assert.Equal("1 hour", queries.GetProperty("bucket").GetString()); + Assert.Contains(TimescaleSupport.QueryStatsHourlyView, queries.GetProperty("aggregate_note").GetString(), StringComparison.Ordinal); + Assert.Contains("3,600 seconds", queries.GetProperty("aggregate_note").GetString(), StringComparison.Ordinal); + Assert.StartsWith("2026-03-04T10:00:00", queries.GetProperty("effective_start").GetString()!, StringComparison.Ordinal); + Assert.Equal(2.0, queries.GetProperty("effective_hours_back").GetDouble()); + Assert.True(queries.GetProperty("truncated").GetBoolean(), "the served series begins four hours after the requested start"); + + var queryTrend = queries.GetProperty("trend"); + Assert.Equal(2, queryTrend.GetArrayLength()); + Assert.StartsWith("2026-03-04T10:00:00", queryTrend[0].GetProperty("time").GetString()!, StringComparison.Ordinal); + Assert.Equal(3.0, queryTrend[0].GetProperty("value").GetDouble(), 9); + Assert.Equal(3.0, queryTrend[0].GetProperty("elapsed_ms_per_second").GetDouble(), 9); + Assert.Equal(0.03, queryTrend[0].GetProperty("executions_per_second").GetDouble(), 9); + Assert.StartsWith("2026-03-04T11:00:00", queryTrend[1].GetProperty("time").GetString()!, StringComparison.Ordinal); + Assert.Equal(0.5, queryTrend[1].GetProperty("value").GetDouble(), 9); + Assert.Equal(0.005, queryTrend[1].GetProperty("executions_per_second").GetDouble(), 9); + + /* ── the procedure trend, same window, its own pair ── */ + var procedures = JsonDocument.Parse(await DarlingMcpTrendTools.GetProcedureDurationTrend( + postgres, ServerName, hours_back: 6, as_of: "2026-03-04T12:00:00Z")).RootElement; + + Assert.Equal("hourly", procedures.GetProperty("source").GetString()); + Assert.Contains(TimescaleSupport.ProcedureStatsHourlyView, procedures.GetProperty("aggregate_note").GetString(), StringComparison.Ordinal); + var procedureTrend = procedures.GetProperty("trend"); + Assert.Equal(2, procedureTrend.GetArrayLength()); + Assert.Equal(1.0, procedureTrend[0].GetProperty("value").GetDouble(), 9); + Assert.Equal(0.005, procedureTrend[0].GetProperty("executions_per_second").GetDouble(), 9); + Assert.Equal(0.1, procedureTrend[1].GetProperty("value").GetDouble(), 9); + + /* ── the raw-only read of the SAME fixture: the estimator this replaced, and the revert-proof. + Three per-collection points, the first UNRATED (null) because the LAG idiom has no previous + collection to difference against — before #3541 A12 that point was published as a fabricated + 0.0, the quiet instant the bucket-width denominator never produced. Restoring the raw-only + read unconditionally fails the count, the rates and the `source` word above, not just a + routing flag; restoring the ELSE 0 fails the null here. ── */ + var rawRoute = route with { Tier = RetentionTier.Raw }; + var raw = await DarlingTrendReader.GetQueryDurationTrendAsync(postgres, ServerId, hour10.AddHours(-4), hour12, rawRoute, ct); + + Assert.Equal(3, raw.Points.Count); + Assert.Equal(hour10.AddMinutes(5), raw.Points[0].CollectionTime); + Assert.False(raw.Points[0].HasRate); + Assert.Null(raw.Points[0].Value); + Assert.Null(raw.Points[0].ExecutionCount); + Assert.Null(raw.Points[0].ExecutionsPerSecond); + Assert.Equal(8.0, raw.Points[1].Value!.Value, 9); /* 7,200 ms over the 900 s since 10:05 */ + Assert.Equal(1800d / 3300d, raw.Points[2].Value!.Value, 9); /* 1,800 ms over the 3,300 s since 10:20 */ + /* The unrated point is KEPT, so the series still says truthfully where the store's data begins. */ + Assert.Equal(hour10.AddMinutes(5), raw.EffectiveStartUtc); + Assert.Equal("raw", rawRoute.Source); + + /* ── never sampled on this route: not an empty window. The raw probe finds nothing and so does the + rollup — "unavailable", with the disclosure block still attached. ── */ + var never = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryDurationTrend( + postgres, NeverSampledServerName, hours_back: 6, as_of: "2026-03-04T12:00:00Z")).RootElement; + + Assert.Equal("unavailable", never.GetProperty("status").GetString()); + Assert.Contains("EVER", never.GetProperty("message").GetString()!, StringComparison.Ordinal); + DarlingPerformanceTrendsReadTests.AssertDisclosureBlock(never); + Assert.Equal("hourly", never.GetProperty("source").GetString()); + + /* ── the branch this fix exists for: sampled, nothing in a window whose head sits BELOW the rollup's + floor. The pre-routing answer here was "genuinely quiet — widen hours_back", which is false (the + raw rows were dropped, not absent) and harmful (widening reaches further into what nothing + holds). The answer now names the tier, its measured floor, the unserved head and the remedy. ── */ + var unserved = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryDurationTrend( + postgres, ServerName, hours_back: 2, as_of: "2026-03-04T09:00:00Z")).RootElement; + + Assert.Equal("empty", unserved.GetProperty("status").GetString()); + var message = unserved.GetProperty("message").GetString()!; + Assert.Contains(TimescaleSupport.QueryStatsHourlyView, message, StringComparison.Ordinal); + Assert.Contains("2026-03-04T10:00:00", message, StringComparison.Ordinal); + Assert.Contains("UNSERVED rather than quiet", message, StringComparison.Ordinal); + Assert.Contains("--backfill-rollups", message, StringComparison.Ordinal); + Assert.Contains("Widening hours_back cannot help", message, StringComparison.Ordinal); + Assert.DoesNotContain("genuinely quiet", message, StringComparison.Ordinal); + Assert.DoesNotContain("EVER", message, StringComparison.Ordinal); + + /* The disclosure on that empty answer: the tier could first have served at its 10:00 floor, which is + past the window's end, so the served span is honestly zero and the head is truncated. */ + Assert.Equal("hourly", unserved.GetProperty("source").GetString()); + Assert.StartsWith("2026-03-04T09:00:00", unserved.GetProperty("effective_start").GetString()!, StringComparison.Ordinal); + Assert.Equal(0.0, unserved.GetProperty("effective_hours_back").GetDouble()); + Assert.True(unserved.GetProperty("truncated").GetBoolean()); + + /* ── and get_query_trend, the read whose routing the trio now shares, over the same rows: the same + tier, the same coverage words, the rollup's per-hour sums for the one query. ── */ + var single = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryTrend( + postgres, "0xTIERHASH", "AppDb", ServerName, hours_back: 6, as_of: "2026-03-04T12:00:00Z")).RootElement; + + Assert.Equal("hourly", single.GetProperty("source").GetString()); + Assert.Equal(queries.GetProperty("effective_start").GetString(), single.GetProperty("effective_start").GetString()); + Assert.Equal(2, single.GetProperty("data_points").GetInt32()); + Assert.Equal(108, single.GetProperty("trend")[0].GetProperty("execution_count").GetInt64()); + } + + private static async Task SeedQueryAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, long executions, long elapsedUs) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO query_stats + (collection_id, collection_time, server_id, server_name, database_name, query_hash, sql_handle, + delta_execution_count, delta_elapsed_time, delta_worker_time) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + CollectionIdGenerator.Next(), collectionTimeUtc, ServerId, ServerName, + "AppDb", "0xTIERHASH", "0xSQLH", executions, elapsedUs, elapsedUs / 2); + + private static async Task SeedProcedureAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, long executions, long elapsedUs) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO procedure_stats + (collection_id, collection_time, server_id, server_name, database_name, schema_name, object_name, + delta_execution_count, delta_elapsed_time, delta_worker_time) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + CollectionIdGenerator.Next(), collectionTimeUtc, ServerId, ServerName, + "AppDb", "dbo", "usp_Tier", executions, elapsedUs, elapsedUs / 2); + + /// + /// Builds the aggregates, then strips every refresh policy the sweep attached — this test refreshes + /// manually over exact ranges and asserts exact floors, so a background refresh materializing another + /// bucket mid-test would move the floor out from under the assertions. Same discipline as + /// QueryStoreTrendRoutingLiveTests. + /// + private static async Task EnsureAggregatesWithoutRefreshPoliciesAsync(NpgsqlConnection connection, CancellationToken ct) + { + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + foreach (var (view, _, _, _, _) in TimescaleSupport.RollupViews) + { + await using var remove = new NpgsqlCommand( + $"SELECT remove_continuous_aggregate_policy('collect.{view}', if_exists => true)", connection); + await remove.ExecuteNonQueryAsync(ct); + } + } + + /// Materializes a range, retrying while TimescaleDB reports a CONCURRENT REFRESH (55P03) — the + /// policy's first check fires immediately on creation, so the scheduler can be mid-materialization when + /// the manual refresh lands. Bounded, so a genuine stall fails rather than hangs. + private static async Task RefreshRangeAsync( + NpgsqlConnection connection, string view, DateTime from, DateTime to, CancellationToken ct) + { + const int maxAttempts = 12; + + for (var attempt = 1; ; attempt++) + { + try + { + await using var refresh = new NpgsqlCommand( + $"CALL refresh_continuous_aggregate('collect.{view}', $1::timestamp, $2::timestamp)", connection); + refresh.Parameters.AddWithValue(from); + refresh.Parameters.AddWithValue(to); + await refresh.ExecuteNonQueryAsync(ct); + return; + } + catch (PostgresException ex) when (ex.SqlState == "55P03" && attempt < maxAttempts) + { + await Task.Delay(TimeSpan.FromSeconds(1), ct); + } + } + } +} diff --git a/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs b/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs index 1c1d230e2..55e41bb86 100644 --- a/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs +++ b/Darling/Darling.Tests/DarlingPgAutovacuumReaderTests.cs @@ -191,6 +191,20 @@ public void FarPastThresholdIsCriticalEvenWhenFlat() Assert.Equal("critical_far_past_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 25.0, true)); } + /// + /// Growth null — a one-sample window — never escalates to the growing verdict and never reads as + /// flat's "blocked or not running" implication either: the band's plain label carries it, and the + /// bands growth does not refine are untouched by it. + /// + [Fact] + public void UnknownGrowthNeverEscalatesAndNeverReadsFlat() + { + Assert.Equal("warning_past_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 3.0, null)); + Assert.Equal("critical_far_past_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 25.0, null)); + Assert.Equal("info_at_threshold", DarlingMcpPgAutovacuumTools.Classify(false, 1.5, null)); + Assert.Equal("ok", DarlingMcpPgAutovacuumTools.Classify(false, 0.5, null)); + } + /// Every severity is a distinct string, so a caller can switch on it. [Fact] public void SeveritiesAreDistinct() diff --git a/Darling/Darling.Tests/DarlingPgAutovacuumVerdictLiveTests.cs b/Darling/Darling.Tests/DarlingPgAutovacuumVerdictLiveTests.cs new file mode 100644 index 000000000..cf5390132 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgAutovacuumVerdictLiveTests.cs @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// get_pg_autovacuum_health end to end against a real store — the #3534 verdict seam. +/// +/// The test that carries this class is the append-only table. The read ranks by +/// GREATEST(dead ratio, insert ratio), so a table ten times past its INSERT threshold with zero dead +/// tuples arrives as worst_table — and the tool used to classify it from the dead ratio alone, handing +/// the #1-ranked table severity "ok" with a 0 threshold_ratio. An agent reads that as "worst thing here +/// is fine" and moves on from the classic wraparound route the collector gathers inserts_since_vacuum +/// for in the first place. The projection arithmetic lives inline in the tool, so only a real round-trip +/// exercises it. +/// +/// And the growth claim a single sample cannot make. One reading means first == latest, and +/// dead_tuples_growing = false from it converts into "autovacuum blocked or not running" territory the +/// window cannot support. Null, with first_seen_at published so the caller can see how much history +/// stands behind the claim. +/// +[Collection("live-postgres")] +public sealed class DarlingPgAutovacuumVerdictLiveTests +{ + private const int ServerId = -853917; + private const string ServerName = "pg-autovacuum-verdict-e2e"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task InsertDrivenWorstTableCarriesItsRankSeverity_AndOneSampleGrowthIsUnknown() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live autovacuum verdict test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var dataSource = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + + var t0 = MinutesAgo(30); + var t1 = t0.AddMinutes(10); + + /* appendonly: zero dead tuples, 10x past its INSERT threshold. GREATEST ranks it #1; the + dead-only severity called it "ok" (#3534's scenario, verbatim). */ + await SeedAsync(connection, ct, t0, "appendonly", deadTuples: 0, vacuumThreshold: 1_050, + insertsSinceVacuum: 100_000, insertVacuumThreshold: 10_000); + await SeedAsync(connection, ct, t1, "appendonly", deadTuples: 0, vacuumThreshold: 1_050, + insertsSinceVacuum: 100_000, insertVacuumThreshold: 10_000); + + /* churny: past its dead threshold and climbing — the case the dead axis already handled, + kept as the control that the insert axis widens the verdict rather than replacing it. */ + await SeedAsync(connection, ct, t0, "churny", deadTuples: 1_500, vacuumThreshold: 1_000, + insertsSinceVacuum: 0, insertVacuumThreshold: 10_000); + await SeedAsync(connection, ct, t1, "churny", deadTuples: 2_500, vacuumThreshold: 1_000, + insertsSinceVacuum: 0, insertVacuumThreshold: 10_000); + + /* onesample: a single reading. Growth is unmeasurable, which is not the same as "not + growing". */ + await SeedAsync(connection, ct, t1, "onesample", deadTuples: 3_000, vacuumThreshold: 1_000, + insertsSinceVacuum: 0, insertVacuumThreshold: 10_000); + + var payload = JsonDocument.Parse( + await DarlingMcpPgAutovacuumTools.GetPgAutovacuumHealth(dataSource, ServerName, 4)).RootElement; + + Assert.Equal("tables_with_pending_maintenance", payload.GetProperty("status").GetString()); + + var tables = payload.GetProperty("tables").EnumerateArray().ToArray(); + + /* THE assertion: worst_table and its severity come from the SAME axis. Ten times past the + insert threshold is critical, and the insert-side ratio is published beside the dead one + so the figure the verdict came from is visible. */ + Assert.Equal("public.appendonly", payload.GetProperty("worst_table").GetString()); + Assert.Equal("critical_far_past_threshold", payload.GetProperty("worst_severity").GetString()); + + var append = Row(tables, "public.appendonly"); + Assert.Equal("critical_far_past_threshold", append.GetProperty("severity").GetString()); + Assert.Equal(10.0, append.GetProperty("insert_threshold_ratio").GetDouble(), 2); + Assert.Equal(0.0, append.GetProperty("threshold_ratio").GetDouble(), 2); + + /* The control: a dead-driven table classifies exactly as it always did. */ + var churny = Row(tables, "public.churny"); + Assert.Equal("warning_past_threshold_and_growing", churny.GetProperty("severity").GetString()); + Assert.True(churny.GetProperty("dead_tuples_growing").GetBoolean()); + Assert.Equal(1_000, churny.GetProperty("dead_tuple_change").GetInt64()); + + /* One sample: growth and the change figure are NULL, and first_seen_at == measured_at says + why — the window holds one reading, not a flat line. The band's plain label carries it. */ + var oneSample = Row(tables, "public.onesample"); + Assert.Equal(JsonValueKind.Null, oneSample.GetProperty("dead_tuples_growing").ValueKind); + Assert.Equal(JsonValueKind.Null, oneSample.GetProperty("dead_tuple_change").ValueKind); + Assert.Equal( + oneSample.GetProperty("measured_at").GetDateTime(), + oneSample.GetProperty("first_seen_at").GetDateTime()); + Assert.Equal("warning_past_threshold", oneSample.GetProperty("severity").GetString()); + + /* The summary counts read both axes, growing counts only MEASURED growth, and the + undersized call discloses its page scope (the get_pg_database_stats pattern). */ + Assert.Equal(3, payload.GetProperty("past_threshold_count").GetInt32()); + Assert.Equal(1, payload.GetProperty("growing_count").GetInt32()); + Assert.False(payload.GetProperty("limit_reached").GetBoolean()); + + var truncated = JsonDocument.Parse( + await DarlingMcpPgAutovacuumTools.GetPgAutovacuumHealth(dataSource, ServerName, 4, 2)).RootElement; + Assert.Equal(2, truncated.GetProperty("table_count").GetInt32()); + Assert.True(truncated.GetProperty("limit_reached").GetBoolean()); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// The row for one table, asserted present with a message naming what DID come back. + private static JsonElement Row(JsonElement[] rows, string tableName) + { + var row = rows.FirstOrDefault(r => r.GetProperty("table_name").GetString() == tableName); + + Assert.True( + row.ValueKind == JsonValueKind.Object, + $"no row for '{tableName}' — the read returned [{string.Join(", ", rows.Select(r => r.GetProperty("table_name").GetString()))}]"); + + return row; + } + + private static DateTime MinutesAgo(int minutes) => + DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); + + private static async Task SeedAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, string tableName, + long deadTuples, long vacuumThreshold, long insertsSinceVacuum, long insertVacuumThreshold) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_autovacuum_stats + (collection_id, collection_time, server_id, server_name, database_name, schema_name, table_name, + live_tuples, dead_tuples, vacuum_threshold, mods_since_analyze, analyze_threshold, + inserts_since_vacuum, insert_vacuum_threshold, autovacuum_disabled, total_bytes, autovacuum_count) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(collectionTimeUtc), + ServerId, ServerName, "appdb", "public", tableName, + 10_000L, deadTuples, vacuumThreshold, 0L, 500L, + insertsSinceVacuum, insertVacuumThreshold, false, 1_000_000L, 1L); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM pg_autovacuum_stats WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM servers WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM config_monitored_servers WHERE server_id = $1", ServerId); + } +} diff --git a/Darling/Darling.Tests/DarlingPgReadSqlParsesLiveTests.cs b/Darling/Darling.Tests/DarlingPgReadSqlParsesLiveTests.cs index 986429317..896e767c8 100644 --- a/Darling/Darling.Tests/DarlingPgReadSqlParsesLiveTests.cs +++ b/Darling/Darling.Tests/DarlingPgReadSqlParsesLiveTests.cs @@ -74,8 +74,8 @@ public sealed class DarlingPgReadSqlParsesLiveTests /// explicitly rather than inferred, because every rule for inferring it reads the field's VALUE — which /// is reflection, the very thing the census exists to check independently. /// - /// Two of the three are SQL FRAGMENTS interpolated into a query rather than queries themselves, - /// and the third is the trap that makes a name-shaped rule useless here: + /// All but one are SQL FRAGMENTS interpolated into a query rather than queries themselves, + /// and the one is the trap that makes a name-shaped rule useless here: /// StaleStatisticsChurnRatioSql is spelled like a query and holds "0.2". Keyed per SITE /// rather than per name, so the same name on a different reader is not excused by inheritance from /// this one — and the unused-entry clause below makes a move show up as an edit here rather than as @@ -90,6 +90,12 @@ statement. The query it is spliced into - CoverageEvidenceSql - IS in the parse- population, so the fragment is verified where it is used rather than left unverified. */ "DarlingPgIndexBloatReader.EvidenceStatusList", "DarlingPgServerConfigReader.SessionScopedSources", + /* #3539: the PostgreSQL Long-Running Query read's switchable dump/restore opt-out - one AND + predicate spliced into CurrentLongRunningSessionsSqlTemplate by BuildCurrentLongRunningSessionsSql. + Not a statement; both renderings of the template it is spliced into + (CurrentLongRunningSessionsSql with it, CurrentLongRunningSessionsSqlBackupsIncluded without) ARE + in the parse-checked population, so the fragment is verified where it is used. */ + "DarlingPgSessionStatesReader.BackupUtilitiesFilter", "DarlingPgTableBloatReader.StaleStatisticsChurnRatioSql", }; diff --git a/Darling/Darling.Tests/DarlingPgSessionStatesReaderTests.cs b/Darling/Darling.Tests/DarlingPgSessionStatesReaderTests.cs index 7522e5ba1..14954c1c3 100644 --- a/Darling/Darling.Tests/DarlingPgSessionStatesReaderTests.cs +++ b/Darling/Darling.Tests/DarlingPgSessionStatesReaderTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; using PerformanceMonitor.Collectors; @@ -106,6 +107,92 @@ public void LongRunningSql_ExcludesIdleInTransactionSessions() Assert.Contains("s.is_idle_in_transaction = false", LongRunningSql, StringComparison.Ordinal); } + // ── #3539 — the noise opt-outs SQL Server's read has had all along ─────────────────────────── + + /// + /// Non-client backends are out unconditionally — the sibling of SQL Server's session_id > 50. + /// Autovacuum workers and walsenders are the two that reached the alert in the field (an autovacuum of + /// a large table at minute 31, a streaming standby's walsender for as long as it is connected). NULL + /// keeps the row: backend_type is privileged and comes back NULL without pg_monitor, and a + /// filter that dropped NULL would silence the alert on exactly the targets the collector has already + /// flagged as redacted. + /// + [Fact] + public void LongRunningSql_ExcludesNonClientBackends_KeepingRedactedNulls() + { + Assert.Contains("coalesce(s.backend_type, 'client backend') = 'client backend'", LongRunningSql, StringComparison.Ordinal); + } + + /// + /// The four maintenance statements are out unconditionally, by the whitelisted command tag — the only + /// statement handle this table has, because it stores no text. CREATE is deliberately not among + /// them: the tag cannot separate an index build from a CREATE TABLE AS, so a CREATE is reported + /// with its tag rather than dropped on a guess. NULL-safe in the KEEPING direction like the backend + /// filter — a NULL tag is a session the collector could not classify, not a maintenance statement. + /// + [Fact] + public void LongRunningSql_ExcludesMaintenanceStatementsByCommandTag_ButNotCreate() + { + Assert.Contains("coalesce(s.command_tag, '') NOT IN ('VACUUM', 'ANALYZE', 'REINDEX', 'CLUSTER')", LongRunningSql, StringComparison.Ordinal); + Assert.DoesNotContain("'CREATE'", LongRunningSql, StringComparison.Ordinal); + } + + /// + /// The dump/restore utilities ride the SHARED longRunningQueryExcludeBackups switch (SQL Server's + /// BackupsFilter), so the filter is in the text exactly when the switch is on, and the four names + /// are libpq's fallback_application_name for those tools. psql is not in the list — an + /// operator's ad-hoc statement running long IS a long-running query. The default rendering (the constant + /// under the pre-#3539 name) is the switch-on shape, because that is the shipped default. + /// + [Fact] + public void LongRunningSql_DropsDumpAndRestoreUtilities_OnlyWhenTheSharedBackupsSwitchIsOn() + { + var on = DarlingPgSessionStatesReader.BuildCurrentLongRunningSessionsSql(excludeBackups: true); + var off = DarlingPgSessionStatesReader.BuildCurrentLongRunningSessionsSql(excludeBackups: false); + + Assert.Equal( + "AND coalesce(s.application_name, '') NOT IN ('pg_dump', 'pg_dumpall', 'pg_restore', 'pg_basebackup')", + DarlingPgSessionStatesReader.BackupUtilitiesFilter); + Assert.Contains(DarlingPgSessionStatesReader.BackupUtilitiesFilter, on, StringComparison.Ordinal); + Assert.DoesNotContain("application_name", off.Replace("s.application_name,", ""), StringComparison.Ordinal); + Assert.DoesNotContain("'psql'", on, StringComparison.Ordinal); + Assert.Equal(on, LongRunningSql); + + /* The placeholder never reaches the server, on either setting. */ + Assert.DoesNotContain("{0}", on, StringComparison.Ordinal); + Assert.DoesNotContain("{0}", off, StringComparison.Ordinal); + + /* And the unconditional filters are in BOTH renderings - the switch governs only the utilities. */ + foreach (var sql in new[] { on, off }) + { + Assert.Contains("'client backend'", sql, StringComparison.Ordinal); + Assert.Contains("'VACUUM'", sql, StringComparison.Ordinal); + Assert.Contains("s.is_idle_in_transaction = false", sql, StringComparison.Ordinal); + } + } + + /// + /// excludedDatabases is applied after the read with the SQL Server adapter's exact rule: ordinal + /// ignore-case on the name, a row with no database name kept, null/empty list a no-op. Executed here, + /// not just pinned — the rule is pure. + /// + [Fact] + public void ExcludedDatabases_AppliedAfterTheRead_CaseInsensitively_KeepingUnnamedRows() + { + var rows = new List + { + new(1, 101, "Orders", "app", "svc", "SELECT", 3_600_000), + new(2, 102, "billing", "app", "svc", "UPDATE", 2_400_000), + new(3, 103, null, "app", "svc", "(other)", 1_900_000), + }; + + var filtered = DarlingPgSessionStatesReader.FilterExcludedDatabases(rows, new[] { "ORDERS" }); + Assert.Equal(new long[] { 2, 3 }, filtered.Select(r => r.BackendId).ToArray()); + + Assert.Same(rows, DarlingPgSessionStatesReader.FilterExcludedDatabases(rows, null)); + Assert.Same(rows, DarlingPgSessionStatesReader.FilterExcludedDatabases(rows, Array.Empty())); + } + // ── Scoping and parameterisation ───────────────────────────────────────────────────────────── [Fact] diff --git a/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs b/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs index a62df35ce..a14d175cc 100644 --- a/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs +++ b/Darling/Darling.Tests/DarlingPgSlotReaderTests.cs @@ -88,6 +88,7 @@ public void TerminalWalStatesAreCriticalRegardlessOfActivityOrGrowth(string walS { Assert.Equal(expected, DarlingMcpPgSlotTools.Classify(walStatus, isActive: true, retainedWalGrowing: false)); Assert.Equal(expected, DarlingMcpPgSlotTools.Classify(walStatus, isActive: false, retainedWalGrowing: true)); + Assert.Equal(expected, DarlingMcpPgSlotTools.Classify(walStatus, isActive: false, retainedWalGrowing: null)); } /// @@ -137,6 +138,92 @@ public void ActiveReservedSlotIsOk() Assert.Equal("ok", DarlingMcpPgSlotTools.Classify("reserved", isActive: true, retainedWalGrowing: false)); } + /// + /// Unknown growth — the collector's -1 sentinel, or a one-sample window — is its own verdict, never + /// the flat one. "Flat" is a measured claim, and it is the exact claim that separates a consumer + /// between polls from a volume filling; a sentinel spelled as measured zero read every unmeasurable + /// slot as stable (#3535). + /// + [Fact] + public void UnknownGrowthIsItsOwnVerdictNeverFlat() + { + /* The orphan-candidate shape with its discriminator unmeasured: a warning that says so, not + the measured-flat warning and not a fabricated critical. */ + Assert.Equal( + "warning_retaining_wal_growth_unknown", + DarlingMcpPgSlotTools.Classify("extended", isActive: false, retainedWalGrowing: null)); + + Assert.Equal( + "info_inactive_growth_unknown", + DarlingMcpPgSlotTools.Classify("reserved", isActive: false, retainedWalGrowing: null)); + Assert.Equal( + "info_inactive_growth_unknown", + DarlingMcpPgSlotTools.Classify(null, isActive: false, retainedWalGrowing: null)); + + /* Where measured growth would not escalate anyway, unknown growth manufactures nothing: an + active consumer stays ok, and an active extended slot stays the plain retaining warning. */ + Assert.Equal("ok", DarlingMcpPgSlotTools.Classify("reserved", isActive: true, retainedWalGrowing: null)); + Assert.Equal( + "warning_retaining_wal", + DarlingMcpPgSlotTools.Classify("extended", isActive: true, retainedWalGrowing: null)); + } + + /// + /// worst_slot is picked by severity rank with size only as the tiebreak, so the ladder itself is + /// pinned: strictly descending, criticals above warnings above infos above ok, and a label Rank does + /// not know sorts with ok rather than above anything it does. + /// + [Fact] + public void RankOrdersEverySeverityWorstFirst() + { + var ladder = new[] + { + "critical_slot_lost", + "critical_wal_already_removed", + "critical_orphan_filling_disk", + "warning_inactive_and_growing", + "warning_retaining_wal_growth_unknown", + "warning_retaining_wal", + "info_inactive_growth_unknown", + "info_inactive", + "ok", + }; + + for (var i = 1; i < ladder.Length; i++) + { + Assert.True( + DarlingMcpPgSlotTools.Rank(ladder[i - 1]) > DarlingMcpPgSlotTools.Rank(ladder[i]), + $"'{ladder[i - 1]}' must outrank '{ladder[i]}'"); + } + + Assert.Equal(0, DarlingMcpPgSlotTools.Rank("ok")); + Assert.Equal(0, DarlingMcpPgSlotTools.Rank("a_label_rank_does_not_know")); + } + + /// + /// Every non-ok label Classify can emit holds a rung above ok, so a future Classify arm that skips + /// Rank cannot ship a severity that silently never headlines. + /// + [Fact] + public void EveryClassifiableSeverityHasARung() + { + var emittable = new[] + { + DarlingMcpPgSlotTools.Classify("lost", false, false), + DarlingMcpPgSlotTools.Classify("unreserved", false, false), + DarlingMcpPgSlotTools.Classify("extended", false, true), + DarlingMcpPgSlotTools.Classify("extended", false, null), + DarlingMcpPgSlotTools.Classify("extended", true, false), + DarlingMcpPgSlotTools.Classify("reserved", false, true), + DarlingMcpPgSlotTools.Classify("reserved", false, null), + DarlingMcpPgSlotTools.Classify("reserved", false, false), + }; + + Assert.All(emittable, severity => Assert.True( + DarlingMcpPgSlotTools.Rank(severity) > 0, + $"'{severity}' ranks 0 — it ties with ok and can never be picked as worst_slot")); + } + /// /// wal_status is NULL on a slot with no restart_lsn yet, and on Aurora it can be absent entirely. /// That must fall through to the activity/growth branches rather than throwing or reading as healthy @@ -161,8 +248,10 @@ public void SeveritiesAreDistinct() DarlingMcpPgSlotTools.Classify("lost", false, false), DarlingMcpPgSlotTools.Classify("unreserved", false, false), DarlingMcpPgSlotTools.Classify("extended", false, true), + DarlingMcpPgSlotTools.Classify("extended", false, null), DarlingMcpPgSlotTools.Classify("extended", true, false), DarlingMcpPgSlotTools.Classify("reserved", false, true), + DarlingMcpPgSlotTools.Classify("reserved", false, null), DarlingMcpPgSlotTools.Classify("reserved", false, false), DarlingMcpPgSlotTools.Classify("reserved", true, false), }; diff --git a/Darling/Darling.Tests/DarlingPgSlotVerdictLiveTests.cs b/Darling/Darling.Tests/DarlingPgSlotVerdictLiveTests.cs new file mode 100644 index 000000000..7a97c5208 --- /dev/null +++ b/Darling/Darling.Tests/DarlingPgSlotVerdictLiveTests.cs @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// get_pg_replication_slots end to end against a real store — the #3535 verdict seam. +/// +/// The test that carries this class is the severity-versus-size pick. The tool's own design +/// note says the size of the hole matters less than whether it is still being dug, and then worst_slot +/// was picked by retained bytes: an active 45 GB keeping-pace slot ("ok") headlined over an inactive +/// 2→8 GB grower ("critical_orphan_filling_disk") — the one slot the caller needed to see first. The +/// ordering and pick live inline in the tool, so only a real round-trip exercises them. +/// +/// And the growth a sentinel cannot measure. The collector's -1 not-applicable sentinel used +/// to become a measured zero — "no growth" — so an unmeasurable slot read as stable; a one-sample window +/// is the same fabrication from a single point. Both surface as null growth plus an unknown-growth +/// verdict, never as flat. +/// +[Collection("live-postgres")] +public sealed class DarlingPgSlotVerdictLiveTests +{ + private const int ServerId = -853918; + private const string ServerName = "pg-slot-verdict-e2e"; + + private const long Gb = 1024L * 1024 * 1024; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task WorstSlotIsTheWorstClassified_AndUnmeasurableGrowthReadsUnknown() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live slot verdict test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var dataSource = NpgsqlDataSource.Create(cs!); + var bodySucceeded = false; + + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + + var t0 = MinutesAgo(30); + var t1 = t0.AddMinutes(10); + + /* fat_ok: the biggest hole and the healthiest slot here — active, reserved, holding 45 GB + steady. The size pick made this the headline (#3535's scenario, verbatim). */ + await SeedAsync(connection, ct, t0, "fat_ok", isActive: true, walStatus: "reserved", retainedWalBytes: 45 * Gb); + await SeedAsync(connection, ct, t1, "fat_ok", isActive: true, walStatus: "reserved", retainedWalBytes: 45 * Gb); + + /* thin_orphan: a fraction of the size, inactive, extended, and GROWING — the disk bomb. */ + await SeedAsync(connection, ct, t0, "thin_orphan", isActive: false, walStatus: "extended", retainedWalBytes: 2 * Gb); + await SeedAsync(connection, ct, t1, "thin_orphan", isActive: false, walStatus: "extended", retainedWalBytes: 8 * Gb); + + /* sentinel: retained bytes NULL in both readings (the -1 sentinel downstream) — growth is + unknowable, and "stable" would be a fabricated claim. */ + await SeedAsync(connection, ct, t0, "sentinel", isActive: false, walStatus: "extended", retainedWalBytes: null); + await SeedAsync(connection, ct, t1, "sentinel", isActive: false, walStatus: "extended", retainedWalBytes: null); + + /* single: one reading with a measured size — the size is real, the growth claim is not. */ + await SeedAsync(connection, ct, t1, "single", isActive: false, walStatus: "reserved", retainedWalBytes: 1 * Gb); + + var payload = JsonDocument.Parse( + await DarlingMcpPgSlotTools.GetPgReplicationSlots(dataSource, ServerName, 4)).RootElement; + + Assert.Equal("slots_present", payload.GetProperty("status").GetString()); + + /* THE assertion: the headline is the worst-CLASSIFIED slot, not the fattest, and the list + leads with it. */ + Assert.Equal("thin_orphan", payload.GetProperty("worst_slot").GetString()); + Assert.Equal("critical_orphan_filling_disk", payload.GetProperty("worst_severity").GetString()); + + var slots = payload.GetProperty("slots").EnumerateArray().ToArray(); + Assert.Equal("thin_orphan", slots[0].GetProperty("slot_name").GetString()); + + /* The fat slot keeps its honest verdict and its figure — it just no longer buys the + headline with it. */ + var fat = Row(slots, "fat_ok"); + Assert.Equal("ok", fat.GetProperty("severity").GetString()); + Assert.Equal(45.0, fat.GetProperty("retained_wal_gb").GetDouble(), 2); + + /* Sentinel: null size (raw AND _gb — the raw field used to serialize -1), null growth, and + the unknown-growth verdict rather than the measured-flat one. */ + var sentinel = Row(slots, "sentinel"); + Assert.Equal("warning_retaining_wal_growth_unknown", sentinel.GetProperty("severity").GetString()); + Assert.Equal(JsonValueKind.Null, sentinel.GetProperty("retained_wal_bytes").ValueKind); + Assert.Equal(JsonValueKind.Null, sentinel.GetProperty("retained_wal_gb").ValueKind); + Assert.Equal(JsonValueKind.Null, sentinel.GetProperty("retained_wal_growth_bytes").ValueKind); + Assert.Equal(JsonValueKind.Null, sentinel.GetProperty("retained_wal_growth_gb_per_hour").ValueKind); + + /* One sample: measured size, unmeasurable growth. */ + var single = Row(slots, "single"); + Assert.Equal("info_inactive_growth_unknown", single.GetProperty("severity").GetString()); + Assert.Equal(1.0, single.GetProperty("retained_wal_gb").GetDouble(), 2); + Assert.Equal(JsonValueKind.Null, single.GetProperty("retained_wal_growth_bytes").ValueKind); + + /* The total sums only measured sizes — a sentinel never subtracts phantom gigabytes. */ + Assert.Equal(54.0, payload.GetProperty("total_retained_wal_gb").GetDouble(), 1); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// The row for one slot, asserted present with a message naming what DID come back. + private static JsonElement Row(JsonElement[] rows, string slotName) + { + var row = rows.FirstOrDefault(r => r.GetProperty("slot_name").GetString() == slotName); + + Assert.True( + row.ValueKind == JsonValueKind.Object, + $"no row for '{slotName}' — the read returned [{string.Join(", ", rows.Select(r => r.GetProperty("slot_name").GetString()))}]"); + + return row; + } + + private static DateTime MinutesAgo(int minutes) => + DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); + + private static async Task SeedAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, string slotName, + bool isActive, string walStatus, long? retainedWalBytes) => + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO collect.pg_replication_slot_stats + (collection_id, collection_time, server_id, server_name, slot_name, slot_type, plugin, + database_name, is_active, wal_status, retained_wal_bytes, conflicting) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(collectionTimeUtc), + ServerId, ServerName, slotName, "logical", "pgoutput", "appdb", isActive, walStatus, + retainedWalBytes, false); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM collect.pg_replication_slot_stats WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM servers WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM config_monitored_servers WHERE server_id = $1", ServerId); + } +} diff --git a/Darling/Darling.Tests/DarlingSelfAlertTests.cs b/Darling/Darling.Tests/DarlingSelfAlertTests.cs index 96cc96170..a3d0aa38d 100644 --- a/Darling/Darling.Tests/DarlingSelfAlertTests.cs +++ b/Darling/Darling.Tests/DarlingSelfAlertTests.cs @@ -78,6 +78,7 @@ private sealed class FakeSettings : IAlertEngineSettings public int DiskCriticalFreePercent { get; set; } = 3; public int DiskCriticalFreeGb { get; set; } = 2; public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int SelfDiskFreeWarnGb { get; set; } = 50; public int CollectionStaleMinutes { get; set; } = 30; public int CollectionFailureThreshold { get; set; } = 10; public int PvsThresholdPercent { get; set; } = 40; @@ -105,6 +106,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private sealed class FakeHistoryStore : IAlertHistoryStore @@ -824,11 +834,39 @@ this must not collapse to 0 on the not-pressure path. */ [Fact] public void IsDiskPressure_JustBelowThreshold_Pressure() { - /* 9.9% free trips it — the threshold is a real edge, not a wide band. */ - Assert.True(DarlingSelfAlertEvaluator.IsDiskPressure(99 * Gib, 1000 * Gib, out _, out var percentFree)); + /* 9.9% free trips it — the threshold is a real edge, not a wide band. The volume is small enough + (100 GiB) that the #3528 GB floor is far above the free space, so the percent is what decides. */ + Assert.True(DarlingSelfAlertEvaluator.IsDiskPressure( + (long)(9.9 * Gib), 100 * Gib, out _, out var percentFree)); Assert.Equal(9.9, percentFree, precision: 6); } + [Fact] + public void IsDiskPressure_BigVolumeAtLowPercent_IsNotPressure_TheGbFloorQualifies() + { + /* #3528's own example, scaled: 99 GiB free on a 1000 GiB store volume is 9.9% — below the percent + threshold — but 99 GiB of runway is nothing to page CRITICAL about, and it is ABOVE the shipped + 50 GB floor, so the composed shipped default stays quiet. Before #3528 this exact call fired. */ + Assert.False(DarlingSelfAlertEvaluator.IsDiskPressure(99 * Gib, 1000 * Gib, out _, out var percentFree)); + + /* Measured whenever the total is usable, firing or not (#1881). */ + Assert.Equal(9.9, percentFree, precision: 6); + + /* The floor only QUALIFIES: once free space is genuinely below it too, the same volume fires. */ + Assert.True(DarlingSelfAlertEvaluator.IsDiskPressure(45 * Gib, 1000 * Gib, out _, out _)); + } + + [Fact] + public void IsDiskPressure_FloorOfZero_RestoresThePercentOnlyCondition() + { + /* 0 removes the floor (the pvs_floor_gb reading), so the pre-#3528 percent-only behaviour is one + setting away — and the percent-only overload is that same condition, pinned equal here. */ + Assert.True(DarlingSelfAlertEvaluator.IsDiskPressure( + 99 * Gib, 1000 * Gib, DarlingSelfAlertEvaluator.DiskFreeWarnPercent, 0.0, out _, out _)); + Assert.True(DarlingSelfAlertEvaluator.IsDiskPressure( + 99 * Gib, 1000 * Gib, DarlingSelfAlertEvaluator.DiskFreeWarnPercent, out _, out _)); + } + [Fact] public void IsDiskPressure_PlentyFree_NotPressure() { @@ -892,6 +930,36 @@ public async Task DiskPressure_FiresOnce_ThenStaysQuietAtUnchangedLevel_ReFiresO Assert.Equal(2, h.Deliverer.Outcomes.Count); } + [Fact] + public async Task DiskPressure_ReadsTheGbFloorThroughTheSettingsSeam() + { + var h = new Harness(); + var e = h.Build(); + + /* 9% free on a 1 TiB store volume: the percent is breached, but ~92 GiB of runway sits above the + fake's 50 GB floor — the sweep stays quiet. This drives the SEAM (the store-backed knob the + sweep reads), not the constant the pure tests pin, and the fired threshold text below is what + proves which condition judged. */ + await e.ApplyDiskPressureAsync(92 * Gib, 1024 * Gib, null, Ct); + Assert.Empty(h.Deliverer.Outcomes); + + /* An operator setting the floor to 0 restores the percent-only condition on the next sweep — + read live through the by-reference seam, no rebuild. */ + h.Settings.SelfDiskFreeWarnGb = 0; + await e.ApplyDiskPressureAsync(92 * Gib, 1024 * Gib, null, Ct); + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("Store Disk Pressure", fired.MetricName); + /* With the floor off the threshold string names the percent alone... */ + Assert.Equal("10% free", fired.ThresholdValue); + + /* ...and with it on, a breach BELOW both gates fires and the string names both. */ + var h2 = new Harness(); + var e2 = h2.Build(); + await e2.ApplyDiskPressureAsync(40 * Gib, 1024 * Gib, null, Ct); + var both = Assert.Single(h2.Deliverer.Outcomes); + Assert.Equal("10% free and under 50 GB", both.ThresholdValue); + } + /* ---------------- custom-alert-rule health edge (#3304) ---------------- */ private static CustomAlertHealthReport HealthReport(int broken, int neverFiring) @@ -1061,12 +1129,38 @@ public async Task StaleMute_UnboundedRulePastTheAge_FiresOnce_WithTheCountAndThe private static readonly DateTime CertClock = new(2026, 7, 1, 12, 0, 0, DateTimeKind.Utc); + /// A SERVED certificate (the host's lifetime verdict was Usable): NotBefore a year back, the + /// given NotAfter. The expiry arms care only about NotAfter. private static DarlingSelfAlertEvaluator.WebTlsCertReport Cert( DateTime notAfterUtc, string subject = "CN=Darling Web", string thumbprint = "ABC123DEF456") => - new(Configured: true, NotAfterUtc: new DateTimeOffset(notAfterUtc, TimeSpan.Zero), Subject: subject, Thumbprint: thumbprint); + new( + Configured: true, + NotBeforeUtc: new DateTimeOffset(CertClock.AddDays(-365), TimeSpan.Zero), + NotAfterUtc: new DateTimeOffset(notAfterUtc, TimeSpan.Zero), + Subject: subject, + Thumbprint: thumbprint, + RefusedNotYetValid: false); + + /// A certificate the host REFUSED at load because its window had not opened (#3517): the host's + /// verdict rides the flag; NotAfter is far out, which is exactly what made it read as healthy before. + private static DarlingSelfAlertEvaluator.WebTlsCertReport NotYetValidCert( + DateTime notBeforeUtc, string thumbprint = "FUTURE0123") => + new( + Configured: true, + NotBeforeUtc: new DateTimeOffset(notBeforeUtc, TimeSpan.Zero), + NotAfterUtc: new DateTimeOffset(notBeforeUtc.AddDays(365), TimeSpan.Zero), + Subject: "CN=Darling Web (rotation)", + Thumbprint: thumbprint, + RefusedNotYetValid: true); private static readonly DarlingSelfAlertEvaluator.WebTlsCertReport NoWebTlsCert = - new(Configured: false, NotAfterUtc: default, Subject: string.Empty, Thumbprint: string.Empty); + new( + Configured: false, + NotBeforeUtc: default, + NotAfterUtc: default, + Subject: string.Empty, + Thumbprint: string.Empty, + RefusedNotYetValid: false); private static double WebTlsWarnDays => DarlingSelfAlertEvaluator.WebTlsCertWarnWindow.TotalDays; @@ -1203,14 +1297,25 @@ public void BuildWebTlsCertReport_MapsNullToUnconfigured_AndASnapshotToItsFields { var none = DarlingWorker.BuildWebTlsCertReport(null); Assert.False(none.Configured); + Assert.False(none.RefusedNotYetValid); + var fromUtc = new DateTimeOffset(2026, 1, 2, 3, 4, 5, TimeSpan.Zero); var whenUtc = new DateTimeOffset(2027, 1, 2, 3, 4, 5, TimeSpan.Zero); - var snap = new WebTlsCertificateState.Snapshot(whenUtc, "CN=x", "THUMB"); + var snap = new WebTlsCertificateState.Snapshot(fromUtc, whenUtc, "CN=x", "THUMB", RefusedNotYetValid: false); var report = DarlingWorker.BuildWebTlsCertReport(snap); Assert.True(report.Configured); + Assert.Equal(fromUtc, report.NotBeforeUtc); Assert.Equal(whenUtc, report.NotAfterUtc); Assert.Equal("CN=x", report.Subject); Assert.Equal("THUMB", report.Thumbprint); + Assert.False(report.RefusedNotYetValid); + + /* #3517: the host's not-yet-valid verdict crosses the seam untouched — the builder neither drops it nor + re-derives it from the dates (here NotBefore is in the past relative to nothing; the flag is the + host's word and that is what the evaluator fires on). */ + var refused = DarlingWorker.BuildWebTlsCertReport( + new WebTlsCertificateState.Snapshot(fromUtc, whenUtc, "CN=x", "THUMB", RefusedNotYetValid: true)); + Assert.True(refused.RefusedNotYetValid); } /// WebTlsCertificateState is the host→worker seam: Publish sets, Clear (the #3514 follow-up) @@ -1221,13 +1326,158 @@ public void WebTlsCertificateState_PublishThenClear_ReadsBackNull() var state = new WebTlsCertificateState(); Assert.Null(state.Read()); - state.Publish(new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero), "CN=x", "THUMB"); + state.Publish( + new DateTimeOffset(2029, 1, 1, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero), + "CN=x", "THUMB", refusedNotYetValid: false); Assert.NotNull(state.Read()); state.Clear(); Assert.Null(state.Read()); } + /* ---------------- web dashboard TLS certificate not yet valid (#3517) ---------------- */ + + /// + /// The #3517 gap: a certificate the host refused at load for a future NotBefore has a far-out + /// NotAfter, so on the expiry test alone it was the healthiest certificate imaginable while the LAN + /// dashboard sat loopback-only. The host's carried verdict fires the SAME family — same fleet key, same + /// metric, so an operator's mute rule for it still applies — at Critical, and the text says what happened + /// (not yet valid, until when, loopback-only), why (clock, or a future-dated certificate) and what to do + /// (fix it, then restart — the host will not re-decide on its own). + /// + [Fact] + public async Task WebTlsCert_RefusedNotYetValid_FiresCritical_UnderTheExpiryFamilyKey_NamingTheCauseAndTheRestart() + { + var h = new Harness { Now = CertClock }; + var e = h.Build(); + + await e.ApplyWebTlsCertificateAsync(NotYetValidCert(CertClock.AddDays(3)), Ct); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(DarlingSelfAlertEvaluator.WebTlsCertExpiryMetric, fired.MetricName); // the family, not a new metric + Assert.Equal("webtlscert", fired.ServerKey); // the family's fleet key + Assert.Equal(DarlingSelfAlertEvaluator.StoreServerLabel, fired.ServerName); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); // as unreachable as expired + Assert.Contains("NOT YET VALID", fired.ShortMessage); + Assert.Contains("loopback-only", fired.ShortMessage); + Assert.DoesNotContain("EXPIRED", fired.ShortMessage); + Assert.Contains($"{CertClock.AddDays(3):u}", fired.DetailText); // until when + Assert.Contains("LOOPBACK-ONLY", fired.DetailText); // what happened + Assert.Contains("clock", fired.DetailText); // why (likely) + Assert.Contains("future rotation", fired.DetailText); // why (the other one) + Assert.Contains("restart the service", fired.DetailText); // what to do + Assert.Contains("FUTURE0123", fired.DetailText); // ties to the host's own log line + Assert.Equal(0d, fired.NumericCurrentValue); // state-only, like every firing of this family + Assert.Empty(h.History.Records); + } + + /// The verdict is the host's, not the clock's: once NotBefore has passed the host is STILL + /// loopback-only (it decided at load and does not revisit), so the alert must keep standing rather than + /// resolve on the date — and its text switches to the fact that now matters, that a restart is needed. + [Fact] + public async Task WebTlsCert_RefusedNotYetValid_WindowOpensLater_StillFires_SaysTheHostDecidedAtLoad() + { + var h = new Harness { Now = CertClock }; + var e = h.Build(); + var notBefore = CertClock.AddHours(2); + + await e.ApplyWebTlsCertificateAsync(NotYetValidCert(notBefore), Ct); // fires on entry + Assert.Single(h.Deliverer.Outcomes); + + /* The window opened and the daily refire has elapsed; the host has NOT restarted, so the snapshot it + published at load is unchanged and still says refused. */ + h.Now = CertClock.Add(DarlingSelfAlertEvaluator.WebTlsCertRefire).AddMinutes(1); + await e.ApplyWebTlsCertificateAsync(NotYetValidCert(notBefore), Ct); + + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.Empty(h.History.Records); // no false resolution on the date + var restated = h.Deliverer.Outcomes[1]; + Assert.Equal(AlertSeverityLevel.Critical, restated.Severity); + Assert.Contains("after the service started", restated.DetailText); + Assert.Contains("until it is restarted", restated.DetailText); + } + + /// A served certificate (the host's verdict was Usable) with a future-looking but past-relative + /// NotBefore and a far-out NotAfter is healthy — the arm keys off the carried verdict alone, so a Usable + /// verdict with any NotBefore raises nothing. The #3514 silence contract, restated over the new field. + [Fact] + public async Task WebTlsCert_ServedAndFarOut_StaysSilent_WhateverNotBeforeSays() + { + var h = new Harness { Now = CertClock }; + + /* NotBefore AHEAD of the clock but the host said Usable (it is the host's clock that decides, and the + evaluator must not second-guess it from the date). */ + var servedDespiteDate = NotYetValidCert(CertClock.AddDays(3)) with { RefusedNotYetValid = false }; + await h.Build().ApplyWebTlsCertificateAsync(servedDespiteDate, Ct); + + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.History.Records); + } + + /// The transition the issue asked for: not-yet-valid → the host stops serving (Clear() on stop, + /// or the operator fixes the clock/certificate and the restart's stop half runs first) → the family's ONE + /// resolution, under the same metric pairing the triage endpoint and the history already know. + [Fact] + public async Task WebTlsCert_RefusedNotYetValid_ThenNoLongerServing_ResolvesUnderTheFamilyPairing() + { + var h = new Harness { Now = CertClock }; + var e = h.Build(); + + await e.ApplyWebTlsCertificateAsync(NotYetValidCert(CertClock.AddDays(3)), Ct); // fires + Assert.Single(h.Deliverer.Outcomes); + + await e.ApplyWebTlsCertificateAsync(NoWebTlsCert, Ct); // host Clear()ed the snapshot + + var resolution = Assert.Single(h.History.Records); + Assert.Equal(DarlingSelfAlertEvaluator.WebTlsCertRenewedMetric, resolution.MetricName); + + /* Once resolved, a fresh usable publish (the restart's start half, same process — a dashboard toggle) + is healthy and writes nothing more. */ + await e.ApplyWebTlsCertificateAsync(Cert(CertClock.AddDays(400)), Ct); + Assert.Single(h.Deliverer.Outcomes); + Assert.Single(h.History.Records); + } + + /// The back-to-back rebind: Stop and Start ran inside one supervisor tick, so the sweep never saw + /// the null and goes straight from the refused snapshot to a served one. That is the Configured=true + /// resolution arm, and its line has to be true of THIS transition too — "being served", not only "outside + /// the expiry window". + [Fact] + public async Task WebTlsCert_RefusedNotYetValid_ThenServedDirectly_ResolvesWithALineTrueOfBothArms() + { + var h = new Harness { Now = CertClock }; + var e = h.Build(); + + await e.ApplyWebTlsCertificateAsync(NotYetValidCert(CertClock.AddDays(3)), Ct); + Assert.Single(h.Deliverer.Outcomes); + + await e.ApplyWebTlsCertificateAsync(Cert(CertClock.AddDays(400)), Ct); // re-published usable, no null in between + + var resolution = Assert.Single(h.History.Records); + Assert.Equal(DarlingSelfAlertEvaluator.WebTlsCertRenewedMetric, resolution.MetricName); + Assert.Contains("being served", resolution.DetailText, StringComparison.Ordinal); + } + + /// Expired outranks not-yet-valid when a refused-at-start certificate is then outlived by the + /// process: a clock fix cannot cure an expired certificate, so that is the fact to lead with. + [Fact] + public async Task WebTlsCert_RefusedNotYetValid_ButNowAlsoExpired_ReadsAsExpired() + { + var h = new Harness { Now = CertClock }; + var e = h.Build(); + + /* NotBefore 400 days back, NotAfter 35 days back — a certificate that was not yet valid when this + (very long-lived) process started and has since expired outright. */ + var refusedThenLapsed = NotYetValidCert(CertClock.AddDays(-400)); // NotAfter = NotBefore + 365 = 35 days ago + await e.ApplyWebTlsCertificateAsync(refusedThenLapsed, Ct); + + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + Assert.Contains("EXPIRED", fired.ShortMessage); + Assert.DoesNotContain("NOT YET VALID", fired.ShortMessage); + } + [Fact] public async Task StaleMute_AFreshUnboundedRule_StaysSilent() { @@ -2495,6 +2745,122 @@ public async Task CompressionJobs_EvaluateWrapper_IsolatesAThrowingSeam_DoesNotP await e2.EvaluateCompressionJobsAsync(Stuck(1002), _ => throw new InvalidOperationException("boom"), Ct); } + /* ---------------- #3591: a crash-backoff row the scheduler recovers by itself ---------------- */ + + /// The 2.26.4+ shape of the -infinity row: the reader marks it SchedulerRetries with the version's sentence. + private static IReadOnlyList CrashBackoff(params long[] jobIds) => + jobIds.Select(id => new StuckCompressionJob( + id, "wait_stats", TimescaleSupport.NextStartNegativeInfinityCrashBackoffReason, SchedulerRetries: true)).ToList(); + + [Fact] + public async Task CompressionJobs_SchedulerRetries_FirstSight_NoRearm_NoPage_LoggedAtInformation() + { + /* Measured on a 2.28.1 rig: alter_job(next_start => now()) against a job in crash backoff RESETS the + backoff (the retry moved from crash+5:00 to re-arm+5:04, for the re-armed job and its un-re-armed + sibling) and overwrites the -infinity, so a re-arm here would be a later retry and two untrue + messages. First sight is remembered and written to the log; the scheduler gets one check cadence. */ + var h = new Harness(); + var e = h.Build(); + var rearm = new RearmRecorder(); + + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + + Assert.Empty(rearm.Calls); + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.History.Records); + var info = Assert.Single(h.Log.Entries, x => x.Level == Microsoft.Extensions.Logging.LogLevel.Information); + Assert.Contains("1001", info.Message, StringComparison.Ordinal); + Assert.Contains("crash backoff", info.Message, StringComparison.Ordinal); + Assert.Contains("not re-armed, not alerted", info.Message, StringComparison.Ordinal); + Assert.Contains("#3591", info.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task CompressionJobs_SchedulerRetries_StillThereAnHourOn_EscalatesOnce_NeverRearms() + { + /* The scheduler's own retry did not clear it within a check cadence — the jittered backoff fell past + the check, or the job crashed again and its backoff doubled. Either is a human's to read about in + the PostgreSQL log; neither is helped by alter_job. One Critical page, then the escalated state's + cooldown re-fires, and no re-arm at any point. */ + var h = new Harness(); + var e = h.Build(); + var rearm = new RearmRecorder(); + + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + h.Now = h.Now.AddHours(1); + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + + Assert.Empty(rearm.Calls); + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("Compression Job Stuck", fired.MetricName); + Assert.Equal(AlertSeverityLevel.Critical, fired.Severity); + Assert.Equal("compressjob:1001", fired.ServerKey); + Assert.Contains("still in crash backoff", fired.ShortMessage, StringComparison.Ordinal); + Assert.Contains("escalated", fired.ShortMessage, StringComparison.Ordinal); + Assert.DoesNotContain("re-armed", fired.ShortMessage, StringComparison.Ordinal); + + /* Escalated: an hour later, still there, still no re-arm; re-fires only on the cooldown. */ + h.Now = h.Now.AddHours(1); + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + Assert.Empty(rearm.Calls); + Assert.Equal(2, h.Deliverer.Outcomes.Count); + Assert.Contains("after escalation", h.Deliverer.Outcomes[1].ShortMessage, StringComparison.Ordinal); + } + + [Fact] + public async Task CompressionJobs_SchedulerRetries_ClearsBeforeTheSecondCheck_NoResolutionRow_StateDropped() + { + /* The expected path on the fleet: the scheduler re-ran the job inside the hour. Nothing was paged, so + nothing is "Recovered" — a lone resolution with no alert before it is the message shape #3575 + removed. The state is gone, so a later genuine sighting is a fresh first sight. */ + var h = new Harness(); + var e = h.Build(); + var rearm = new RearmRecorder(); + + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + h.Now = h.Now.AddHours(1); + await e.ApplyCompressionJobsStuckAsync(Stuck(), rearm.Delegate, Ct); + + Assert.Empty(rearm.Calls); + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(h.History.Records); + Assert.Contains(h.Log.Entries, x => x.Level == Microsoft.Extensions.Logging.LogLevel.Information + && x.Message.Contains("running on schedule again", StringComparison.Ordinal) + && x.Message.Contains("#3591", StringComparison.Ordinal)); + + /* Fresh first sight afterwards: deferred again, not escalated — the state was dropped. */ + h.Now = h.Now.AddHours(1); + await e.ApplyCompressionJobsStuckAsync(CrashBackoff(1001), rearm.Delegate, Ct); + Assert.Empty(h.Deliverer.Outcomes); + Assert.Empty(rearm.Calls); + } + + [Fact] + public async Task CompressionJobs_PreFixRow_AndSchedulerRetriesRow_InOnePass_OnlyTheOldOneIsRearmedAndPaged() + { + /* The two kinds side by side, so the flag and not the position decides: the pre-2.26.4 row keeps every + #1581 semantic (re-armed once, paged Critical); the crash-backoff row is deferred. */ + var h = new Harness(); + var e = h.Build(); + var rearm = new RearmRecorder(); + + var both = new List(CrashBackoff(1001)); + both.AddRange(Stuck(1002)); + await e.ApplyCompressionJobsStuckAsync(both, rearm.Delegate, Ct); + + Assert.Equal(1002L, Assert.Single(rearm.Calls)); + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal("compressjob:1002", fired.ServerKey); + Assert.Contains("auto-re-armed", fired.ShortMessage, StringComparison.Ordinal); + } + + [Fact] + public void CompressionJobs_TheOldRowShape_DefaultsToTheRearmPath() + { + /* The three-argument record every pre-#3591 caller and pin builds is the old semantics, by default. */ + Assert.False(new StuckCompressionJob(1L, "wait_stats", "next_start is -infinity — the scheduler will never run it again").SchedulerRetries); + } + /* ---------------- #991 Availability Groups: sync-behind decision (pure) ---------------- */ private static AgSyncJudgement Judge( @@ -3281,8 +3647,8 @@ public Task> GetRecentBlockedProcessReportsAsync(st Task.FromResult(null); public Task> GetRecentDeadlocksAsync(string serverKey, int hoursBack, CancellationToken cancellationToken = default) => Task.FromResult(new List()); - public Task> GetPoisonWaitDeltasAsync(string serverKey, double thresholdMs, CancellationToken cancellationToken = default) => - Task.FromResult(new List()); + public Task> GetPoisonWaitAccumulationAsync(string serverKey, int windowMinutes, CancellationToken cancellationToken = default) => + Task.FromResult(new List()); public Task> GetLongRunningQueriesAsync( string serverKey, int thresholdMinutes, int maxResults, bool excludeSpServerDiagnostics, bool excludeWaitFor, bool excludeBackups, bool excludeMiscWaits, bool excludeCdc, diff --git a/Darling/Darling.Tests/DarlingTrendEmptyTests.cs b/Darling/Darling.Tests/DarlingTrendEmptyTests.cs index e5d913b94..8e60ff4fe 100644 --- a/Darling/Darling.Tests/DarlingTrendEmptyTests.cs +++ b/Darling/Darling.Tests/DarlingTrendEmptyTests.cs @@ -83,9 +83,10 @@ public async Task AllThreeTrends_SeparateAQuietWindowFromANeverCollectedServer_A await SeedFileIoAsync(connection, ct, recent); await SeedQueryAsync(connection, ct, recent); + var memoryPayload = await DarlingMcpTrendTools.GetMemoryTrend(postgres, ServerName, 4); foreach (var payload in new[] { - await DarlingMcpTrendTools.GetMemoryTrend(postgres, ServerName, 4), + memoryPayload, await DarlingMcpTrendTools.GetFileIoTrend(postgres, ServerName, 4), await DarlingMcpTrendTools.GetQueryDurationTrend(postgres, ServerName, 4), }) @@ -95,6 +96,17 @@ await DarlingMcpTrendTools.GetQueryDurationTrend(postgres, ServerName, 4), Assert.True(root.GetProperty("trend").GetArrayLength() > 0); } + /* #3529, now the #3548 join's UNCOVERED arm (no memory_grant_stats rows seeded near these + points): total_granted_mb stays an explicit null with the envelope naming the real source — + never the literal 0.0 an agent read as "granted was 0 all window". The covered arms live in + DarlingMemoryTrendGrantJoinTests. */ + var memoryRoot = JsonDocument.Parse(memoryPayload).RootElement; + Assert.Contains("get_memory_grants", memoryRoot.GetProperty("granted_note").GetString(), StringComparison.Ordinal); + foreach (var point in memoryRoot.GetProperty("trend").EnumerateArray()) + { + Assert.Equal(JsonValueKind.Null, point.GetProperty("total_granted_mb").ValueKind); + } + bodySucceeded = true; } finally diff --git a/Darling/Darling.Tests/DarlingWebTlsTests.cs b/Darling/Darling.Tests/DarlingWebTlsTests.cs index b972db18f..96ce9e1fa 100644 --- a/Darling/Darling.Tests/DarlingWebTlsTests.cs +++ b/Darling/Darling.Tests/DarlingWebTlsTests.cs @@ -136,6 +136,34 @@ the log honest rather than starting a listener that lasts zero seconds. */ Assert.NotNull(DarlingWebTls.LifetimeRefusal(Now.AddDays(-10), Now, Now)); } + /// #3517: the gate's KIND is what the web host carries to the worker's self-alert (the host + /// decides once at load and stays loopback-only on it), so the three verdicts pin by kind as well as by + /// line, and the refusal string is non-null exactly when the kind is not Usable. + [Fact] + public void CheckLifetime_ThreeKinds_RefusalPresentExactlyWhenNotUsable() + { + var usable = DarlingWebTls.CheckLifetime(Now.AddDays(-10), Now.AddDays(200), Now); + Assert.Equal(DarlingWebTls.LifetimeStatus.Usable, usable.Status); + Assert.Null(usable.Refusal); + + var expired = DarlingWebTls.CheckLifetime(Now.AddDays(-400), Now.AddDays(-1), Now); + Assert.Equal(DarlingWebTls.LifetimeStatus.Expired, expired.Status); + Assert.Contains("expired", expired.Refusal, StringComparison.Ordinal); + + var early = DarlingWebTls.CheckLifetime(Now.AddDays(5), Now.AddDays(400), Now); + Assert.Equal(DarlingWebTls.LifetimeStatus.NotYetValid, early.Status); + Assert.Contains("not valid until", early.Refusal, StringComparison.Ordinal); + } + + [Fact] + public void CheckLifetime_WindowBothUnopenedAndClosed_ReadsAsExpired() + { + /* Expired is checked first, so a nonsense window (NotBefore past NotAfter, both behind or both ahead + in the wrong order) reads as the fact a restart cannot fix. */ + var nonsense = DarlingWebTls.CheckLifetime(Now.AddDays(10), Now.AddDays(-10), Now); + Assert.Equal(DarlingWebTls.LifetimeStatus.Expired, nonsense.Status); + } + /* ---- PURE: the advance expiry warning ---- */ [Fact] diff --git a/Darling/Darling.Tests/DatabaseFileGrowthReadTests.cs b/Darling/Darling.Tests/DatabaseFileGrowthReadTests.cs new file mode 100644 index 000000000..89937ccf1 --- /dev/null +++ b/Darling/Darling.Tests/DatabaseFileGrowthReadTests.cs @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3636: the alerting pass's file-growth read (#2349) carries the observation's identity. The read's growth +/// figure is a fact about two COLLECTIONS — the newest sample and the oldest inside the window — and the +/// database_size_stats collector lands one per hour, so the row reads byte-identical on every ~30 s alert +/// pass in between. Without the newest sample's collection_time on the row, the engine's rise arm re-fired +/// on every 5-minute cooldown against the same observation: up to twelve cards per growth event. #3579 gave the +/// forced-plan read its observed_at for the identical mechanism at a 5-minute cadence; this is the same +/// column on the sibling read. +/// +/// The ungated pins hold the statement's shape — the stamp is LAST, so the fourteen ordinals the reader +/// already binds do not move, and it is the CTE's own collection_time rather than a new source column. +/// The gated one seeds two hourly collections on a live store and reads them back through the real Npgsql path: +/// the growth is the difference, the window is the measured span, and the stamp is the newest collection's +/// clock to the tick, Kind Utc — the engine compares one file's stamps for equality, so a stamp that came back +/// shifted or truncated would either never match (cooldown-repeat returns) or match a different collection. +/// +/* Live-fixture tests share one Postgres store; the collection serializes them so cross-test row churn + cannot race another class's assertions. */ +[Collection("live-postgres")] +public sealed class DatabaseFileGrowthReadTests +{ + /// Distinctive fake id — a real server_id is a storage-name hash, never this. + private const int TestServerId = -363636; + private static readonly string TestServerKey = TestServerId.ToString(CultureInfo.InvariantCulture); + private const string TestServerName = "file-growth-read-e2e"; + + /* ---------------- ungated shape pins ---------------- */ + + /// + /// The observation stamp is the LAST column of the shipped read and is c.collection_time — the newest + /// sample's collector clock, which current_files already selected for the window-width arithmetic and + /// never projected. Last, because the reader binds ordinals 0–13 to the fourteen pre-#3636 columns and an + /// inserted column would silently shift every one of them onto its neighbour's type. Pinned by splitting the + /// final select list into its lines (one column per line; several carry COALESCE(a, b), so a comma + /// split would over-count) rather than by substring, so a column appended AFTER it would fail here too. + /// + [Fact] + public void TheObservationStamp_IsTheLastColumn_AndIsTheNewestSamplesCollectionTime() + { + var sql = DarlingAlertReadAdapter.DatabaseFileGrowthSql; + var selectStart = sql.LastIndexOf("SELECT", StringComparison.Ordinal); + var fromStart = sql.IndexOf("FROM current_files c", StringComparison.Ordinal); + Assert.True(selectStart >= 0 && fromStart > selectStart, "the final select list was not found"); + + var columns = sql[(selectStart + "SELECT".Length)..fromStart] + .Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal(15, columns.Length); + Assert.Equal("c.max_size_mb,", columns[13]); + Assert.Equal("c.collection_time AS observed_at", columns[14]); + + /* Not a new source column: the CTE selected collection_time before #3636 (the window width needs it). */ + var cte = sql[..sql.IndexOf("baseline AS", StringComparison.Ordinal)]; + Assert.Contains("file_type_desc, collection_time,", cte, StringComparison.Ordinal); + } + + /// The read stays a raw-table, parameter-bound-clock statement like every other alert feed: no bare + /// now() (timestamptz against the naive-UTC columns) and never Lite's v_ view. + [Fact] + public void TheRead_TargetsTheRawTable_AndBindsItsClock() + { + var sql = DarlingAlertReadAdapter.DatabaseFileGrowthSql; + Assert.DoesNotContain("now(", sql.ToLowerInvariant()); + Assert.DoesNotContain("FROM v_", sql, StringComparison.Ordinal); + Assert.Contains("FROM database_size_stats", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + } + + /* ---------------- gated: the round trip ---------------- */ + + /// + /// Two hourly collections of one file plus a single-sample neighbour, read back through the real adapter. + /// The stamp is the NEWEST collection's collection_time, tick-equal to what was seeded (the store + /// holds naive UTC; the adapter names the Kind and never shifts the value), a re-read between collections + /// returns the same stamp, and a third collection moves it. + /// + [Fact] + public async Task TheStamp_IsTheNewestCollectionsClock_ToTheTick_AndMovesWithTheNextCollection() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live file-growth read test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteTestRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var adapter = new DarlingAlertReadAdapter(postgres); + + var bodySucceeded = false; + try + { + /* Floored to whole microseconds (#3579's lesson): PostgreSQL timestamp is microsecond-resolution and + .NET ticks are 100 ns, so a raw UtcNow does not survive the round trip and the tick-equality below + would fail on any clock that is not itself microsecond-aligned. Kind Unspecified: naive-UTC storage. */ + var rawNow = DateTime.UtcNow; + var utcNow = DateTime.SpecifyKind(new DateTime(rawNow.Ticks - (rawNow.Ticks % 10)), DateTimeKind.Unspecified); + var collection0 = utcNow.AddMinutes(-70); + var collection1 = utcNow.AddMinutes(-10); + + /* tempdev: 100 GB at the top of the hour, 120 GB an hour later. templog: one sample only. */ + await SeedFileAsync(connection, ct, 1L, collection0, "tempdb", "tempdev", 102_400m); + await SeedFileAsync(connection, ct, 2L, collection1, "tempdb", "tempdev", 122_880m); + await SeedFileAsync(connection, ct, 2L, collection1, "tempdb", "templog", 4_096m); + + var files = await adapter.GetDatabaseFileGrowthAsync(TestServerKey, lookbackMinutes: 120, ct); + Assert.Equal(2, files.Count); + + var tempdev = Assert.Single(files, f => f.FileName == "tempdev"); + Assert.Equal(122_880d, tempdev.TotalSizeMb, precision: 3); + Assert.Equal(20_480d, tempdev.GrowthMb, precision: 3); + Assert.Equal(60d, tempdev.GrowthWindowMinutes, precision: 3); + /* #3636: the observation's identity is the newest sample's collection_time, to the tick, Kind Utc. */ + Assert.Equal((DateTime?)collection1, tempdev.ObservedAtUtc); + Assert.Equal(DateTimeKind.Utc, tempdev.ObservedAtUtc!.Value.Kind); + + /* The single-sample neighbour: no rise observed, and its own stamp all the same. */ + var templog = Assert.Single(files, f => f.FileName == "templog"); + Assert.Equal(0d, templog.GrowthMb, precision: 3); + Assert.Equal(0d, templog.GrowthWindowMinutes, precision: 3); + Assert.Equal((DateTime?)collection1, templog.ObservedAtUtc); + + /* Re-reading between collections is the same observation: same row, same stamp. This is the read + the engine used to fire on twelve times; the stamp is what lets it recognise the repeat. */ + var reread = await adapter.GetDatabaseFileGrowthAsync(TestServerKey, lookbackMinutes: 120, ct); + Assert.Equal(tempdev.ObservedAtUtc, Assert.Single(reread, f => f.FileName == "tempdev").ObservedAtUtc); + + /* The next collection lands: the stamp moves with it. */ + var collection2 = utcNow.AddMinutes(-2); + await SeedFileAsync(connection, ct, 3L, collection2, "tempdb", "tempdev", 122_880m); + var after = await adapter.GetDatabaseFileGrowthAsync(TestServerKey, lookbackMinutes: 120, ct); + Assert.Equal((DateTime?)collection2, Assert.Single(after, f => f.FileName == "tempdev").ObservedAtUtc); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteTestRowsAsync(cleanup, cleanupCt)); + } + } + + private static async Task SeedFileAsync( + NpgsqlConnection connection, CancellationToken ct, + long collectionId, DateTime collectionTime, string databaseName, string fileName, decimal totalSizeMb) + { + using var command = new NpgsqlCommand(@" +INSERT INTO database_size_stats + (collection_id, collection_time, server_id, server_name, database_name, database_id, + file_id, file_type_desc, file_name, physical_name, total_size_mb, used_size_mb, + volume_mount_point, volume_total_mb, volume_free_mb, auto_growth_mb, is_percent_growth, growth_pct, max_size_mb) +VALUES ($1, $2, $3, $4, $5, 2, $6, $7, $8, $9, $10, $11, 'D:\', 4096000, 3000000, 1024, false, NULL, -1)", connection); + command.Parameters.AddWithValue(collectionId); + command.Parameters.AddWithValue(collectionTime); + command.Parameters.AddWithValue(TestServerId); + command.Parameters.AddWithValue(TestServerName); + command.Parameters.AddWithValue(databaseName); + command.Parameters.AddWithValue(fileName == "tempdev" ? 1 : 2); + command.Parameters.AddWithValue(fileName == "tempdev" ? "ROWS" : "LOG"); + command.Parameters.AddWithValue(fileName); + command.Parameters.AddWithValue(@"D:\data\" + fileName); + command.Parameters.AddWithValue(totalSizeMb); + command.Parameters.AddWithValue(totalSizeMb * 0.8m); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand($"DELETE FROM database_size_stats WHERE server_id = {TestServerId}", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/DeadlockRateBandRungTests.cs b/Darling/Darling.Tests/DeadlockRateBandRungTests.cs index 850842fb5..6c9f639c7 100644 --- a/Darling/Darling.Tests/DeadlockRateBandRungTests.cs +++ b/Darling/Darling.Tests/DeadlockRateBandRungTests.cs @@ -287,6 +287,11 @@ public void EverySurfaceNeedingTheShippedDefault_NamesTheConstant() /// /// Comments and strings are stripped with the shared walker first, per the repo pin about /// hand-rolled maskers giving different wrong answers. + /// + /// Widened by #3539 A3/A8d to the two denominators that landed beside this rung's: the + /// blocking window (a bundle omitting it bands every blocking server Warning-by-count and never + /// Critical) and the collector count (a bundle omitting it grades a failing count presence-flat again). + /// Same census, four members. /// [Fact] public void EveryProductionMetricBundleDeclaresTheWindowAndTheTiers() @@ -303,7 +308,9 @@ public void EveryProductionMetricBundleDeclaresTheWindowAndTheTiers() found++; if (!AssignsMember(initializer, "DeadlockWindow") - || !AssignsMember(initializer, "DeadlockRateThresholds")) + || !AssignsMember(initializer, "DeadlockRateThresholds") + || !AssignsMember(initializer, "BlockingWindow") + || !AssignsMember(initializer, "CollectorCount")) { offenders.Add(System.IO.Path.GetFileName(file)); } @@ -319,8 +326,9 @@ count below three means the regex stopped matching and not that the code got bet Assert.True( offenders.Count == 0, - "these production metric bundles carry a deadlock count with no window or no tiers beside it, " - + "so they band a bare count or band on the shipped pair: " + "these production metric bundles carry a count with no denominator beside it (deadlock window, " + + "deadlock tiers, blocking window or collector count), so they band a bare count, band on the " + + "shipped pair, or grade presence-flat: " + string.Join(", ", offenders.Distinct().OrderBy(f => f, StringComparer.Ordinal))); } diff --git a/Darling/Darling.Tests/DeadlockRateBandTests.cs b/Darling/Darling.Tests/DeadlockRateBandTests.cs index 0c93b9f29..2cbed0f9b 100644 --- a/Darling/Darling.Tests/DeadlockRateBandTests.cs +++ b/Darling/Darling.Tests/DeadlockRateBandTests.cs @@ -237,23 +237,20 @@ public void TheMinimumWindowIsTheSmallestWindowAnySurfaceCanAskFor() /* ─────────────────────── the null (no-source) arm ─────────────────────── */ /// - /// A PostgreSQL target has no SQL-Server deadlock reading, so it bands off none (#3272/#3017) — and that - /// survives the rate change at every window, including the unrateable ones where a COUNT above zero - /// reads Warning. The null arm is tested first in the method for exactly this reason: an absent source - /// must not be reachable by the arm that exists for an absent denominator. + /// A caller with no deadlock source bands off none (#3272) — and that survives the rate change at every + /// window, including the unrateable ones where a COUNT above zero reads Warning. The null arm is tested + /// first in the method for exactly this reason: an absent source must not be reachable by the arm that + /// exists for an absent denominator. Since #3539 neither engine's CARD takes this arm (a PostgreSQL + /// target's count is its own counter difference); the arm stays for the daily classifier's empty cells + /// and for any caller that genuinely reads nothing. /// [Fact] - public void NoDeadlockSourceForTheEngine_StaysUnknown() + public void NoDeadlockSource_StaysUnknown() { foreach (var hours in new[] { 0, 1, 24, 168 }) { Assert.Equal(HealthSeverity.Unknown, Band(null, TimeSpan.FromHours(hours))); } - - Assert.Equal( - HealthSeverity.Unknown, - ServerHealthClassifier.DeadlockSeverity( - ServerMetricSources.DmvSourced(0, isPostgres: true), Hour, DeadlockRateThresholds.Default)); } /* ─────────────────────── the tiers are settable ─────────────────────── */ diff --git a/Darling/Darling.Tests/DeltaFamilyIntervalColumnsRungTests.cs b/Darling/Darling.Tests/DeltaFamilyIntervalColumnsRungTests.cs new file mode 100644 index 000000000..9966215c8 --- /dev/null +++ b/Darling/Darling.Tests/DeltaFamilyIntervalColumnsRungTests.cs @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Linq; +using System.Reflection; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// V127 / #3540: sample_interval_seconds on the four delta families that persisted their deltas NAKED — +/// wait_stats, file_io_stats, latch_stats, spinlock_stats. The measurement-layer +/// keystone. +/// +/// The shared delta calculator reports (delta 0, interval 0) when no delta is knowable and (0, n) when an +/// interval was genuinely idle; the interval is the ONLY thing that tells those apart, and these four +/// collectors discarded it at the write. So a restart's fabricated zero survived as a measured one, and every +/// per-second reader LAG-divided it into a confident 0.00 ms/sec at exactly the moments it was unknowable. +/// perfmon_stats and query_stats carried the column from the start; this rung gives the other four the same +/// column in the same type. +/// +/// The "I am the top rung" claims have moved ON to +/// (V128), the same handoff this file received from (V126) and +/// that file received from (V125). What stays here is +/// everything true of this rung wherever it sits in the ladder; what left is every claim that was really +/// about being NEWEST — keeping a copy of those would assert this rung is still the top, which is how the +/// NEXT rung's build goes red. +/// +public sealed class DeltaFamilyIntervalColumnsRungTests +{ + private const int RungVersion = 127; + + private const int PreviousVersion = 126; + + /// This rung's sentinel ordinal in the viewer probe. No longer the last argument — V128 + /// appended its own — so this is a position within the signature rather than its end. + private const int ProbeOrdinal = 102; + + private const string IntervalColumn = "sample_interval_seconds"; + + private static readonly string[] Tables = { "wait_stats", "file_io_stats", "latch_stats", "spinlock_stats" }; + + /* ---- the rung ------------------------------------------------------------------------------------ */ + + [Fact] + public void TheRungIsRegisteredInADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + Assert.Equal( + "delta-family-interval-columns", + PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name); + + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + + /* Not `RungVersion == SchemaVersion` any more: that asserted this rung is the newest, which + stopped being true when V128 landed. The invariant that outlives the handoff is that the + LADDER's top and the declared version agree, which the two lines above already say. */ + Assert.True(RungVersion < StorageVersion.SchemaVersion); + + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + } + + /// + /// The rung adds ONE nullable, default-less integer column to each of the four tables, schema-qualified, + /// idempotent, and refreshes each table's SELECT * passthrough view — and does nothing else. + /// + /// integer is pinned against the type perfmon_stats and query_stats already use through the + /// generator, not as a literal alone, so the six interval columns cannot drift apart and + /// NULLIF(sample_interval_seconds, 0) means the same thing on every one. No DEFAULT and no backfill: + /// a historical row never recorded its interval, so NULL is the honest value and a backfilled 0 would + /// stamp all of history as "unknowable" and blank every rate chart for 30 days. + /// + [Fact] + public void TheRungAddsTheNullableIntegerToAllFour_SchemaQualified_AndRefreshesTheirViews() + { + /* LF-normalised: the repo checks out CRLF on Windows (.gitattributes eol=crlf) and a verbatim string + carries the file's line endings, so the multi-line anchors below are written against LF. */ + var rung = PgMigrations.Scripts.Single(s => s.Version == RungVersion).Sql.Replace("\r\n", "\n", StringComparison.Ordinal); + + var perfmonType = PgSchemaGenerator.TypeFor( + PerfmonStatsCollector.Instance.PayloadColumns.Single(c => c.Name == IntervalColumn)); + Assert.Equal("integer", perfmonType); + + foreach (var table in Tables) + { + /* Schema-qualified: the migrate session's search_path puts collect first, so a bare name would + work today and stop working the day that changes — and CONTRIBUTING makes it the rule. */ + Assert.Equal(1, CountOf(rung, $"ALTER TABLE collect.{table}\n")); + Assert.DoesNotContain($"ALTER TABLE {table}\n", rung, StringComparison.Ordinal); + + Assert.Contains( + $"ALTER TABLE collect.{table}\n ADD COLUMN IF NOT EXISTS {IntervalColumn} {perfmonType};", + rung, StringComparison.Ordinal); + + /* Postgres freezes a view's SELECT * at CREATE (V14, V80, V81): without this line the passthrough + every reader uses would never show the column. */ + Assert.Equal(1, CountOf(rung, $"CREATE OR REPLACE VIEW collect.v_{table} AS SELECT * FROM collect.{table};")); + } + + Assert.Equal(4, CountOf(rung, "ADD COLUMN IF NOT EXISTS")); + /* Counted on the statement prefix, not the bare phrase — the rung's own SQL comment names + CREATE OR REPLACE VIEW in prose. */ + Assert.Equal(4, CountOf(rung, "CREATE OR REPLACE VIEW collect.v_")); + + /* Nullable, no default, no backfill, no CHECK, no GRANT, and no touch of the wait_stats_baseline + continuous aggregate — that change is the documented follow-up (a new aggregate under a new name, + the #2007 retirement shape), because a CAGG cannot change its query in place and a rebuild forfeits + baseline history raw can no longer refill. */ + Assert.DoesNotContain("DEFAULT", rung, StringComparison.Ordinal); + Assert.DoesNotContain("NOT NULL", rung, StringComparison.Ordinal); + Assert.DoesNotContain("UPDATE ", rung, StringComparison.Ordinal); + Assert.DoesNotContain("CHECK", rung, StringComparison.Ordinal); + Assert.DoesNotContain("GRANT", rung, StringComparison.Ordinal); + Assert.DoesNotContain("MATERIALIZED", rung, StringComparison.Ordinal); + Assert.DoesNotContain("wait_stats_baseline", rung, StringComparison.Ordinal); + } + + /// + /// A FRESH store gets the column from the generated CREATE TABLE (the collector definitions carry it now), + /// as the trailing column in the same type the rung's ALTER adds — so fresh-through-V127 and + /// upgraded-to-V127 stores are shaped identically and the rung's ALTERs no-op on the former. + /// + [Fact] + public void TheGeneratedCreateTable_CarriesTheColumnLast_OnAllFour() + { + foreach (var table in Tables) + { + var definition = CollectorCatalog.Find(table); + Assert.NotNull(definition); + + Assert.Equal(IntervalColumn, definition!.PayloadColumns[^1].Name); + Assert.Equal(CollectorColumnType.Integer, definition.PayloadColumns[^1].Type); + + var ddl = PgSchemaGenerator.CreateTable(definition); + Assert.EndsWith($" {IntervalColumn} integer\n);", ddl, StringComparison.Ordinal); + } + } + + /* ---- the probe (three sites) ---------------------------------------------------------------------- */ + + /// + /// The viewer probe's three sites carry this rung's sentinel, and a store that stopped here maps to it. + /// + /// The probe asks the question, the caller reads the answer, the map has the parameter — three + /// sites, and a sentinel present at only some of them shifts every LATER ordinal onto the wrong column. + /// The top-arm claims (last argument, returns the build's version) moved to + /// with V128. + /// + [Fact] + public void TheProbeCarriesThisRungsSentinel_AndAFullyMigratedStoreMapsToTheLaddersTop() + { + Assert.Contains( + $"table_name = 'wait_stats'\n AND column_name = '{IntervalColumn}'", + ViewerDataService.StoreSchemaProbeSql.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal); + + var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal); + Assert.Contains("hasDeltaFamilyIntervalColumns", viewer, StringComparison.Ordinal); + + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!; + var arity = method.GetParameters().Length; + + /* A position within the signature, not its end: `ProbeOrdinal == arity - 1` asserted this rung is + the NEWEST sentinel, which stopped being true the moment V128 appended its own. Strictly-less is + the form every other non-top rung's test here uses. */ + Assert.True(ProbeOrdinal < arity - 1); + + /* Every sentinel true = a fully-migrated store, which must map to exactly this version. Built by + reflection so the arity tracks the signature. */ + var all = Enumerable.Repeat((object)true, arity).ToArray(); + Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!); + + /* This rung's own arm answers for a store that stopped here. Expressed as "false above" rather than + as one named ordinal, so a rung landing on top of this one does not quietly turn this case into a + test of that rung. */ + var atThisRung = Enumerable.Range(0, arity).Select(i => (object)(i <= ProbeOrdinal)).ToArray(); + Assert.Equal(RungVersion, (int)method.Invoke(null, atThisRung)!); + + /* One rung behind: the same store WITHOUT this rung's sentinel reports the previous rung. */ + var behind = (object[])atThisRung.Clone(); + behind[ProbeOrdinal] = false; + Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!); + + /* And in the source, the arm sits ABOVE the previous rung's — newest-first is the whole contract of + that method. It returns this rung's own literal now, not the build's version: the "returns + StorageVersion.SchemaVersion" half of the top-arm claim moved to V128's test with the top. */ + var thisArm = viewer.IndexOf("if (hasDeltaFamilyIntervalColumns)", StringComparison.Ordinal); + var previousArm = viewer.IndexOf("if (hasSelfDiskWarnGbFloor)", StringComparison.Ordinal); + Assert.True(thisArm >= 0, "the viewer has no V127 sentinel arm — a store that stopped here would map to 126"); + Assert.True(previousArm >= 0, "the previous rung's arm is gone, so this pin is comparing against nothing"); + Assert.True(thisArm < previousArm, "the V127 arm sits below the previous rung's, so a V127 store maps one rung low"); + Assert.Contains( + "return " + RungVersion.ToString(CultureInfo.InvariantCulture) + ";", + viewer[thisArm..], StringComparison.Ordinal); + } + + /* ---- the writers ---------------------------------------------------------------------------------- */ + + /// + /// The four collectors now WRITE the interval — through CalculateDeltaWithInterval, never the bare + /// CalculateDelta — as the minimum over each row's delta groups. The column without the writer would + /// be a NULL forever; the writer taking one headline group's interval would let an independently reset + /// sibling counter's 0 read as idle over a real interval. + /// + [Fact] + public void TheFourCollectors_WriteTheIntervalAsTheMinimumOverTheirDeltaGroups() + { + foreach (var (file, groups) in new[] + { + ("WaitStatsCollector.cs", 3), + ("FileIoStatsCollector.cs", 8), + ("LatchStatsCollector.cs", 3), + ("SpinlockStatsCollector.cs", 4), + }) + { + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Collectors", file); + + Assert.Equal(groups, CountOf(source, "context.Deltas.CalculateDeltaWithInterval(")); + Assert.DoesNotContain("context.Deltas.CalculateDelta(", source, StringComparison.Ordinal); + Assert.Contains("var sampleIntervalSeconds = Math.Min(", source, StringComparison.Ordinal); + Assert.Contains(".Value(sampleIntervalSeconds);", source, StringComparison.Ordinal); + } + } + + /// + /// The calculator's own doc claim is TRUE again. It said "every consumer already maps 0 to NULL via + /// NULLIF(sample_interval_seconds, 0)" while four of six families discarded the interval at the write; + /// the sentence now names which families this rung dressed and when, so the history of how the claim + /// went false and came back is in the file that made it. (The "and which still do not" half of this + /// pin moved to when V128 emptied that list.) + /// + [Fact] + public void TheCalculatorDoc_NamesTheFamiliesThisRungDressed() + { + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Collectors", "CollectorDeltaCalculator.cs"); + + Assert.DoesNotContain("every consumer already maps 0 to NULL", source, StringComparison.Ordinal); + Assert.Contains("wait_stats, file_io_stats,", source, StringComparison.Ordinal); + Assert.Contains("latch_stats and spinlock_stats since Darling V127 / Lite v60, #3540", source, StringComparison.Ordinal); + Assert.Contains("sample_interval_seconds IS DISTINCT FROM 0", source, StringComparison.Ordinal); + } + + private static int CountOf(string haystack, string needle) + { + var count = 0; + for (var at = haystack.IndexOf(needle, StringComparison.Ordinal); + at >= 0; + at = haystack.IndexOf(needle, at + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Darling/Darling.Tests/DeltaFamilyIntervalCompletionLivePostgresTests.cs b/Darling/Darling.Tests/DeltaFamilyIntervalCompletionLivePostgresTests.cs new file mode 100644 index 000000000..2f05ae870 --- /dev/null +++ b/Darling/Darling.Tests/DeltaFamilyIntervalCompletionLivePostgresTests.cs @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3540 (V128), the read half of the completion, proven against LIVE Postgres (gated on +/// DARLING_TEST_PG) through the real readers: a collection every row of which the collector stored with +/// sample_interval_seconds = 0 — the calculator's "no delta knowable" marker, in practice a restart — +/// is NOT a point on the procedure duration trend or the per-statement PostgreSQL trend. It used to be a +/// confident 0.00 ms/sec and 0.00 calls/sec at exactly the moment nothing was knowable. +/// +/// Three states per collection, read distinctly: a MEASURED interval (MAX over the collection's rows) +/// divides the summed deltas and wins over the LAG derivation; the marker (every row 0) yields no point; and +/// NULL — every row collected before V128 — falls back to the LAG over collection_time those reads always +/// used. The procedure history grid shows the stored interval itself, the marker's 0 included, because a +/// displayed interval is not a rate. Lite's twin of these claims runs in-process on DuckDB +/// (DeltaFamilyUnknowableRowReadTests). +/// +/// Every expected value below was worked by hand from the rows and reproduced against PG18 + +/// TimescaleDB 2.28.1 before this was written. +/// +[Collection("live-postgres")] +public sealed class DeltaFamilyIntervalCompletionLivePostgresTests +{ + private const int ServerId = -128128; + private const string ServerName = "delta-interval-completion-e2e"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + /// + /// The procedure duration trend, both copies (the viewer's read and the MCP's SQL, which the pin in + /// DarlingMcpTrendToolsTests proves are one string): t1/t2 pre-V128 (NULL) — t1 no prior, no rate (the + /// viewer drops it, the MCP keeps it unrated); t2 the LAG's 300 s. t3 a restart — every row 0 — no rate + /// either. t4 a steady pass with a readmitted plan (its row 0) + /// beside a measured 120 s row: MAX 120 wins over the LAG's 300, and the readmitted plan adds 0. + /// + [Fact] + public async Task ProcedureDurationTrend_DropsTheUnknowableCollection_PrefersTheStoredInterval_AgainstDevPostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live procedure-trend test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var viewer = new ViewerDataService(cs!); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + var t1 = Naive(TruncateToSeconds(DateTime.UtcNow.AddHours(-2))); + var t2 = t1.AddMinutes(5); + var t3 = t2.AddMinutes(5); + var t4 = t3.AddMinutes(5); + + await ProcedureAsync(connection, t1, "usp_A", deltaExecutions: 5, deltaElapsedUs: 100_000, interval: null, ct); + await ProcedureAsync(connection, t2, "usp_A", 30, 600_000, null, ct); + await ProcedureAsync(connection, t3, "usp_A", 0, 0, 0, ct); + await ProcedureAsync(connection, t4, "usp_A", 24, 1_200_000, 120, ct); + await ProcedureAsync(connection, t4, "usp_New", 0, 0, 0, ct); + + var points = await viewer.GetProcedureDurationTrendAsync(ServerId, t1.AddMinutes(-1), t4.AddMinutes(1), cancellationToken: ct); + Assert.Equal(new[] { t2, t4 }, points.Select(p => p.CollectionTime).ToArray()); + Assert.Equal(2.0, points[0].Value, precision: 6); /* 600 ms / LAG 300 s */ + Assert.Equal(0, points[0].ExecutionCount); /* 30 / 300 = 0.1 executions/sec, truncated to long as always */ + Assert.Equal(10.0, points[1].Value, precision: 6); /* 1200 ms / STORED 120 s, not the LAG's 4.0 */ + + /* The MCP copy, run as the tool would run it on the raw tier. */ + await using (var command = postgres.CreateCommand(DarlingTrendReader.ProcedureDurationTrendSql)) + { + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(t1.AddMinutes(-1)); + command.Parameters.AddWithValue(t4.AddMinutes(1)); + var mcp = new List<(DateTime At, double? Rate, double? Executions)>(); + await using var reader = await command.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + mcp.Add((reader.GetDateTime(0), + reader.IsDBNull(1) ? null : Convert.ToDouble(reader.GetValue(1)), + reader.IsDBNull(2) ? null : Convert.ToDouble(reader.GetValue(2)))); + } + + /* The SQL returns all four collections; t1 and t3 carry NULL rates — the viewer's chart reader drops + those (above), the MCP reader keeps them as unrated points (#3541 A12). */ + Assert.Equal(new[] { t1, t2, t3, t4 }, mcp.Select(m => m.At).ToArray()); + Assert.Null(mcp[0].Rate); + Assert.Equal(2.0, mcp[1].Rate!.Value, precision: 6); + Assert.Null(mcp[2].Rate); + Assert.Null(mcp[2].Executions); + Assert.Equal(10.0, mcp[3].Rate!.Value, precision: 6); + Assert.Equal(0.2, mcp[3].Executions!.Value, precision: 6); + } + + /* The history grid: the stored interval as stored, the marker's 0 included; LAG only for NULL. */ + var history = await viewer.GetProcedureStatsHistoryAsync(ServerId, "AppDb", "dbo", "usp_A", t1.AddMinutes(-1), t4.AddMinutes(1), ct); + Assert.Equal(new[] { t1, t2, t3, t4 }, history.Select(h => h.CollectionTime).ToArray()); + Assert.Null(history[0].SampleIntervalSeconds); + Assert.Equal(300, history[1].SampleIntervalSeconds); + Assert.Equal(0, history[2].SampleIntervalSeconds); + Assert.Equal(120, history[3].SampleIntervalSeconds); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The per-statement PostgreSQL trend (): the same + /// four collections for one queryid across two (dbid, userid, toplevel) entries. t3 is a + /// pg_stat_statements_reset() — both entries' rows store 0 — and is absent; at t4 one entry is a + /// first sighting (0) beside the other's measured 120 s, so MAX is 120 and calls_per_second is the measured + /// entry's 24 calls over 120 s, not over the LAG's 300. + /// + [Fact] + public async Task PgQueryDurationTrend_DropsTheUnknowableSnapshot_PrefersTheStoredInterval_AgainstDevPostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live pg-statement-trend test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + var t1 = Naive(TruncateToSeconds(DateTime.UtcNow.AddHours(-2))); + var t2 = t1.AddMinutes(5); + var t3 = t2.AddMinutes(5); + var t4 = t3.AddMinutes(5); + const long QueryId = 4242; + + await PgStatementAsync(connection, t1, QueryId, userId: 10, deltaCalls: 5, deltaMs: 100, interval: null, ct); + await PgStatementAsync(connection, t2, QueryId, 10, 30, 600, null, ct); + await PgStatementAsync(connection, t3, QueryId, 10, 0, 0, 0, ct); + await PgStatementAsync(connection, t3, QueryId, 11, 0, 0, 0, ct); + await PgStatementAsync(connection, t4, QueryId, 10, 24, 1200, 120, ct); + await PgStatementAsync(connection, t4, QueryId, 11, 0, 0, 0, ct); + + var points = await DarlingPgTrendReader.GetQueryDurationTrendAsync(postgres, ServerId, QueryId, t1.AddMinutes(-1), t4.AddMinutes(1), ct); + + Assert.Equal(new[] { t2, t4 }, points.Select(p => p.CollectionTimeUtc).ToArray()); + Assert.Equal(30, points[0].Calls); + Assert.Equal(0.1, points[0].CallsPerSecond, precision: 6); /* 30 / LAG 300 s */ + Assert.Equal(20.0, points[0].MeanExecMs, precision: 6); /* 600 / 30 */ + Assert.Equal(24, points[1].Calls); + Assert.Equal(0.2, points[1].CallsPerSecond, precision: 6); /* 24 / STORED 120 s, not the LAG's 0.08 */ + Assert.Equal(50.0, points[1].MeanExecMs, precision: 6); /* 1200 / 24 */ + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /* ---- seeding ---------------------------------------------------------------------------------------- */ + + private static DateTime Naive(DateTime t) => DateTime.SpecifyKind(t, DateTimeKind.Unspecified); + + private static DateTime TruncateToSeconds(DateTime value) => + new(value.Ticks - (value.Ticks % TimeSpan.TicksPerSecond), value.Kind); + + private static async Task ProcedureAsync(NpgsqlConnection connection, DateTime t, string objectName, long deltaExecutions, long deltaElapsedUs, int? interval, CancellationToken ct) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO procedure_stats (collection_id, collection_time, server_id, server_name, database_name, schema_name, object_name, object_type, " + + "execution_count, total_worker_time, total_elapsed_time, total_logical_reads, total_physical_reads, total_logical_writes, " + + "delta_execution_count, delta_worker_time, delta_elapsed_time, sample_interval_seconds) " + + "VALUES (1, $1, $2, $3, 'AppDb', 'dbo', $4, 'PROCEDURE', 0, 0, 0, 0, 0, 0, $5, 0, $6, $7)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(ServerId); + cmd.Parameters.AddWithValue(ServerName); + cmd.Parameters.AddWithValue(objectName); + cmd.Parameters.AddWithValue(deltaExecutions); + cmd.Parameters.AddWithValue(deltaElapsedUs); + cmd.Parameters.AddWithValue(interval.HasValue ? interval.Value : DBNull.Value); + await cmd.ExecuteNonQueryAsync(ct); + } + + private static async Task PgStatementAsync(NpgsqlConnection connection, DateTime t, long queryId, long userId, long deltaCalls, long deltaMs, int? interval, CancellationToken ct) + { + using var cmd = new NpgsqlCommand( + "INSERT INTO pg_statement_stats (collection_id, collection_time, server_id, server_name, queryid, database_id, user_id, toplevel, " + + "calls, total_exec_time_ms, rows_returned, delta_calls, delta_total_exec_time_ms, delta_rows, sample_interval_seconds) " + + "VALUES (1, $1, $2, $3, $4, 16384, $5, true, 0, 0, 0, $6, $7, 0, $8)", connection); + cmd.Parameters.AddWithValue(t); + cmd.Parameters.AddWithValue(ServerId); + cmd.Parameters.AddWithValue(ServerName); + cmd.Parameters.AddWithValue(queryId); + cmd.Parameters.AddWithValue(userId); + cmd.Parameters.AddWithValue(deltaCalls); + cmd.Parameters.AddWithValue(deltaMs); + cmd.Parameters.AddWithValue(interval.HasValue ? interval.Value : DBNull.Value); + await cmd.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM procedure_stats WHERE server_id = {ServerId}; DELETE FROM pg_statement_stats WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/DeltaFamilyIntervalCompletionRungTests.cs b/Darling/Darling.Tests/DeltaFamilyIntervalCompletionRungTests.cs new file mode 100644 index 000000000..0ace3a7bc --- /dev/null +++ b/Darling/Darling.Tests/DeltaFamilyIntervalCompletionRungTests.cs @@ -0,0 +1,407 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Linq; +using System.Reflection; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// V128 / #3540: the completion of V127. sample_interval_seconds on the four delta families V127 left +/// naked — procedure_stats, memory_grant_stats, pg_wait_stats, pg_statement_stats +/// — and the two statement offsets on query_stats that its delta key is made of. After this rung every +/// member of CollectorDeltaCalculator.DeltaFamilyCollectors stores the interval its deltas accrued +/// over, so the calculator's (delta 0, interval 0) "no delta knowable" marker reaches the store from every +/// family, and the restart seed can rebuild the one key the store could not reproduce. +/// +/// This file carries the "I am the top rung" claims that moved off +/// (V127) when this rung landed, the same handoff that file +/// received from (V126) — a fully-migrated store must map to +/// EXACTLY this version, or the viewer's connect-time gate refuses a store that is actually current. +/// +public sealed class DeltaFamilyIntervalCompletionRungTests +{ + private const int RungVersion = 128; + + private const int PreviousVersion = 127; + + /// This rung's sentinel ordinal in the viewer probe — the newest, so the last argument. + private const int ProbeOrdinal = 103; + + private const string IntervalColumn = "sample_interval_seconds"; + + private static readonly string[] IntervalTables = { "procedure_stats", "memory_grant_stats", "pg_wait_stats", "pg_statement_stats" }; + + private static readonly string[] OffsetColumns = { "statement_start_offset", "statement_end_offset" }; + + /* ---- the rung ------------------------------------------------------------------------------------ */ + + [Fact] + public void TheRungIsRegisteredAtTheTopOfADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + Assert.Equal( + "delta-family-interval-completion", + PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name); + + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + Assert.Equal(RungVersion, StorageVersion.SchemaVersion); + + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + } + + /// + /// The hand-written head of the rung — everything before the V54 pre-add the ladder entry concatenates + /// — adds ONE nullable, default-less integer interval column to each of the four tables and the two + /// offset columns to query_stats, all schema-qualified and idempotent; refreshes the ONE + /// SELECT * passthrough among them; drops the payload-resolving v_query_stats for the + /// regenerated definition to follow; and does nothing else. + /// + /// integer is pinned against the type perfmon_stats already uses through the generator, not + /// as a literal alone, so the ten interval columns cannot drift apart and + /// NULLIF(sample_interval_seconds, 0) means the same thing on every one. No DEFAULT and no backfill + /// on any of the six: a historical row never recorded its interval or its offsets, a backfilled 0 + /// interval would stamp all of history "unknowable", and a backfilled 0/-1 offset pair would seed + /// baselines under a key nothing will ever present. + /// + [Fact] + public void TheRungAddsSixNullableIntegers_SchemaQualified_RefreshesTheGrantView_AndDropsTheResolvingView() + { + var full = PgMigrations.Scripts.Single(s => s.Version == RungVersion).Sql.Replace("\r\n", "\n", StringComparison.Ordinal); + + /* The V121 idiom: the hand-written head, then V54's gz pre-add, then every payload column's pre-add, + then the regenerated resolving view. The head is what this test inspects statement by statement; + MigrationLadderPins holds the ordering of the rest. */ + var v54 = full.IndexOf("ALTER TABLE query_plan_dim", StringComparison.Ordinal); + Assert.True(v54 > 0, "the rung does not carry V54's gz pre-add ahead of the regenerated resolving view"); + var head = full[..v54]; + + var perfmonType = PgSchemaGenerator.TypeFor( + PerfmonStatsCollector.Instance.PayloadColumns.Single(c => c.Name == IntervalColumn)); + Assert.Equal("integer", perfmonType); + + foreach (var table in IntervalTables) + { + Assert.Equal(1, CountOf(head, $"ALTER TABLE collect.{table}\n")); + Assert.DoesNotContain($"ALTER TABLE {table}\n", head, StringComparison.Ordinal); + Assert.Contains( + $"ALTER TABLE collect.{table}\n ADD COLUMN IF NOT EXISTS {IntervalColumn} {perfmonType};", + head, StringComparison.Ordinal); + } + + foreach (var column in OffsetColumns) + { + Assert.Contains( + $"ALTER TABLE collect.query_stats\n ADD COLUMN IF NOT EXISTS {column} integer;", + head, StringComparison.Ordinal); + } + + Assert.Equal(6, CountOf(head, "ADD COLUMN IF NOT EXISTS")); + + /* The one SELECT * passthrough among the five tables (PgSchemaGenerator.AllPassthroughViews pins the + set): Postgres freezes a view's column list at CREATE (V14, V80, V81, V127). */ + Assert.Equal(1, CountOf(head, "CREATE OR REPLACE VIEW collect.v_memory_grant_stats AS SELECT * FROM collect.memory_grant_stats;")); + Assert.Equal(1, CountOf(head, "CREATE OR REPLACE VIEW collect.v_")); + foreach (var table in new[] { "procedure_stats", "pg_wait_stats", "pg_statement_stats" }) + { + Assert.DoesNotContain($"v_{table}", head, StringComparison.Ordinal); + Assert.DoesNotContain("v_" + table, PgSchemaGenerator.AllPassthroughViews); + } + + /* v_query_stats is the payload-RESOLVING view (#1767) and the offsets land ahead of its digest columns, + an alteration CREATE OR REPLACE VIEW refuses: DROP here, regenerate in the tail (V51 / V121). */ + Assert.Equal(1, CountOf(head, "DROP VIEW IF EXISTS collect.v_query_stats;")); + Assert.DoesNotContain("CASCADE", head, StringComparison.Ordinal); + var tail = full[v54..]; + Assert.Equal(1, CountOf(tail, "CREATE OR REPLACE VIEW v_query_stats AS")); + Assert.True( + tail.LastIndexOf("CREATE OR REPLACE VIEW v_query_stats AS", StringComparison.Ordinal) + > tail.LastIndexOf("ADD COLUMN IF NOT EXISTS", StringComparison.Ordinal), + "the regenerated resolving view must be the LAST statement, after every pre-add"); + foreach (var column in OffsetColumns) + { + Assert.Contains($"f.{column}", tail, StringComparison.Ordinal); + } + + /* Nullable, no default, no backfill, no CHECK, no GRANT, and no touch of any continuous aggregate — + the wait_stats_baseline follow-up V127 documented is an aggregate-plus-retirement operation, not a + column, and not this rung. */ + Assert.DoesNotContain("DEFAULT", head, StringComparison.Ordinal); + Assert.DoesNotContain("NOT NULL", head, StringComparison.Ordinal); + Assert.DoesNotContain("UPDATE ", head, StringComparison.Ordinal); + Assert.DoesNotContain("CHECK", head, StringComparison.Ordinal); + Assert.DoesNotContain("GRANT", head, StringComparison.Ordinal); + Assert.DoesNotContain("MATERIALIZED", head, StringComparison.Ordinal); + Assert.DoesNotContain("wait_stats_baseline", head, StringComparison.Ordinal); + Assert.DoesNotContain("_hourly", head, StringComparison.Ordinal); + } + + /// + /// The offsets' semantics are stated in the rung's own doc, in the words a reader will search for: they + /// are BYTE offsets into the batch's nvarchar text (so a character slice divides by two), -1 as the + /// end offset means "to the end of the batch", and they are stored VERBATIM — never normalized — because + /// the delta key is built over the raw values. A future reader will second-guess exactly that pair, and + /// the rung doc is where they will look. + /// + [Fact] + public void TheRungDoc_StatesTheOffsetsSemantics_BytesEndOfBatchAndVerbatim() + { + var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "PgMigrations.cs"); + var start = source.IndexOf("/// V128 —", StringComparison.Ordinal); + var end = source.IndexOf("private const string V128Sql", StringComparison.Ordinal); + Assert.True(start >= 0 && end > start, "the V128 rung has no XML doc block ahead of V128Sql"); + var doc = source[start..end]; + + Assert.Contains("BYTES", doc, StringComparison.Ordinal); + Assert.Contains("divides by two", doc, StringComparison.Ordinal); + Assert.Contains("statement_end_offset = -1", doc, StringComparison.Ordinal); + Assert.Contains("means \"to the end of the batch\"", doc, StringComparison.Ordinal); + Assert.Contains("verbatim as the DMV reports them", doc, StringComparison.Ordinal); + Assert.Contains("never normalized", doc, StringComparison.Ordinal); + /* And the SQL itself repeats the two facts beside the ALTERs. */ + var sql = PgMigrations.Scripts.Single(s => s.Version == RungVersion).Sql; + Assert.Contains("BYTE offsets", sql, StringComparison.Ordinal); + Assert.Contains("statement_end_offset = -1 means", sql, StringComparison.Ordinal); + } + + /// + /// A FRESH store gets the columns from the generated CREATE TABLE (the collector definitions carry them + /// now): the interval as the trailing column on all four, in the same type the rung's ALTER adds, and the + /// two offsets as the trailing pair on query_stats — so fresh-through-V128 and upgraded-to-V128 stores are + /// shaped identically and the rung's ALTERs no-op on the former. + /// + [Fact] + public void TheGeneratedCreateTable_CarriesTheColumnsLast_OnAllFive() + { + foreach (var table in IntervalTables) + { + var definition = CollectorCatalog.Find(table); + Assert.NotNull(definition); + + Assert.Equal(IntervalColumn, definition!.PayloadColumns[^1].Name); + Assert.Equal(CollectorColumnType.Integer, definition.PayloadColumns[^1].Type); + + var ddl = PgSchemaGenerator.CreateTable(definition); + Assert.EndsWith($" {IntervalColumn} integer\n);", ddl, StringComparison.Ordinal); + } + + var queryStats = CollectorCatalog.Find("query_stats")!; + Assert.Equal(OffsetColumns[0], queryStats.PayloadColumns[^2].Name); + Assert.Equal(OffsetColumns[1], queryStats.PayloadColumns[^1].Name); + Assert.All(OffsetColumns, c => Assert.Equal(CollectorColumnType.Integer, queryStats.PayloadColumns.Single(p => p.Name == c).Type)); + Assert.EndsWith(" statement_start_offset integer,\n statement_end_offset integer\n);", PgSchemaGenerator.CreateTable(queryStats), StringComparison.Ordinal); + + /* The COPY column list is the PayloadColumns order, so the new columns ride LAST there too. */ + Assert.EndsWith(", query_plan_xml_bytes, statement_start_offset, statement_end_offset) FROM STDIN (FORMAT BINARY)", + PgCollectorRowWriter.CopyCommandFor(QueryStatsCollector.Instance), StringComparison.Ordinal); + foreach (var table in IntervalTables) + { + Assert.EndsWith($", {IntervalColumn}) FROM STDIN (FORMAT BINARY)", + PgCollectorRowWriter.CopyCommandFor(CollectorCatalog.Find(table)!), StringComparison.Ordinal); + } + } + + /// + /// The regenerated resolving view names every query_stats payload column in order — the two offsets ahead + /// of the digests, which is WHY the rung drops and recreates rather than replacing — and the pre-add guard + /// covers them, so a store replaying V51, V54 or V121 on this build (all of which re-emit the view) has + /// the columns before the view names them. + /// + [Fact] + public void TheResolvingView_NamesTheOffsets_AndEveryRungReEmittingItPreAddsThem() + { + var view = PgSchemaGenerator.GenerateQueryStatsResolvingView(); + var startAt = view.IndexOf("f.statement_start_offset", StringComparison.Ordinal); + var endAt = view.IndexOf("f.statement_end_offset", StringComparison.Ordinal); + var digestAt = view.IndexOf("f.query_text_digest", StringComparison.Ordinal); + Assert.True(startAt > 0 && endAt > startAt && digestAt > endAt, + "the offsets must be projected in order and ahead of the digest columns"); + + var preAdds = PgSchemaGenerator.GenerateQueryStatsPayloadColumnPreAdds(); + foreach (var column in OffsetColumns) + { + Assert.Contains($"ALTER TABLE query_stats ADD COLUMN IF NOT EXISTS {column} integer;", preAdds, StringComparison.Ordinal); + } + + /* Every rung that emits the RESOLVING view (V38, V51, V54, V121, V128) names f.; V4 and V14 + define the passthrough and name neither, so they are skipped on `use < 0` the way + MigrationLadderPins skips them. The resolving definers must pre-add first: presence, then order. */ + var resolvingDefiners = 0; + foreach (var rung in PgMigrations.Scripts.Where(m => m.Sql.Contains("VIEW v_query_stats AS", StringComparison.Ordinal))) + { + foreach (var column in OffsetColumns) + { + var use = rung.Sql.IndexOf($"f.{column}", StringComparison.Ordinal); + if (use < 0) + { + continue; + } + + resolvingDefiners++; + var guard = rung.Sql.IndexOf($"ADD COLUMN IF NOT EXISTS {column} ", StringComparison.Ordinal); + Assert.True(guard >= 0, $"V{rung.Version} names f.{column} and never adds it"); + Assert.True(guard < use, $"V{rung.Version} adds {column} AFTER the view uses it"); + } + } + + /* Two offsets × the five resolving definers: a scan that skipped everything would pass while asserting + nothing. */ + Assert.Equal(10, resolvingDefiners); + } + + /* ---- the probe (three sites, top arm) ------------------------------------------------------------- */ + + /// + /// The viewer probe's three sites carry this rung's sentinel, and the map treats it as the TOP arm. + /// + /// The probe asks the question, the caller reads the answer, the map has the parameter — three + /// sites, and a sentinel present at only some of them shifts every LATER ordinal onto the wrong column. + /// Miss all three and a fully-migrated store probes one rung short, so the connect-time gate refuses a + /// store that is in fact current — permanently, because no later upgrade changes the answer. The + /// sentinel shares V127's column NAME on a different TABLE, which is exactly why the table is in the + /// predicate. + /// + [Fact] + public void TheProbeMapsAFullyMigratedStoreToThisTopRung() + { + Assert.Contains( + $"table_name = 'procedure_stats'\n AND column_name = '{IntervalColumn}'", + ViewerDataService.StoreSchemaProbeSql.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal); + + var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal); + Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal); + Assert.Contains("hasDeltaFamilyIntervalCompletion", viewer, StringComparison.Ordinal); + + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!; + var arity = method.GetParameters().Length; + + /* The top rung's sentinel IS the last argument. */ + Assert.Equal(ProbeOrdinal, arity - 1); + + /* Every sentinel true = a fully-migrated store, which must map to exactly this version. Built by + reflection so the arity tracks the signature. */ + var all = Enumerable.Repeat((object)true, arity).ToArray(); + Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!); + + /* This rung's own arm answers for a store that stopped here. Expressed as "false above" rather than + as one named ordinal, so a rung landing on top of this one does not quietly turn this case into a + test of that rung. */ + var atThisRung = Enumerable.Range(0, arity).Select(i => (object)(i <= ProbeOrdinal)).ToArray(); + Assert.Equal(RungVersion, (int)method.Invoke(null, atThisRung)!); + + /* One rung behind: the same store WITHOUT this rung's sentinel reports the previous rung. */ + var behind = (object[])atThisRung.Clone(); + behind[ProbeOrdinal] = false; + Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!); + + /* And in the source, the arm sits ABOVE the previous rung's — newest-first is the whole contract of + that method — and returns this build's version rather than a literal that could drift from it. */ + var thisArm = viewer.IndexOf("if (hasDeltaFamilyIntervalCompletion)", StringComparison.Ordinal); + var previousArm = viewer.IndexOf("if (hasDeltaFamilyIntervalColumns)", StringComparison.Ordinal); + Assert.True(thisArm >= 0, "the viewer has no V128 sentinel arm — a fully-migrated store would map one rung low"); + Assert.True(previousArm >= 0, "the previous rung's arm is gone, so this pin is comparing against nothing"); + Assert.True(thisArm < previousArm, "the V128 arm sits below the previous rung's, so a current store maps one rung low"); + Assert.Contains( + "return " + StorageVersion.SchemaVersion.ToString(CultureInfo.InvariantCulture) + ";", + viewer[thisArm..], StringComparison.Ordinal); + } + + /* ---- the writers ---------------------------------------------------------------------------------- */ + + /// + /// The four collectors now WRITE the interval — through CalculateDeltaWithInterval for every group, + /// never the bare CalculateDelta — as the minimum over each row's delta groups (the V127 rule). The + /// two SQL Server collectors take the minimum in WritePayload; the two PostgreSQL collectors compute + /// their deltas in ReadAsync (the idle-row skip needs them before the row exists), so the minimum + /// rides the Row and WritePayload writes it from there. The column without the writer would be a + /// NULL forever; the writer taking one headline group's interval would let an independently reset sibling + /// counter's 0 read as idle over a real interval. + /// + [Fact] + public void TheFourCollectors_WriteTheIntervalAsTheMinimumOverTheirDeltaGroups() + { + foreach (var (file, groups, written) in new[] + { + ("ProcedureStatsCollector.cs", 7, ".Value(sampleIntervalSeconds);"), + ("MemoryGrantsCollector.cs", 2, ".Value(sampleIntervalSeconds);"), + ("PgWaitStatsCollector.cs", 2, ".Value(row.SampleIntervalSeconds);"), + ("PgStatementStatsCollector.cs", 3, ".Value(row.SampleIntervalSeconds);"), + }) + { + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Collectors", file); + + Assert.Equal(groups, CountOf(source, "context.Deltas.CalculateDeltaWithInterval(")); + Assert.DoesNotContain("context.Deltas.CalculateDelta(", source, StringComparison.Ordinal); + Assert.Contains("var sampleIntervalSeconds = Math.Min(", source, StringComparison.Ordinal); + Assert.Contains(written, source, StringComparison.Ordinal); + } + } + + /// + /// query_stats WRITES the two offsets it keys on, raw, from the same Row fields the key is built from — so + /// the stored pair and the key's pair are one value, not two that agree today. + /// + [Fact] + public void TheQueryStatsCollector_WritesTheOffsetsItKeysOn_Raw() + { + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Collectors", "QueryStatsCollector.cs"); + + Assert.Contains("$\"{row.SqlHandle}:{row.StatementStartOffset}:{row.StatementEndOffset}:{row.PlanHandle}\"", source, StringComparison.Ordinal); + Assert.Contains(".Value(row.StatementStartOffset)", source, StringComparison.Ordinal); + Assert.Contains(".Value(row.StatementEndOffset);", source, StringComparison.Ordinal); + /* No arithmetic on the way to the store: the collector's SUBSTRING divides by two to SLICE the text, + and that is the only place the byte offsets are ever transformed. */ + Assert.DoesNotContain(".Value(row.StatementStartOffset / ", source, StringComparison.Ordinal); + Assert.DoesNotContain(".Value(row.StatementEndOffset / ", source, StringComparison.Ordinal); + } + + /// + /// The calculator's doc claim is now true for ALL families and says so: every member of + /// DeltaFamilyCollectors persists the interval, and the sentence names the census that pins it + /// (Lite.Tests' DeltaFamilyIntervalColumnTests, whose still-naked list is empty), so it cannot + /// silently go false again by an eleventh family shipping naked. + /// + [Fact] + public void TheCalculatorDoc_SaysEveryFamilyStoresTheInterval_AndNamesTheCensus() + { + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Collectors", "CollectorDeltaCalculator.cs"); + + Assert.Contains("for EVERY delta family: all ten members of DeltaFamilyCollectors persist a", source, StringComparison.Ordinal); + Assert.Contains("procedure_stats, memory_grant_stats, pg_wait_stats and pg_statement_stats since", source, StringComparison.Ordinal); + Assert.Contains("Darling V128 / Lite v61, #3540)", source, StringComparison.Ordinal); + Assert.Contains("DeltaFamilyIntervalColumnTests is the census", source, StringComparison.Ordinal); + /* The "still persist no interval" sentence V127 wrote is gone with the list it described. */ + Assert.DoesNotContain("persist no interval", source, StringComparison.Ordinal); + Assert.DoesNotContain("naked list shrinks", source, StringComparison.Ordinal); + } + + private static int CountOf(string haystack, string needle) + { + var count = 0; + for (var at = haystack.IndexOf(needle, StringComparison.Ordinal); + at >= 0; + at = haystack.IndexOf(needle, at + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Darling/Darling.Tests/DeltaSeriesAgeTests.cs b/Darling/Darling.Tests/DeltaSeriesAgeTests.cs index bc85d91c3..148b1b971 100644 --- a/Darling/Darling.Tests/DeltaSeriesAgeTests.cs +++ b/Darling/Darling.Tests/DeltaSeriesAgeTests.cs @@ -231,6 +231,198 @@ public void ThePassWindowIsPerServerAndPerCollector() Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, "other_collector", "k", 500, 5, out _, T1, Gap)); } + /* ---------------- #3540 A4: the pass window survives a restart ---------------- */ + + /// + /// THE restart pin. Before #3540 no host seeded the pass window, so the first pass after a restart + /// always saw "no previous look" and baselined every new key — the rescue was inert on exactly the + /// cycle it exists for. A seeded Current alone arms it: the first post-restart pass rolls it into + /// Previous and measures the gap against it. + /// + [Fact] + public void ASeededPassWindowArmsTheRescueOnTheFirstPostRestartPass() + { + var deltas = new SeedingCalculator(); + deltas.SeedPassWindow(ServerId, Collector, current: T0); + + /* The first pass of the new process: a plan compiled 20 s ago, inside the 60 s since the + predecessor's last look, is credited in full with a real interval. */ + var delta = deltas.CalculateDeltaWithSeriesAge( + ServerId, Collector, "sql:0:99:planB", 900, seriesAgeSeconds: 20, out var interval, T1, Gap); + + Assert.Equal(900, delta); + Assert.Equal(60, interval); + + /* The control: the same first pass on an UNSEEDED calculator is the pre-#3540 behaviour. */ + var cold = new CollectorDeltaCalculator(); + Assert.Equal(0, cold.CalculateDeltaWithSeriesAge(ServerId, Collector, "sql:0:99:planB", 900, 20, out var coldInterval, T1, Gap)); + Assert.Equal(0, coldInterval); + } + + /// + /// The seeded window is still bounded by the gap policy and by the age: a series older than the gap + /// since the seeded look, or a first pass past the policy, is refused exactly as an in-process window + /// would refuse it. Seeding restores the window; it does not loosen the rule. + /// + [Fact] + public void ASeededPassWindowKeepsTheGapAndAgeBounds() + { + var deltas = new SeedingCalculator(); + deltas.SeedPassWindow(ServerId, Collector, current: T0); + + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "old", 999, seriesAgeSeconds: 3_600, out var i1, T1, Gap)); + Assert.Equal(0, i1); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "fresh", 777, seriesAgeSeconds: 30, out var i2, T0.AddHours(2), Gap)); + Assert.Equal(0, i2); + } + + /// + /// A seeded Previous is never what the first post-restart pass reads: that pass carries a NEW + /// collection time, so PreviousPass rolls the seeded Current into Previous before returning. + /// Pinned so a seeder that returns one collection time per server (the wait_stats shape) is known + /// to lose nothing by passing null — and so nobody widens a seed read to fetch a second timestamp + /// on the belief that the rescue needs it. + /// + [Fact] + public void TheSeededPreviousIsNotWhatTheFirstPostRestartPassReads() + { + var withPrevious = new SeedingCalculator(); + withPrevious.SeedPassWindow(ServerId, Collector, current: T0, previous: T0.AddMinutes(-5)); + + var withoutPrevious = new SeedingCalculator(); + withoutPrevious.SeedPassWindow(ServerId, Collector, current: T0); + + var a = withPrevious.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 900, 20, out var ia, T1, Gap); + var b = withoutPrevious.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 900, 20, out var ib, T1, Gap); + + Assert.Equal(900, a); + Assert.Equal(a, b); + Assert.Equal(60, ia); + Assert.Equal(ia, ib); + } + + /// The seeded window is per server and per group, like the live one. + [Fact] + public void ASeededPassWindowIsPerServerAndPerGroup() + { + var deltas = new SeedingCalculator(); + deltas.SeedPassWindow(ServerId, Collector, current: T0); + + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(2, Collector, "k", 500, 5, out _, T1, Gap)); + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, "query_stats_exec", "k", 500, 5, out _, T1, Gap)); + } + + /// ClearServer drops a SEEDED window too — the re-add path both hosts wire. + [Fact] + public void ClearServerDropsASeededPassWindow() + { + var deltas = new SeedingCalculator(); + deltas.SeedPassWindow(ServerId, Collector, current: T0); + + deltas.ClearServer(ServerId); + + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, Collector, "k", 900, 5, out var interval, T1, Gap)); + Assert.Equal(0, interval); + } + + /// + /// SeedPasses seeds EVERY group named, from the tracker's per-server window — a seeder that + /// seeded the family name instead of its groups would arm nothing, because the live window is keyed + /// by the group the collector passes as collectorName. + /// + [Fact] + public void SeedPassesArmsEveryGroupItIsGiven() + { + var deltas = new SeedingCalculator(); + deltas.SeedFromTracker(new[] { (ServerId, T0) }, "query_stats_exec", "query_stats_worker", "query_stats_rows"); + + foreach (var group in new[] { "query_stats_exec", "query_stats_worker", "query_stats_rows" }) + { + Assert.Equal(900, deltas.CalculateDeltaWithSeriesAge(ServerId, group, "k", 900, 20, out _, T1, Gap)); + } + + /* The family name itself is not a group and must not have been armed. */ + Assert.Equal(0, deltas.CalculateDeltaWithSeriesAge(ServerId, "query_stats", "k", 900, 20, out _, T1, Gap)); + } + + /// + /// The tracker keeps the two most recent DISTINCT times per server whatever order the rows stream in, + /// and ignores a null time rather than guessing — the seed reads are unordered by time within a + /// server, and a latest-collection read repeats one time across every row. + /// + [Fact] + public void TheSeedPassTrackerKeepsTheLatestAndTheOneBefore_InAnyArrivalOrder() + { + var a = T0.AddMinutes(-4); + var b = T0.AddMinutes(-2); + var c = T0; + + var arrivals = new[] + { + new[] { a, b, c }, + new[] { c, b, a }, + new[] { b, c, a, c, b, b }, + }; + + foreach (var order in arrivals) + { + var tracker = new SeedingCalculator.Tracker(); + foreach (var t in order) + { + tracker.Observe(ServerId, t); + } + tracker.Observe(ServerId, null); + + var window = Assert.Single(tracker.Servers); + Assert.Equal(ServerId, window.ServerId); + Assert.Equal(c, window.Latest); + Assert.Equal(b, window.Before); + } + + /* One time observed many times: Latest is it, Before is nothing — not a copy of Latest. */ + var single = new SeedingCalculator.Tracker(); + single.Observe(ServerId, c); + single.Observe(ServerId, c); + var only = Assert.Single(single.Servers); + Assert.Equal(c, only.Latest); + Assert.Null(only.Before); + + /* And nothing observed is nothing seeded: a null-only stream yields no server. */ + var empty = new SeedingCalculator.Tracker(); + empty.Observe(ServerId, null); + Assert.Empty(empty.Servers); + Assert.Equal(0, empty.Count); + } + + /// + /// A host stand-in exposing the protected seeding hooks. The real hosts (Lite's DeltaCalculator, + /// DarlingDeltaCalculator) call the same members from their store reads; this one calls them from + /// literals so the window's semantics are pinned without an engine. + /// + private sealed class SeedingCalculator : CollectorDeltaCalculator + { + public void SeedPassWindow(int serverId, string group, DateTime current, DateTime? previous = null) + => SeedPass(serverId, group, current, previous); + + public void SeedFromTracker((int ServerId, DateTime Time)[] rows, params string[] groups) + { + var tracker = new SeedPassTracker(); + foreach (var (serverId, time) in rows) + { + tracker.Observe(serverId, time); + } + SeedPasses(tracker, groups); + } + + public sealed class Tracker + { + private readonly SeedPassTracker _inner = new(); + public void Observe(int serverId, DateTime? t) => _inner.Observe(serverId, t); + public System.Collections.Generic.IEnumerable<(int ServerId, DateTime Latest, DateTime? Before)> Servers => _inner.Servers; + public int Count => _inner.Count; + } + } + /// /// An implementer that never opted in keeps compiling and keeps its old behaviour, which is the /// reason the interface method is default-implemented rather than abstract. diff --git a/Darling/Darling.Tests/DrillDownDopProvenanceLiveTests.cs b/Darling/Darling.Tests/DrillDownDopProvenanceLiveTests.cs new file mode 100644 index 000000000..5c914a757 --- /dev/null +++ b/Darling/Darling.Tests/DrillDownDopProvenanceLiveTests.cs @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Analysis; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Analysis; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Live-Postgres pin for #3648, Darling's twin of Lite's DrillDownDopProvenanceTests: the +/// top_cpu_queries and bad_actor_query drill-downs must headline the NEWEST plan's +/// max_dop and carry the cross-plan maximum as a history with provenance, not fold every plan the +/// hash ever had inside the window into one provenance-free number. +/// +/// The fixture is the live page. One query_hash, two plans: an old parallel plan whose +/// per-plan high-water mark reads 16 and was last seen three hours before the window's end, and a new serial +/// plan reading 1 that spent the CPU at the newest snapshot. The old read said max_dop = 16 for this +/// shape on a MAXDOP-1 instance whose stored plan was serial, and a tuning recommendation was made from it +/// and retracted. +/// +/// Live rather than a string pin because the claim is the ENGINE's answer to ROW_NUMBER() OVER +/// with explicit NULLS LAST tie-breakers combined with a partition-wide MAX() OVER — which row +/// the newest-plan CASE picks on Postgres, and that a NULL reading reaches the reader as NULL. The text half +/// (byte-identity with Lite's SQL) lives in Lite.Tests.DrillDownDopProvenanceParityTests. +/// +[Collection("live-postgres")] +public sealed class DrillDownDopProvenanceLiveTests +{ + private const int TestServerId = -364800; + private const string TestServerName = "DopProvSrv"; + private const string Db = "DopProvDb"; + private const string Hash = "0x3648HASH"; + private const string OldParallelPlan = "0x3648PLANPARALLEL"; + private const string NewSerialPlan = "0x3648PLANSERIAL"; + + private static DateTime TruncateToSeconds(DateTime t) => + DateTime.SpecifyKind(new DateTime(t.Ticks - (t.Ticks % TimeSpan.TicksPerSecond)), DateTimeKind.Unspecified); + + private static async Task OpenWithSearchPathAsync(string connectionString, CancellationToken ct) + { + var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await using var setPath = new NpgsqlCommand("SET search_path = " + PgSchemaGenerator.SearchPath, connection); + await setPath.ExecuteNonQueryAsync(ct); + return connection; + } + + private static async Task SeedAsync( + NpgsqlConnection c, string planHash, DateTime collectionTime, int? maxDop, long workerTimeUs, + DateTime? creationTime, CancellationToken ct) + { + await using var command = new NpgsqlCommand(@" +INSERT INTO query_stats + (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_plan_hash, + sql_handle, plan_handle, creation_time, query_text, delta_execution_count, delta_worker_time, + delta_elapsed_time, delta_logical_reads, delta_spills, min_dop, max_dop) +VALUES + ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)", c); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTime, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(TestServerId); + command.Parameters.AddWithValue(TestServerName); + command.Parameters.AddWithValue(Db); + command.Parameters.AddWithValue(Hash); + command.Parameters.AddWithValue(planHash); + command.Parameters.AddWithValue("0x3648SQLH"); + command.Parameters.AddWithValue(planHash + "H"); + command.Parameters.AddWithValue(DateTime.SpecifyKind(creationTime ?? collectionTime.AddDays(-1), DateTimeKind.Unspecified)); + command.Parameters.AddWithValue("SELECT * FROM DopProvTable"); + command.Parameters.AddWithValue(10L); + command.Parameters.AddWithValue(workerTimeUs); + command.Parameters.AddWithValue(workerTimeUs * 2); + command.Parameters.AddWithValue(1000L); + command.Parameters.AddWithValue(0L); + command.Parameters.AddWithValue(1); + command.Parameters.AddWithValue(maxDop.HasValue ? maxDop.Value : DBNull.Value); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await using var command = new NpgsqlCommand("DELETE FROM query_stats WHERE server_id = $1", connection); + command.Parameters.AddWithValue(TestServerId); + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task CollectAsync( + NpgsqlDataSource postgres, AnalysisContext context, string factKey, string drillDownKey) + { + var finding = new AnalysisFinding + { + RootFactKey = factKey, + StoryPath = factKey, + /* Past the display gate — below it the expensive drill-downs are skipped wholesale and this + collector never runs at all. */ + Severity = 1.0, + }; + + await new PgDrillDownCollector(postgres).EnrichFindingsAsync([finding], context); + + Assert.NotNull(finding.DrillDown); + Assert.True(finding.DrillDown.TryGetValue(drillDownKey, out var raw), $"{drillDownKey} was not collected"); + return JsonSerializer.SerializeToElement(raw); + } + + [Fact] + public async Task TopCpuAndBadActor_HeadlineTheNewestPlansDop_AndCarryTheParallelHistory() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live #3648 DOP-provenance test."); + + var ct = TestContext.Current.CancellationToken; + var bodySucceeded = false; + + await using (var connection = new NpgsqlConnection(connectionString)) + { + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + } + + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + } + + try + { + var periodEnd = TruncateToSeconds(DateTime.UtcNow); + var periodStart = periodEnd.AddHours(-4); + var parallelLastSeen = periodEnd.AddHours(-3); + var context = new AnalysisContext + { + ServerId = TestServerId, + ServerName = TestServerName, + TimeRangeStart = periodStart, + TimeRangeEnd = periodEnd, + ServerUtcOffset = TimeSpan.Zero, + }; + + var expectedNote = $"DOP 1 (a parallel plan ran at 16 until {parallelLastSeen:yyyy-MM-dd}; 2 plans in window)"; + + /* ── The live page, RED before #3648 (max_dop read 16): three parallel snapshots ending three + hours before the window's end, then the serial plan at the newest snapshot. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await SeedAsync(connection, OldParallelPlan, parallelLastSeen.AddMinutes(-20), 16, 100_000, null, ct); + await SeedAsync(connection, OldParallelPlan, parallelLastSeen.AddMinutes(-10), 16, 100_000, null, ct); + await SeedAsync(connection, OldParallelPlan, parallelLastSeen, 16, 100_000, null, ct); + await SeedAsync(connection, NewSerialPlan, periodEnd.AddMinutes(-30), 1, 400_000, null, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var top = Assert.Single((await CollectAsync(postgres, context, "CPU_SQL_PERCENT", "top_cpu_queries")).EnumerateArray()); + Assert.Equal(1, top.GetProperty("max_dop").GetInt32()); + Assert.Equal(16, top.GetProperty("max_dop_any_plan").GetInt32()); + Assert.Equal(2, top.GetProperty("plan_count").GetInt64()); + Assert.Equal(parallelLastSeen, DateTime.Parse(top.GetProperty("max_dop_any_plan_last_seen").GetString()!, null, + System.Globalization.DateTimeStyles.RoundtripKind)); + Assert.Equal(expectedNote, top.GetProperty("dop_note").GetString()); + /* The windowed total still spans BOTH plans: 3 x 100000 + 400000 us = 700 ms. */ + Assert.Equal(700.0, top.GetProperty("total_cpu_ms").GetDouble()); + + var bad = await CollectAsync(postgres, context, "BAD_ACTOR_" + Hash, "bad_actor_query"); + Assert.Equal(1, bad.GetProperty("max_dop").GetInt32()); + Assert.Equal(16, bad.GetProperty("max_dop_any_plan").GetInt32()); + Assert.Equal(2, bad.GetProperty("plan_count").GetInt64()); + Assert.Equal(expectedNote, bad.GetProperty("dop_note").GetString()); + } + + /* ── 0 is unknown: a NULL reading arrives as JSON null on every DOP field and no history is + invented from nothing. ── */ + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedAsync(connection, NewSerialPlan, periodEnd.AddMinutes(-30), null, 400_000, null, ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var top = Assert.Single((await CollectAsync(postgres, context, "CPU_SQL_PERCENT", "top_cpu_queries")).EnumerateArray()); + Assert.Equal(JsonValueKind.Null, top.GetProperty("max_dop").ValueKind); + Assert.Equal(JsonValueKind.Null, top.GetProperty("max_dop_any_plan").ValueKind); + Assert.Equal(JsonValueKind.Null, top.GetProperty("max_dop_any_plan_last_seen").ValueKind); + Assert.Equal(JsonValueKind.Null, top.GetProperty("dop_note").ValueKind); + } + + /* ── Both plans at the newest snapshot (the stale parallel plan still cached beside the serial + one that replaced it): collection_time ties, so compile time decides, and the plan + compiled LATER is the headline whatever its counter says. Postgres would default a DESC + sort to NULLS FIRST where DuckDB defaults NULLS LAST — the explicit NULLS LAST is what + keeps this row choice identical across the SKUs. ── */ + var newest = periodEnd.AddMinutes(-30); + await using (var connection = await OpenWithSearchPathAsync(connectionString!, ct)) + { + await DeleteTestRowsAsync(connection, ct); + await SeedAsync(connection, OldParallelPlan, newest, 16, 500_000, periodEnd.AddDays(-20), ct); + await SeedAsync(connection, NewSerialPlan, newest, 1, 100_000, periodEnd.AddDays(-1), ct); + } + + await using (var postgres = NpgsqlDataSource.Create(connectionString!)) + { + var top = Assert.Single((await CollectAsync(postgres, context, "CPU_SQL_PERCENT", "top_cpu_queries")).EnumerateArray()); + Assert.Equal(1, top.GetProperty("max_dop").GetInt32()); + Assert.Equal(16, top.GetProperty("max_dop_any_plan").GetInt32()); + Assert.Equal(newest, DateTime.Parse(top.GetProperty("max_dop_any_plan_last_seen").GetString()!, null, + System.Globalization.DateTimeStyles.RoundtripKind)); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, DeleteTestRowsAsync); + } + } +} diff --git a/Darling/Darling.Tests/DrillDownDopProvenanceParityTests.cs b/Darling/Darling.Tests/DrillDownDopProvenanceParityTests.cs new file mode 100644 index 000000000..b56371f53 --- /dev/null +++ b/Darling/Darling.Tests/DrillDownDopProvenanceParityTests.cs @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Analysis; +using Xunit; + +namespace Darling.Tests; + +/// +/// Darling's half of the cross-SKU text guard for #3648: the top_cpu_queries and bad_actor_query +/// drill-down SQL must be BYTE-IDENTICAL between Lite's inline cmd.CommandText and Darling's +/// / , +/// and both must carry the per-plan provenance shape rather than the old hash-folded MAX(max_dop). +/// +/// Why identical text and not two independent pins. #2705 fixed Darling's stale-max_dop +/// cross-check and closed without a record that Lite had a twin; #2999 is what that cost. The two drill-down +/// reads here were already character-for-character the same before #3648 (the port kept Lite's text), and the +/// provenance columns are engine-neutral (ROW_NUMBER() OVER, MAX() OVER, CASE aggregates, +/// explicit NULLS LAST), so the strongest guard available is equality: a future edit to one SKU's read +/// fails here until the other is brought along. +/// +/// Why this side hosts the comparison. build.yml's darling path filter covers +/// every Lite .cs file as well as the whole Darling tree, so this suite runs on an edit to EITHER +/// analysis tree; the lite filter does not reach Darling/PerformanceMonitor.Darling.Analysis, and +/// a guard comparing the two apps cannot live behind a filter that fires for only one of them (#2839, +/// CrossAppGuardCiGateTests). Lite's DrillDownDopProvenanceParityTests pins Lite's own shape and +/// meta-pins this file. Darling's text is read from the compiled constants rather than from source, so +/// nothing here depends on parsing Darling's own file. +/// +public sealed class DrillDownDopProvenanceParityTests +{ + private const string LiteFile = "Lite/Analysis/DrillDownCollector.Queries.cs"; + + public static TheoryData Reads => new() + { + { "CollectTopCpuQueries", PgDrillDownCollector.TopCpuQueriesSql }, + { "CollectBadActorDetail", PgDrillDownCollector.BadActorDetailSql }, + }; + + [Theory] + [MemberData(nameof(Reads))] + public void TheDrillDownSql_IsByteIdenticalAcrossSkus(string liteMethod, string darlingSql) + { + /* Both sides LF-normalised: the constant carries the line endings of the checkout it was compiled + from and the source read carries the checkout's on-disk endings, which agree on any one runner but + are not the claim under test. */ + Assert.Equal(Lf(darlingSql), Lf(LiteInlineSql(liteMethod))); + } + + [Theory] + [MemberData(nameof(Reads))] + public void TheDrillDownSql_CarriesThePerPlanProvenanceShape_NotTheHashFoldedMaximum(string liteMethod, string darlingSql) + { + _ = liteMethod; + var sql = Lf(darlingSql); + + /* The headline is the NEWEST plan's reading, picked by a newest-first ranking with the tie-breakers + spelled out (compile time, then CPU spent) and their null placement made explicit — DuckDB and + Postgres default DESC null placement differently, and an implicit default here would make the two + SKUs pick different rows on the same data. */ + Assert.Matches( + new Regex(@"ROW_NUMBER\(\)\s+OVER\s*\(\s*PARTITION BY database_name, query_hash\s+ORDER BY collection_time DESC, creation_time DESC NULLS LAST, delta_worker_time DESC NULLS LAST\s*\)\s+AS newest_rn", RegexOptions.Singleline), + sql); + Assert.Contains("MAX(CASE WHEN newest_rn = 1 THEN max_dop END) AS max_dop", sql, StringComparison.Ordinal); + + /* The history: how many plans the group spans, the cross-plan maximum, and when it was last seen. */ + Assert.Contains("COUNT(DISTINCT query_plan_hash) AS plan_count", sql, StringComparison.Ordinal); + Assert.Contains("MAX(max_dop) OVER (PARTITION BY database_name, query_hash) AS max_dop_any_plan", sql, StringComparison.Ordinal); + Assert.Contains("MAX(max_dop_any_plan) AS max_dop_any_plan", sql, StringComparison.Ordinal); + Assert.Contains("MAX(CASE WHEN max_dop = max_dop_any_plan THEN collection_time END) AS max_dop_any_plan_last_seen", sql, StringComparison.Ordinal); + + /* And the lie itself is gone: no bare hash-folded MAX(max_dop) projected as max_dop. */ + Assert.DoesNotContain("MAX(max_dop) AS max_dop", sql, StringComparison.Ordinal); + } + + [Fact] + public void NeitherSku_CoercesAnUnknownDopToZero() + { + /* The DMV never reports 0 — a serial plan is 1 — so `IsDBNull ? 0` on a DOP ordinal was "no reading" + rendered as a degree of parallelism. Both readers must project null and hand the four provenance + values to the shared note composer. Scoped to the two files' max_dop reads: the other drill-downs' + zero-coercions (counts, sums) are legitimately 0-when-absent. */ + foreach (var source in new[] + { + RepoFile.ReadRepoFile("Lite", "Analysis", "DrillDownCollector.Queries.cs"), + RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Analysis", "PgDrillDownCollector.Queries.cs"), + }) + { + Assert.DoesNotMatch(new Regex(@"max_dop\s*=\s*reader\.IsDBNull\(\d+\)\s*\?\s*0\b"), source); + Assert.Matches(new Regex(@"var maxDop = reader\.IsDBNull\(4\) \? \(int\?\)null"), source); + Assert.Matches(new Regex(@"var maxDop = reader\.IsDBNull\(10\) \? \(int\?\)null"), source); + Assert.Contains("dop_note = QueryDopProvenance.Note(maxDop, maxDopAnyPlan, maxDopAnyPlanLastSeen, planCount)", source, StringComparison.Ordinal); + } + } + + private static string Lf(string text) => text.Replace("\r\n", "\n", StringComparison.Ordinal); + + /// + /// Lite's inline SQL: the first cmd.CommandText = @"..." verbatim literal after the named method's + /// declaration. The two reads under test contain no doubled quotes, so the literal ends at the first + /// ";. + /// + private static string LiteInlineSql(string methodName) + { + var source = RepoFile.ReadRepoFile("Lite", "Analysis", "DrillDownCollector.Queries.cs"); + var start = source.IndexOf($"Task {methodName}(", StringComparison.Ordinal); + Assert.True(start >= 0, $"{methodName} not found in {LiteFile}"); + const string Marker = "cmd.CommandText = @\""; + var literalStart = source.IndexOf(Marker, start, StringComparison.Ordinal) + Marker.Length; + var literalEnd = source.IndexOf("\";", literalStart, StringComparison.Ordinal); + return source[literalStart..literalEnd]; + } +} diff --git a/Darling/Darling.Tests/EngineCapabilityMissTests.cs b/Darling/Darling.Tests/EngineCapabilityMissTests.cs index 721b1d57a..71c86fb85 100644 --- a/Darling/Darling.Tests/EngineCapabilityMissTests.cs +++ b/Darling/Darling.Tests/EngineCapabilityMissTests.cs @@ -322,6 +322,12 @@ public sealed class EngineCapabilityReadWiringTests private static readonly Regex CollectorConst = new( @"private const string (\w+) = ""([a-z_0-9]+)"";", RegexOptions.Compiled); + /* A private helper a tool body may delegate its miss path to (#3541 A12: the health-parser family's + shared EmptyAsync ladder). A wiring call inside one is attributed to every tool whose body calls the + helper — the read still asks the question, one method further down. */ + private static readonly Regex HelperDeclaration = new( + @"private static async Task (\w+)(?:<\w+>)?\(", RegexOptions.Compiled); + /// /// Every collector name a shipped read asks the capability question about, across both SKUs. Exposed so /// @@ -353,6 +359,8 @@ internal static SortedDictionary> WiredReads(string mc var consts = CollectorConst.Matches(source) .ToDictionary(m => m.Groups[1].Value, m => m.Groups[2].Value, StringComparer.Ordinal); var marks = ToolMark.Matches(source); + var helpers = HelperDeclaration.Matches(source); + var viaHelper = new Dictionary>(StringComparer.Ordinal); foreach (Match call in WiringCall.Matches(source)) { @@ -360,8 +368,22 @@ internal static SortedDictionary> WiredReads(string mc ? call.Groups[1].Value : consts.TryGetValue(call.Groups[2].Value, out var resolved) ? resolved : call.Groups[2].Value; - /* The enclosing tool is the last McpServerTool mark before the call. */ + /* The enclosing tool is the last McpServerTool mark before the call — unless a private + helper is declared between that mark and the call, in which case the call belongs to the + helper and reaches every tool that calls it (#3541 A12). */ var owner = marks.Where(m => m.Index < call.Index).LastOrDefault(); + var helper = helpers.Where(h => h.Index < call.Index && (owner is null || h.Index > owner.Index)).LastOrDefault(); + if (helper is not null) + { + if (!viaHelper.TryGetValue(helper.Groups[1].Value, out var helperCollectors)) + { + viaHelper[helper.Groups[1].Value] = helperCollectors = new SortedSet(StringComparer.Ordinal); + } + + helperCollectors.Add(collector); + continue; + } + Assert.True(owner is not null, $"{Path.GetFileName(file)}: a capability call sits outside any MCP tool"); if (!wired.TryGetValue(owner!.Groups[1].Value, out var collectors)) @@ -371,6 +393,39 @@ internal static SortedDictionary> WiredReads(string mc collectors.Add(collector); } + + /* Each tool body (from its mark to the next) that calls a wired helper asks the helper's question. */ + for (var i = 0; i < marks.Count; i++) + { + var end = i + 1 < marks.Count ? marks[i + 1].Index : source.Length; + var body = source[marks[i].Index..end]; + foreach (var (helperName, helperCollectors) in viaHelper) + { + if (!Regex.IsMatch(body, $@"\b{Regex.Escape(helperName)}\(")) + { + continue; + } + + if (!wired.TryGetValue(marks[i].Groups[1].Value, out var collectors)) + { + wired[marks[i].Groups[1].Value] = collectors = new SortedSet(StringComparer.Ordinal); + } + + collectors.UnionWith(helperCollectors); + } + } + + /* A helper nobody calls would let the question go unasked while this scan still counted it. */ + foreach (var helperName in viaHelper.Keys) + { + Assert.True( + Enumerable.Range(0, marks.Count).Any(i => + { + var end = i + 1 < marks.Count ? marks[i + 1].Index : source.Length; + return Regex.IsMatch(source[marks[i].Index..end], $@"\b{Regex.Escape(helperName)}\("); + }), + $"{Path.GetFileName(file)}: helper {helperName} asks the capability question but no tool calls it"); + } } return wired; @@ -594,8 +649,13 @@ tempdb_stats stops being a permanent gap. Picking it as the example here would t Assert.Equal("not_collected", DarlingMcpTestData.StatusOf(azureTrace)); Assert.Contains("default_trace_events", azureTrace, StringComparison.Ordinal); - /* ── An Enterprise box, same empty store: every one of them keeps its own miss ── */ - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSystemHealth(postgres, BoxServerName))); + /* ── An Enterprise box, same empty store: every one of them keeps its own miss. For the + health-parser family that own miss is "unavailable" since #3541 A12 (a server whose + system_health session has never been read into the store is not a clean bill), the answer + significant_waits alone used to give and the other eight now share. ── */ + var boxHealth = await DarlingMcpHealthParserTools.GetSystemHealth(postgres, BoxServerName); + Assert.Equal("unavailable", DarlingMcpTestData.StatusOf(boxHealth)); + Assert.Contains("system_health session is started", boxHealth, StringComparison.Ordinal); var boxWaits = await DarlingMcpHealthParserTools.GetSignificantWaits(postgres, BoxServerName); Assert.Equal("unavailable", DarlingMcpTestData.StatusOf(boxWaits)); @@ -703,8 +763,10 @@ off on no engine edition at all — so nothing but the kind axis could ever make Assert.Contains("runs PostgreSQL.", stockFlags, StringComparison.Ordinal); Assert.DoesNotContain("Aurora", stockFlags, StringComparison.Ordinal); - /* ── And the server nobody has probed keeps every one of its old misses ── */ - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSystemHealth(postgres, UnprobedServerName))); + /* ── And the server nobody has probed keeps every one of its old misses — for the health-parser + family that own miss is "unavailable" since #3541 A12 (a never-read session is not a clean + bill); the point here is that it is NOT "not_collected": unknown is not never. ── */ + Assert.Equal("unavailable", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSystemHealth(postgres, UnprobedServerName))); Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpConfigTools.GetTraceFlags(postgres, UnprobedServerName))); var unprobedWaits = await DarlingMcpHealthParserTools.GetSignificantWaits(postgres, UnprobedServerName); @@ -743,7 +805,9 @@ public async Task AServerWithNoProbedEdition_KeepsItsOldMiss() { await RegisterAsync(connection, ct, BoxServerId, BoxServerName, engineEdition: null); - Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSystemHealth(postgres, BoxServerName))); + /* The health parsers' own miss for a never-read session is "unavailable" (#3541 A12) — the + claim under test is that an unprobed edition does not turn it into "not_collected". */ + Assert.Equal("unavailable", DarlingMcpTestData.StatusOf(await DarlingMcpHealthParserTools.GetSystemHealth(postgres, BoxServerName))); Assert.Equal("empty", DarlingMcpTestData.StatusOf(await DarlingMcpDefaultTraceTools.GetDefaultTraceEvents(postgres, BoxServerName))); bodySucceeded = true; diff --git a/Darling/Darling.Tests/FactSourceRegistryTests.cs b/Darling/Darling.Tests/FactSourceRegistryTests.cs new file mode 100644 index 000000000..b294d110c --- /dev/null +++ b/Darling/Darling.Tests/FactSourceRegistryTests.cs @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using ModelContextProtocol.Server; +using PerformanceMonitor.Analysis; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3541 A13: get_analysis_facts' source filter documented four of the engine's fifteen sources +/// and applied an unknown one as an equality filter, so a caller who typed any of the other eleven read +/// [] as "no facts of that kind" for a value that could never have matched. The accepted set is now +/// , published in both SKUs' descriptions and enforced by refusal. +/// +/// A registry is only the truth if nothing can emit a source it does not list, so the register is +/// pinned three ways: against every Source = "..." literal in the three fact-collector assemblies +/// (exact set equality — a new source that lands in a collector without landing here fails HERE, not in an +/// agent's empty result), against the scorer's own switch arms (a subset — two sources carry context and +/// are deliberately not scored), and against the two tools' descriptions and refusals. +/// +public sealed class FactSourceRegistryTests +{ + /// A source literal wherever it is stamped: Source = "waits" on a fact, or the one named + /// constant (AnalysisContext.FactSource = "coverage") the coverage fact is stamped from. + private static readonly Regex SourceLiteral = new(@"Source\s*=\s*""([a-z_]+)""", RegexOptions.Compiled); + + /// The three assemblies whose collectors stamp Fact.Source. The frozen Dashboard is not + /// swept: it is on bug-fix support and its collectors are a copy of Lite's. + private static readonly string[] CollectorDirectories = + { + "PerformanceMonitor.Analysis", + "Lite/Analysis", + "Darling/PerformanceMonitor.Darling.Analysis", + }; + + [Fact] + public void TheRegistry_IsExactlyTheSourcesTheCollectorsEmit() + { + var emitted = new SortedSet(StringComparer.Ordinal); + var filesSeen = 0; + foreach (var directory in CollectorDirectories) + { + var root = RepoFile.PathTo(directory); + foreach (var file in Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories)) + { + filesSeen++; + foreach (Match m in SourceLiteral.Matches(File.ReadAllText(file))) + emitted.Add(m.Groups[1].Value); + } + } + + Assert.True(filesSeen >= 30, $"only {filesSeen} collector sources were swept; the directories have moved"); + Assert.Equal(emitted.ToArray(), FactScorer.KnownSources.ToArray()); + } + + [Fact] + public void TheRegistry_IsSorted_AndLowercaseSnakeCase() + { + Assert.Equal(FactScorer.KnownSources.Order(StringComparer.Ordinal).ToArray(), FactScorer.KnownSources.ToArray()); + Assert.Equal(FactScorer.KnownSources.Distinct(StringComparer.Ordinal).Count(), FactScorer.KnownSources.Count); + Assert.All(FactScorer.KnownSources, s => Assert.Matches("^[a-z_]+$", s)); + /* The count the campaign wrote down was fourteen; coverage (#3538) made fifteen. A moved count is a + moved contract, and the description on both SKUs spells the list out. */ + Assert.Equal(15, FactScorer.KnownSources.Count); + } + + /// Every source the scorer's Layer-1 switch scores is registered. The reverse is deliberately + /// NOT asserted: coverage and sessions are emitted as context with base severity 0. + [Fact] + public void EveryScoredSource_IsRegistered() + { + var scorer = RepoFile.ReadRepoFile("PerformanceMonitor.Analysis", "FactScorer.cs"); + var switchStart = scorer.IndexOf("fact.BaseSeverity = fact.Source switch", StringComparison.Ordinal); + Assert.True(switchStart >= 0, "the scorer's source switch has moved"); + var switchEnd = scorer.IndexOf("};", switchStart, StringComparison.Ordinal); + var arms = Regex.Matches(scorer[switchStart..switchEnd], @"""([a-z_]+)""\s*=>").Select(m => m.Groups[1].Value).ToArray(); + Assert.True(arms.Length >= 10, "the switch-arm scan found too few arms"); + Assert.All(arms, arm => Assert.Contains(arm, FactScorer.KnownSources)); + } + + /// + /// Both SKUs' source descriptions spell out the registry verbatim, in its order — an attribute + /// argument must be a constant, so the pin is what ties the constant to the list. + /// + [Fact] + public void BothDescriptions_SpellOutTheRegistry() + { + var expected = string.Join(", ", FactScorer.KnownSources); + + var darling = typeof(DarlingMcpTools).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == "get_analysis_facts") + .GetParameters().Single(p => p.Name == "source") + .GetCustomAttribute()!.Description; + Assert.Contains(expected, darling, StringComparison.Ordinal); + Assert.Contains("refused otherwise", darling, StringComparison.Ordinal); + + /* Lite's, from source: this project does not reference the desktop app. */ + var lite = RepoFile.ReadRepoFile("Lite", "Mcp", "McpAnalysisTools.cs"); + Assert.Contains(expected, lite, StringComparison.Ordinal); + Assert.Contains("FactSourceFilterDescription", lite, StringComparison.Ordinal); + Assert.Contains("McpHelpers.ValidateChoice(source, FactScorer.KnownSources, \"source\")", lite, StringComparison.Ordinal); + Assert.DoesNotContain("Filter to a specific source category: waits, blocking, config, memory", lite, StringComparison.Ordinal); + } + + [Fact] + public void AnUnknownSource_IsRefusedWithTheWholeSet_AndAKnownOneInAnyCasePasses() + { + var refusal = McpHelpers.ValidateChoice("perfmon", FactScorer.KnownSources, "source"); + Assert.NotNull(refusal); + Assert.StartsWith("Invalid source value 'perfmon'", refusal, StringComparison.Ordinal); + Assert.Contains(string.Join(", ", FactScorer.KnownSources), refusal, StringComparison.Ordinal); + + Assert.Null(McpHelpers.ValidateChoice("waits", FactScorer.KnownSources, "source")); + Assert.Null(McpHelpers.ValidateChoice("Bad_Actor", FactScorer.KnownSources, "source")); + Assert.Null(McpHelpers.ValidateChoice(null, FactScorer.KnownSources, "source")); + Assert.Null(McpHelpers.ValidateChoice(" ", FactScorer.KnownSources, "source")); + } +} diff --git a/Darling/Darling.Tests/FileGrowthRiseUnitCensusTests.cs b/Darling/Darling.Tests/FileGrowthRiseUnitCensusTests.cs new file mode 100644 index 000000000..c23236d28 --- /dev/null +++ b/Darling/Darling.Tests/FileGrowthRiseUnitCensusTests.cs @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using PerformanceMonitor.Alerting; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// #3539 A8c: the File Growth rise threshold means ONE thing — megabytes per hour, averaged over the lookback +/// — and every surface that shows the number says so in the same words. +/// +/// The defect this holds shut. The knob shipped (#2349) as "a file grew at least this many MB +/// inside the lookback window", and the lookback shipped as a second knob clamped 5–1440 minutes. Nothing +/// related the two: the same 10,240 meant 10 GB per five minutes on a store whose operator had shortened the +/// window and 10 GB per day on one who had lengthened it, a 288× swing in what the threshold asked for, and +/// each surface described the number in its own words — "MB within N min" on the Settings rows, +/// "10240MB/60m" on the preview line, "rise ≥ 10240 MB" on the alert, a bare rise_mb on the wire — +/// none of which was wrong on its own and none of which could be compared with another. The fix made the +/// number a rate; this census is what keeps the surfaces from drifting back into private vocabularies, one +/// label at a time. +/// +/// What it asserts. Each surface either references +/// (code) or spells its literal value (XAML, which cannot +/// reference a C# constant without x:Static plumbing this row does not otherwise need), beside the +/// phrase that names the averaging window. And the phrases of the per-window reading are asserted ABSENT, so +/// a revert of one surface is a red build rather than a quiet regression — presence alone would pass with the +/// old label re-added beside the new one, which is the likelier accident. +/// +/// Both SKUs, one roster. The Darling viewer's Settings window is a copy of Lite's, so the two +/// XAML files and the two code-behinds are four rows here rather than two, and a fix that lands on one twin +/// without the other fails by name. The MCP descriptions are the fifth pair: Darling's are asserted by +/// reflection in DarlingMcpAlertToolsTests (where its wire contract is pinned) and by source here, so +/// the roster is complete on its own; Lite's has no reflection pin and is source-only. +/// +public class FileGrowthRiseUnitCensusTests +{ + /// The literal the XAML labels carry. Pinned to the constant so the two cannot drift apart: + /// XAML cannot reference the constant, so the constant's VALUE is the contract the XAML rows below are + /// held to. + [Fact] + public void TheUnitPhrase_IsMbPerHour() + { + Assert.Equal("MB/hr", AlertContextBuilders.FileGrowthRiseUnit); + } + + public static IEnumerable Surfaces() + { + /* Settings rows, both SKUs: the label between the rise box and the lookback box names the unit and + the averaging, and the old "MB within" (which read the box as a per-window total) is gone. */ + foreach (var xaml in new[] + { + "Lite/Windows/SettingsWindow.xaml", + "Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml", + }) + { + yield return new object[] + { + xaml, + new[] { $"Text=\"{AlertContextBuilders.FileGrowthRiseUnit} averaged over\"" }, + new[] { "Text=\"MB within\"" }, + }; + } + + /* Preview lines, both SKUs: "Will alert when: file growth > 10240 MB/hr over 60m" — the constant, + and the old "10240MB/60m" shape gone. */ + foreach (var codeBehind in new[] + { + "Lite/Windows/SettingsWindow.xaml.cs", + "Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml.cs", + }) + { + yield return new object[] + { + codeBehind, + new[] { "{AlertContextBuilders.FileGrowthRiseUnit} over {AlertFileGrowthLookbackMinutesBox.Text}m" }, + new[] { "}MB/{AlertFileGrowthLookbackMinutesBox.Text}m" }, + }; + } + + /* The alert's threshold line: rate, unit, the window it was averaged over, and the bar in megabytes + that rate amounts to inside it. The old "rise ≥ N MB or" is gone. */ + yield return new object[] + { + "PerformanceMonitor.Alerting/AlertEngine.cs", + new[] + { + "{AlertContextBuilders.FileGrowthRiseUnit} averaged over {_settings.FileGrowthLookbackMinutes} min", + "MB in the window)", + }, + new[] { "rise ≥ {_settings.FileGrowthRiseMb} MB or" }, + }; + + /* The card's Growth field — the rate the operator compares against the threshold line. */ + yield return new object[] + { + "PerformanceMonitor.Alerting/AlertContextBuilders.cs", + new[] { "({f.GrowthMbPerHour:F0} {FileGrowthRiseUnit})" }, + new[] { "MB/hr)\")," }, + }; + + /* The engine settings contract — the doc every implementation reads. */ + yield return new object[] + { + "PerformanceMonitor.Alerting/IAlertEngineSettings.cs", + new[] { "at least this many MB PER HOUR, averaged over" }, + new[] { "grew at least this many MB inside the lookback window" }, + }; + + /* The MCP descriptions, both SKUs: the key keeps its spelling, so the unit lives in the description. */ + yield return new object[] + { + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs", + new[] + { + "rise_mb is megabytes per HOUR, averaged over file_growth.lookback_minutes", + "file_growth.rise_mb is megabytes per HOUR averaged over file_growth.lookback_minutes", + }, + Array.Empty(), + }; + yield return new object[] + { + "Lite/Mcp/McpAlertTools.cs", + new[] { "rise_mb is megabytes per HOUR, averaged over file_growth.lookback_minutes" }, + Array.Empty(), + }; + } + + [Theory] + [MemberData(nameof(Surfaces))] + public void EverySurface_SpellsTheRiseAsARate_AndNotAsAPerWindowTotal( + string path, string[] mustContain, string[] mustNotContain) + { + var source = ReadRepoFile(path); + + foreach (var phrase in mustContain) + { + Assert.True( + source.Contains(phrase, StringComparison.Ordinal), + $"{path} no longer says «{phrase}» — the File Growth rise is MB per hour averaged over the lookback, and this surface must say so in the shared words"); + } + + foreach (var phrase in mustNotContain) + { + Assert.False( + source.Contains(phrase, StringComparison.Ordinal), + $"{path} still says «{phrase}», the per-window reading the rate replaced"); + } + } + + /// The roster above is the population; this is the floor under it, so a roster edited down to + /// nothing cannot read as clean. Nine rows: two XAML, two code-behinds, engine, builder, settings contract, + /// two MCP files — counted here rather than trusted. + [Fact] + public void TheRoster_CoversBothSkusAndTheSharedLibrary() + { + var paths = Surfaces().Select(row => (string)row[0]).ToList(); + + Assert.Equal(9, paths.Count); + Assert.Equal(paths.Count, paths.Distinct(StringComparer.Ordinal).Count()); + Assert.Contains(paths, p => p.StartsWith("Lite/", StringComparison.Ordinal)); + Assert.Contains(paths, p => p.StartsWith("Darling/", StringComparison.Ordinal)); + Assert.Contains(paths, p => p.StartsWith("PerformanceMonitor.Alerting/", StringComparison.Ordinal)); + } +} diff --git a/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs b/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs index dcaf2f823..bc35e729e 100644 --- a/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs +++ b/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs @@ -157,13 +157,14 @@ public void TheReasonString_AgreesWithTheFlagsName() var failingButCurrent = Card(TimeSpan.FromSeconds(5), failing: 1); var reason = DarlingFleetReader.BuildReason(failingButCurrent); - Assert.Equal("1 collector failing", reason); + /* "of 40": the helper declares the denominator (#3539 A8d names it when there is one). */ + Assert.Equal("1 of 40 collectors failing", reason); Assert.DoesNotContain("stale", reason, StringComparison.Ordinal); /* A card carrying both reports both, in their own clauses — the axes are additive in the prose exactly as they are in the payload. */ var both = Card(ServerHealthThresholds.StaleThreshold + TimeSpan.FromMinutes(1), failing: 2); - Assert.Equal("2 collectors failing, collection stale", DarlingFleetReader.BuildReason(both)); + Assert.Equal("2 of 40 collectors failing, collection stale", DarlingFleetReader.BuildReason(both)); } /// @@ -310,7 +311,10 @@ private static FleetServerCard Card(TimeSpan sinceLastCollection, int failing) var flags = ServerCollectionStatusRules.FlagsFor( ServerHealthClassifier.ClassifyFreshness(lastCollection, Now)); - var metrics = new ServerHealthMetrics { CpuPercentForAlert = 4, FailedCollectorCount = failing }; + /* Forty collectors banded (#3539 A6): the collectors row is a MEASURED reading, so "nothing failing" + is Healthy here and the two collection axes stay the only variables. */ + const int bandedCollectors = 40; + var metrics = new ServerHealthMetrics { CpuPercentForAlert = 4, FailedCollectorCount = failing, CollectorCount = bandedCollectors }; var overall = ServerHealthClassifier.OverallMetricSeverity(metrics); return new FleetServerCard @@ -325,7 +329,10 @@ private static FleetServerCard Card(TimeSpan sinceLastCollection, int failing) Status = ServerCollectionStatusRules .Classify(flags.IsOnline, flags.CollectionStale, flags.AwaitingFirstCollection).Word(), FailedCollectorCount = failing, - CollectorSeverity = ServerHealthClassifier.CollectorSeverity(failing), + CollectorCount = bandedCollectors, + /* Every failing count this file uses is under the 20% bar of forty, so a failing count is + Warning and never Critical (#3539 A8d) — the two collection axes stay the only variables. */ + CollectorSeverity = ServerHealthClassifier.CollectorSeverity(failing, bandedCollectors), OverallMetricSeverity = overall, Band = ServerHealthClassifier.ClassifyBand( flags.IsOnline, flags.AwaitingFirstCollection, flags.CollectionStale, overall), diff --git a/Darling/Darling.Tests/FleetCardPostgresCpuLivePostgresTests.cs b/Darling/Darling.Tests/FleetCardPostgresCpuLivePostgresTests.cs index 4a70fc60c..b8684e800 100644 --- a/Darling/Darling.Tests/FleetCardPostgresCpuLivePostgresTests.cs +++ b/Darling/Darling.Tests/FleetCardPostgresCpuLivePostgresTests.cs @@ -52,6 +52,23 @@ namespace Darling.Tests; /// nullable columns, where a coalesce anywhere in the read path would turn SQL NULL into a measured 0 and /// claim headroom nobody measured. Neither is visible to an in-memory fixture, which is the whole reason /// this file exists beside the shape pins. +/// +/// And #3539's deadlock arm, on the same fixture. The Aurora target carries a +/// pg_database_stats counter series shaped to defeat every wrong read at once: two databases, one +/// of which steps its lifetime counter 40 → 42 → 42 → 45 (five new deadlocks across one flat interval) and +/// the other of which is RESET mid-window (7 → 7 → 0 → 1: one new deadlock after the reset, and a −7 a +/// naive difference would subtract), plus an out-of-window row far below the series and a second server's +/// series under the same database name. SUM(deadlocks) over the window would answer 184; +/// last-minus-first per database would answer 5 − 6 = −1; an unbounded read would add 37; a read that lost +/// its server partition would fold the other server's 50 in. The right answer is 6, and the card must band +/// it Warning (6/hr is past the shipped 5/hr bar) with the rate published and deadlock_source +/// reading the counter arm because the pg_database_stats collector has a HEALTHY row in the health +/// window. The self-hosted target has the same collector row and a FLAT series (two samples, one +/// difference of zero), so its card is the measured, earned Healthy zero; the silent Aurora target has +/// stats rows and no collector row, so its count is read (50) and its source is CollectorSilent — the +/// engine no longer answers on its own. A PostgreSQL target with NO rows in the window would band Unknown: +/// a difference of nothing is not a zero, which is what keeps #3539 A6's never-collected card measuring +/// nothing, and the unit matrix pins that arm. /// [Collection("live-postgres")] public sealed class FleetCardPostgresCpuLivePostgresTests @@ -148,6 +165,33 @@ await InsertPgCpuAsync(connection, AuroraNoCapacityServerId, AuroraNoCapacityNam /* The SQL Server arm, so "additive only" is asserted against a live read rather than argued. */ await InsertSqlServerCpuAsync(connection, SqlServerServerId, SqlServerName, now.AddMinutes(-1), 30, 4, ct); + /* #3539: the PostgreSQL deadlock arm. The pg_database_stats collector's health row makes the + two targets COVERED; the counter series below is what the count is differenced from. */ + await InsertCollectionLogAsync(connection, AuroraServerId, AuroraName, now.AddSeconds(-30), ct, PgDatabaseStatsCollector.Instance.Name); + await InsertCollectionLogAsync(connection, SelfHostedServerId, SelfHostedName, now.AddSeconds(-30), ct, PgDatabaseStatsCollector.Instance.Name); + /* Database "orders": 40 -> 42 -> 42 -> 45 = 2 + 0 + 3 = 5 new deadlocks. */ + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "orders", now.AddMinutes(-40), 40, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "orders", now.AddMinutes(-30), 42, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "orders", now.AddMinutes(-20), 42, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "orders", now.AddMinutes(-10), 45, ct); + /* Database "billing": 7 -> 7 -> 0 (reset) -> 1 = 0 + clamp(-7) + 1 = 1 new deadlock. */ + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "billing", now.AddMinutes(-40), 7, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "billing", now.AddMinutes(-30), 7, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "billing", now.AddMinutes(-20), 0, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "billing", now.AddMinutes(-10), 1, ct); + /* The covered, MEASURED zero: two samples, one flat difference. Without the second sample the + card would honestly read Unknown - a difference needs two samples - so the fixture supplies + it, and asserts Healthy is EARNED rather than defaulted. */ + await InsertPgDatabaseStatsAsync(connection, SelfHostedServerId, SelfHostedName, "app", now.AddMinutes(-20), 12, ct); + await InsertPgDatabaseStatsAsync(connection, SelfHostedServerId, SelfHostedName, "app", now.AddMinutes(-10), 12, ct); + /* Trap: a row OUTSIDE the window with a much lower counter. A read that did not bound its + window on the start would see 3 -> 40 and add 37. */ + await InsertPgDatabaseStatsAsync(connection, AuroraServerId, AuroraName, "orders", now.AddMinutes(-90), 3, ct); + /* Trap: a series on ANOTHER server, so a read that lost its per-server partition would fold + these into the Aurora card. */ + await InsertPgDatabaseStatsAsync(connection, AuroraSilentServerId, AuroraSilentName, "orders", now.AddMinutes(-30), 100, ct); + await InsertPgDatabaseStatsAsync(connection, AuroraSilentServerId, AuroraSilentName, "orders", now.AddMinutes(-20), 150, ct); + var result = await DarlingFleetReader.GetFleetOverviewAsync( postgres, now.AddHours(-1), now, now, cancellationToken: ct); @@ -181,24 +225,42 @@ makes the band's input and the number beside it describe one minute. */ Assert.Null(aurora.TotalThreads); Assert.Equal(HealthSeverity.Unknown, aurora.ThreadsSeverity); - /* #3272, end to end: the three DMV-sourced bands claim nothing for this engine. Worth a live + /* #3272, end to end: the two DMV-sourced bands claim nothing for this engine. Worth a live arm and not only the unit matrix, because the engine gate reads `servers.engine_kind` out of the store — a column this test wrote and the reader round-tripped, which is the one part of the decision no in-memory fixture exercises. */ Assert.Equal(HealthSeverity.Unknown, aurora.MemorySeverity); Assert.Equal(HealthSeverity.Unknown, aurora.BlockingSeverity); - Assert.Equal(HealthSeverity.Unknown, aurora.DeadlockSeverity); - /* The counts and #3017's disclosure are deliberately untouched, so the fleet total and its - coverage denominator still reconcile. */ Assert.Equal(0, aurora.BlockingCount); - Assert.Equal(0, aurora.DeadlockCount); + + /* #3539, end to end: the deadlock count is the per-database counter DIFFERENCE, clamped across + the reset, summed - 5 + 1 = 6 - and not the 184 a SUM(deadlocks) gives, the -1 a per-database + last-minus-first gives, the 43 an unbounded read gives, or the extra 50 a read that lost its + server partition folds in. Banded through the shared tiers over the one-hour window: 6/hr is + past the shipped 5/hr Warning bar. */ + Assert.Equal(6, aurora.DeadlockCount); + Assert.True(aurora.DeadlockMeasured); + Assert.Equal(6.0, aurora.DeadlockRatePerHour); + Assert.Equal(HealthSeverity.Warning, aurora.DeadlockSeverity); + /* The sample that first showed the newest step - both series stepped at -10 min. */ + Assert.Equal(DateTime.SpecifyKind(now.AddMinutes(-10), DateTimeKind.Unspecified).Ticks / TimeSpan.TicksPerSecond, + aurora.DeadlockLastSeen!.Value.Ticks / TimeSpan.TicksPerSecond); + /* Covered through the counter arm, because pg_database_stats has a health row. */ Assert.Equal(FleetDeadlockSource.PostgresTarget, aurora.DeadlockSource); + Assert.Equal(CollectorHealthClassifier.Healthy, aurora.DeadlockCollectorBand); // ── a PostgreSQL target this build collects no instance CPU for ───────────────────────── var selfHosted = seeded.Single(c => c.ServerId == SelfHostedServerId); Assert.Null(selfHosted.TotalCpuPercent); Assert.Equal(HealthSeverity.Unknown, selfHosted.CpuSeverity); Assert.Equal(FleetCpuSource.NoSourceForEngine, selfHosted.CpuSource); + /* #3539: the covered zero - a running pg_database_stats collector and a flat counter across the + window is a measured Healthy, exactly as a quiet SQL Server's is. */ + Assert.Equal(0, selfHosted.DeadlockCount); + Assert.True(selfHosted.DeadlockMeasured); + Assert.Equal(0.0, selfHosted.DeadlockRatePerHour); + Assert.Equal(HealthSeverity.Healthy, selfHosted.DeadlockSeverity); + Assert.Equal(FleetDeadlockSource.PostgresTarget, selfHosted.DeadlockSource); // ── an Aurora target whose ingest has produced nothing CURRENT ────────────────────────── /* Its only row is 90 minutes old, so the bound excludes it. The arm has to be NotCollected and @@ -207,6 +269,14 @@ coverage denominator still reconcile. */ Assert.Null(silent.TotalCpuPercent); Assert.Equal(HealthSeverity.Unknown, silent.CpuSeverity); Assert.Equal(FleetCpuSource.NotCollected, silent.CpuSource); + /* #3539: this target has stats rows (the partition trap, 100 -> 150) but NO pg_database_stats + health row, so the count is read - 50 - and the coverage says its collector is silent + rather than the pre-#3539 "PostgreSQL, cannot count". Both facts on one card: the count + is what the store holds, the source is what the health window knows. */ + Assert.Equal(50, silent.DeadlockCount); + Assert.True(silent.DeadlockMeasured); + Assert.Equal(HealthSeverity.Critical, silent.DeadlockSeverity); + Assert.Equal(FleetDeadlockSource.CollectorSilent, silent.DeadlockSource); // ── an Aurora target with a CURRENT reading and no capacity sample (#3281) ────────────── /* Unknown, never Healthy, and never the Critical the raw reading alone would earn. The source @@ -270,15 +340,36 @@ INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_engin } private static async Task InsertCollectionLogAsync( - NpgsqlConnection connection, int serverId, string name, DateTime collectionTime, CancellationToken ct) + NpgsqlConnection connection, int serverId, string name, DateTime collectionTime, CancellationToken ct, + string collectorName = "pg_cpu_utilization") { using var command = new NpgsqlCommand(@" INSERT INTO collection_log (log_id, server_id, server_name, collector_name, collection_time, status) -VALUES ($1, $2, $3, 'pg_cpu_utilization', $4, 'SUCCESS')", connection); +VALUES ($1, $2, $3, $5, $4, 'SUCCESS')", connection); command.Parameters.AddWithValue(CollectionIdGenerator.Next()); command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(name); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTime, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(collectorName); + await command.ExecuteNonQueryAsync(ct); + } + + /// Seeds one collect.pg_database_stats row carrying only the deadlock counter (#3539) — + /// the other counters are left NULL, which the read must tolerate (it differences one column). + private static async Task InsertPgDatabaseStatsAsync( + NpgsqlConnection connection, int serverId, string name, string databaseName, + DateTime collectionTime, long deadlocks, CancellationToken ct) + { + using var command = new NpgsqlCommand(@" +INSERT INTO pg_database_stats + (collection_id, collection_time, server_id, server_name, database_name, deadlocks) +VALUES ($1, $2, $3, $4, $5, $6)", connection); + command.Parameters.AddWithValue(CollectionIdGenerator.Next()); + command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTime, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(name); + command.Parameters.AddWithValue(databaseName); + command.Parameters.AddWithValue(deadlocks); await command.ExecuteNonQueryAsync(ct); } @@ -332,7 +423,7 @@ private static async Task DeleteSentinelRowsAsync(NpgsqlConnection connection, C { var ids = string.Join(", ", SentinelIds.Select(i => i.ToString(CultureInfo.InvariantCulture))); - foreach (var table in new[] { "pg_cpu_utilization", "cpu_utilization_stats", "collection_log", "servers" }) + foreach (var table in new[] { "pg_database_stats", "pg_cpu_utilization", "cpu_utilization_stats", "collection_log", "servers" }) { using var cleanup = new NpgsqlCommand($"DELETE FROM {table} WHERE server_id IN ({ids});", connection); await cleanup.ExecuteNonQueryAsync(ct); diff --git a/Darling/Darling.Tests/FleetCardPostgresCpuTests.cs b/Darling/Darling.Tests/FleetCardPostgresCpuTests.cs index fac08d8ef..a41baed4a 100644 --- a/Darling/Darling.Tests/FleetCardPostgresCpuTests.cs +++ b/Darling/Darling.Tests/FleetCardPostgresCpuTests.cs @@ -64,6 +64,7 @@ private static FleetServerCard Card( default, default, default, + default, Now.AddSeconds(-30), default, null, diff --git a/Darling/Darling.Tests/FleetSweepCadenceKnobRungTests.cs b/Darling/Darling.Tests/FleetSweepCadenceKnobRungTests.cs index 1cfd579fc..66fe91f61 100644 --- a/Darling/Darling.Tests/FleetSweepCadenceKnobRungTests.cs +++ b/Darling/Darling.Tests/FleetSweepCadenceKnobRungTests.cs @@ -344,10 +344,13 @@ would otherwise still present the right value at one of the two positions. */ .Max(); Assert.Equal(command.Parameters.Count, highestPlaceholder); - /* The two new columns ride at the END — appended, the rule every knob rung on this table follows, - so every earlier ordinal keeps its column. */ - Assert.False(Assert.IsType>(command.Parameters[^2]).TypedValue); - Assert.Equal(240, Assert.IsType>(command.Parameters[^1]).TypedValue); + /* The two columns ride at THEIR appended ordinals ($65/$66) — fixed forever by the append rule, + which is what keeps every earlier ordinal on its column. Not `[^1]`/`[^2]` any more: that + end-anchored form asserted these are the NEWEST bound columns, which stopped being true when the + viewer pass appended V126's floor ($67) — the top-of-bind claim moved to + SelfDiskWarnGbFloorRungTests the way the probe's top-arm claims hand off between rung files. */ + Assert.False(Assert.IsType>(command.Parameters[64]).TypedValue); + Assert.Equal(240, Assert.IsType>(command.Parameters[65]).TypedValue); } /// Non-overlapping occurrences of . diff --git a/Darling/Darling.Tests/FleetSweepEngineTests.cs b/Darling/Darling.Tests/FleetSweepEngineTests.cs index 5de912823..1ff0aa7c9 100644 --- a/Darling/Darling.Tests/FleetSweepEngineTests.cs +++ b/Darling/Darling.Tests/FleetSweepEngineTests.cs @@ -41,11 +41,17 @@ the window states its own start instant instead. */ private static FleetSweepInstrumentCounters SteadyInstruments(long passes = 500) => new(StartedLongAgo, passes, AlertReadFailuresTotal: 0); + /* Every fixture's signals carry the sweep span ComposeSimple declares (1h) — the deadlock band is a + RATE over that window (#3525), so a fixture omitting the window would take the unrateable arm and + max out at Warning. */ + private static readonly TimeSpan FixtureSpan = TimeSpan.FromHours(1); + private static FleetSweepServerReading Healthy(int id, string name) => - new(id, name, new DailyHealthSignals { HasData = true }, 0, null); + new(id, name, new DailyHealthSignals { HasData = true, Window = FixtureSpan }, 0, null); - private static FleetSweepServerReading CriticalDeadlocks(int id, string name, long deadlocks = 3) => - new(id, name, new DailyHealthSignals { HasData = true, Deadlocks = deadlocks }, 0, null); + /* 25 deadlocks over the 1-hour span = 25/hr, past the shipped Critical tier (20/hr). */ + private static FleetSweepServerReading CriticalDeadlocks(int id, string name, long deadlocks = 25) => + new(id, name, new DailyHealthSignals { HasData = true, Deadlocks = deadlocks, Window = FixtureSpan }, 0, null); private static FleetSweepServerReading NoData(int id, string name) => new(id, name, default, 0, null); @@ -57,7 +63,8 @@ private static FleetSweepComposition ComposeSimple( IReadOnlyList? previousVerdicts = null, IReadOnlyList? activeItems = null, FleetSweepInstrumentCounters? instruments = null, - DateTime? now = null) + DateTime? now = null, + DeadlockRateThresholds? deadlockTiers = null) { return FleetSweepEngine.Compose( now ?? Now, @@ -68,7 +75,8 @@ private static FleetSweepComposition ComposeSimple( previousRun, previousVerdicts ?? Array.Empty(), activeItems ?? Array.Empty(), - instruments ?? SteadyInstruments()); + instruments ?? SteadyInstruments(), + deadlockTiers ?? DeadlockRateThresholds.Default); } /* ─────────────────────── verdicts: the shared scorer, unforked ─────────────────────── */ @@ -103,9 +111,16 @@ public void Verdicts_ComeFromTheSharedScorer_WithItsOwnReasons() Assert.Null(verdictB.BandReason); /* Verdict beside its inputs: the evidence payload carries the signals, so a reader can - disagree with the band rather than believe it. */ + disagree with the band rather than believe it — including, since #3525, the rate the deadlock + signal banded on and the window it was normalised over, without which the deadlock count is + unfalsifiable. */ Assert.NotNull(verdictA.VerdictJson); - Assert.Contains("\"deadlocks\":3", verdictA.VerdictJson, StringComparison.Ordinal); + Assert.Contains("\"deadlocks\":25", verdictA.VerdictJson, StringComparison.Ordinal); + Assert.Contains("\"deadlock_rate_per_hour\":25", verdictA.VerdictJson, StringComparison.Ordinal); + Assert.Contains("\"window_minutes\":60", verdictA.VerdictJson, StringComparison.Ordinal); + /* #3539 A2/A3, additive: the error share's denominator and the blocking rate beside its count. */ + Assert.Contains("\"collection_runs\":0", verdictA.VerdictJson, StringComparison.Ordinal); + Assert.Contains("\"blocking_rate_per_hour\":0", verdictA.VerdictJson, StringComparison.Ordinal); } /* ─────────────────────── the diff: sweep N against sweep N−1's rows ─────────────────────── */ @@ -365,9 +380,10 @@ public void UnderMasterOff_TheSweepProducesRows_AndTheLedgerCarriesTheWouldHaveP new FleetSweepServerReading(1, "server-a", new DailyHealthSignals { HasData = true, - Deadlocks = 2, + Deadlocks = 25, // 25/hr over the 1h span — past the Critical tier (#3525) HighCpuEvents = 10, BlockingEvents = 20, + Window = FixtureSpan, }, 1500, null), Healthy(2, "server-b"), }; @@ -398,6 +414,222 @@ crossed their own summary-scoring critical trigger. */ Assert.True(unmuted.Run.AlertsEnabled); } + /* ─────────────────────── the deadlock family is the RATE's, not the count's (#3525) ─────────────────────── */ + + /// + /// The would-have-paged deadlock family fires on the RATE the shared scorer banded Critical with — + /// never on a bare count. A day Critical from another trigger, carrying deadlocks below the Critical + /// tier, writes no deadlock row: under the old count trigger its threshold was literally 1, so every + /// sweep span containing any deadlock claimed a page the new banding does not stand behind. + /// + [Fact] + public void TheDeadlockFamily_FiresOnTheRate_NotTheCount() + { + /* Critical via heavy blocking; 3 deadlocks over the 1h span is 3/hr — Healthy on the deadlock + band, so the ledger must not name the family. */ + var subRate = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + Deadlocks = 3, + BlockingEvents = 20, + Window = FixtureSpan, + }, 0, null); + + var families = ComposeSimple(new[] { subRate }, alertsEnabled: false) + .WouldHavePaged.Select(w => w.AlertFamily).ToList(); + Assert.Contains(FleetSweepEngine.FamilyBlocking, families); + Assert.DoesNotContain(FleetSweepEngine.FamilyDeadlocks, families); + + /* And when the family DOES fire, its evidence names the rate as the value, the Critical tier as + the threshold, and the raw count beside them — figures an operator can audit against + get_alert_settings and the deadlock grid. */ + var paged = ComposeSimple(new[] { CriticalDeadlocks(1, "server-a") }, alertsEnabled: false) + .WouldHavePaged.Single(w => w.AlertFamily == FleetSweepEngine.FamilyDeadlocks); + using var evidence = JsonDocument.Parse(paged.EvidenceJson); + Assert.Equal("deadlocks per hour over the sweep span", evidence.RootElement.GetProperty("trigger").GetString()); + Assert.Equal(25.0, evidence.RootElement.GetProperty("value").GetDouble()); + Assert.Equal( + ServerHealthThresholds.DeadlockCriticalPerHourDefault, + evidence.RootElement.GetProperty("threshold").GetDouble()); + Assert.Equal(25, evidence.RootElement.GetProperty("deadlock_count").GetInt64()); + } + + /// + /// A sub-hour sweep span is not rateable (#3368's arm), so deadlocks alone cannot band the span + /// Critical — and even when ANOTHER trigger makes the day Critical, the deadlock family stays out of + /// the ledger: 10,000 deadlocks in 15 minutes is 40,000/hr arithmetically, and declining to claim it + /// is the honest reading the whole rate band is built on. The other trigger here is severe memory + /// pressure, the one presence-banded Critical left after #3539 A2. + /// + [Fact] + public void ASubHourSpan_NeverPagesTheDeadlockFamily() + { + var reading = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + Deadlocks = 10_000, + MemoryCriticalEvents = 1, + Window = TimeSpan.FromMinutes(15), + }, 0, null); + + var composition = ComposeSimple(new[] { reading }, alertsEnabled: false); + + Assert.Equal("Critical", composition.Verdicts.Single().Band); + var families = composition.WouldHavePaged.Select(w => w.AlertFamily).ToList(); + Assert.Contains(FleetSweepEngine.FamilyMemoryCritical, families); + Assert.DoesNotContain(FleetSweepEngine.FamilyDeadlocks, families); + + /* Deadlocks alone on the same span: Warning, not Critical — the unrateable arm. */ + var alone = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + Deadlocks = 10_000, + Window = TimeSpan.FromMinutes(15), + }, 0, null); + Assert.Equal("Warning", ComposeSimple(new[] { alone }).Verdicts.Single().Band); + } + + /* ─────────────────────── #3539 A2/A3: the CPU, blocking and collection-error triggers ─────────────────────── */ + + /// + /// The collection-error family produces NO new ledger rows: the arm is a share of the span's runs with + /// a Warning ceiling now, so no Critical verdict can be attributed to it. A span whose every run + /// errored is Warning, never Critical, and the ledger (Critical triggers only) is empty for it. The + /// family constant stays as vocabulary for rows already stored. + /// + [Fact] + public void TheCollectionErrorFamily_NeverPages_AndAnAllErrorSpanIsWarning() + { + var allErrors = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + CollectionErrors = 900, + CollectionRuns = 900, + Window = FixtureSpan, + }, 0, null); + + var composition = ComposeSimple(new[] { allErrors }, alertsEnabled: false); + Assert.Equal("Warning", composition.Verdicts.Single().Band); + Assert.Empty(composition.WouldHavePaged); + Assert.Contains("900 collection errors (100.0% of 900 runs)", composition.Verdicts.Single().BandReason, StringComparison.Ordinal); + + /* One transient error among the span's runs: below the 20% bar, disclosed, and the span is Healthy. */ + var oneError = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + CollectionErrors = 1, + CollectionRuns = 900, + Window = FixtureSpan, + }, 0, null); + Assert.Equal("Healthy", ComposeSimple(new[] { oneError }).Verdicts.Single().Band); + Assert.Equal("collection-errors", FleetSweepEngine.FamilyCollectionErrors); + } + + /// + /// The CPU family fires on the arm the verdict banded with — the hot-sample count against the bar + /// SCALED to the span. Over the hourly span six is the bar (unchanged from the old constant); over a + /// day-long span the same ten samples are far under the 30 the sustained-heat rate demands, so a + /// day-ceiling sweep does not page on what an hourly one would — and the evidence names the bar it + /// used. + /// + [Fact] + public void TheCpuFamily_FiresAgainstTheSpanScaledBar() + { + var hourly = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + HighCpuEvents = 10, + Window = FixtureSpan, + }, 0, null); + var paged = ComposeSimple(new[] { hourly }, alertsEnabled: false) + .WouldHavePaged.Single(w => w.AlertFamily == FleetSweepEngine.FamilyHighCpu); + using (var evidence = JsonDocument.Parse(paged.EvidenceJson)) + { + Assert.Equal(10, evidence.RootElement.GetProperty("value").GetInt64()); + Assert.Equal(6.0, evidence.RootElement.GetProperty("threshold").GetDouble()); + } + + var daily = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + HighCpuEvents = 10, + Window = TimeSpan.FromHours(24), + }, 0, null); + var dayComposition = ComposeSimple(new[] { daily }, alertsEnabled: false); + Assert.Equal("Warning", dayComposition.Verdicts.Single().Band); + Assert.Empty(dayComposition.WouldHavePaged); + } + + /// + /// The blocking family fires on the same BlockingSeverity call Classify makes, and its evidence names + /// the arm that decided: the rate row carries the per-hour rate against the 20/hr tier with the raw + /// count beside it; a 60-second block is the wait arm's row whatever the rate. Twenty events in an + /// hour is the Critical tier; twenty in a day is 0.8/hr and pages nothing. + /// + [Fact] + public void TheBlockingFamily_FiresOnTheRateOrTheWaitArm_AndNamesWhich() + { + var storm = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + BlockingEvents = 20, + Window = FixtureSpan, + }, 0, null); + var rateRow = ComposeSimple(new[] { storm }, alertsEnabled: false) + .WouldHavePaged.Single(w => w.AlertFamily == FleetSweepEngine.FamilyBlocking); + using (var evidence = JsonDocument.Parse(rateRow.EvidenceJson)) + { + Assert.Equal("blocking events per hour over the sweep span", evidence.RootElement.GetProperty("trigger").GetString()); + Assert.Equal(20.0, evidence.RootElement.GetProperty("value").GetDouble()); + Assert.Equal(ServerHealthThresholds.BlockingCriticalPerHour, evidence.RootElement.GetProperty("threshold").GetDouble()); + Assert.Equal(20, evidence.RootElement.GetProperty("blocking_count").GetInt64()); + } + + var longBlock = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + BlockingEvents = 1, + PeakBlockWaitMs = 90_000, + Window = TimeSpan.FromMinutes(15), + }, 90_000, null); + var waitRow = ComposeSimple(new[] { longBlock }, alertsEnabled: false) + .WouldHavePaged.Single(w => w.AlertFamily == FleetSweepEngine.FamilyBlocking); + using (var evidence = JsonDocument.Parse(waitRow.EvidenceJson)) + { + Assert.Equal("longest single block in the sweep span, seconds", evidence.RootElement.GetProperty("trigger").GetString()); + Assert.Equal(90.0, evidence.RootElement.GetProperty("value").GetDouble()); + Assert.Equal(ServerHealthThresholds.BlockingCriticalWaitSeconds, evidence.RootElement.GetProperty("threshold").GetDouble()); + } + + var diluted = new FleetSweepServerReading(1, "server-a", new DailyHealthSignals + { + HasData = true, + BlockingEvents = 20, + Window = TimeSpan.FromHours(24), + }, 0, null); + var dayComposition = ComposeSimple(new[] { diluted }, alertsEnabled: false); + Assert.Equal("Healthy", dayComposition.Verdicts.Single().Band); + Assert.Empty(dayComposition.WouldHavePaged); + } + + /// + /// The tiers handed to Compose are the tiers the verdicts band on (#3525) — the store's pair + /// travels into the shared scorer, so a fleet whose knobs were raised sweeps on the raised pair + /// rather than the shipped one while get_alert_settings reports the raised numbers. + /// + [Fact] + public void TheVerdicts_BandOnTheTiersHandedIn() + { + var reading = CriticalDeadlocks(1, "server-a"); // 25/hr: Critical on the shipped pair + + Assert.Equal("Critical", ComposeSimple(new[] { reading }).Verdicts.Single().Band); + + var raised = new DeadlockRateThresholds(100.0, 500.0); + var onRaised = ComposeSimple(new[] { reading }, alertsEnabled: false, deadlockTiers: raised); + Assert.Equal("Healthy", onRaised.Verdicts.Single().Band); + Assert.Empty(onRaised.WouldHavePaged); + } + /// /// The stored DOCUMENT carries the ledger key exactly when the check ran (#3478): absent on an /// alerts-on sweep, because the would-have-paged derivation never executed there and a stored diff --git a/Darling/Darling.Tests/FleetSweepRollupTests.cs b/Darling/Darling.Tests/FleetSweepRollupTests.cs index 179199a55..10e479824 100644 --- a/Darling/Darling.Tests/FleetSweepRollupTests.cs +++ b/Darling/Darling.Tests/FleetSweepRollupTests.cs @@ -95,6 +95,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private sealed class RecordingHistoryStore : IAlertHistoryStore diff --git a/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs new file mode 100644 index 000000000..9062e2188 --- /dev/null +++ b/Darling/Darling.Tests/ForcePlanFailuresAccessPathTests.cs @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3573: the alerting pass's forced-plan-failures read and the covering index that gives it an access path +/// the planner will actually take. +/// +/// What went wrong is not that an index was missing. V1 generates +/// idx_query_store_stats_time (server_id, collection_time), the exact composite the read's predicate +/// wants, and the production catalog carried it on the hypertable and on the chunk in the failing plan. The +/// planner priced it out: server_id has near-zero physical correlation (43 servers interleaved by +/// collection pass), so the cost model charged one random page per tuple for the composite's heap fetches +/// and preferred streaming the entire fleet's two-hour slice through the perfectly-correlated time index +/// and filtering 95% of it away. Forced under random_page_cost = 1.1 the same statement took the +/// composite and touched 5,063 buffers instead of 57,307 \u2014 the composite was right all along. A second plain +/// composite would have been priced, and ignored, identically. +/// +/// Covering is the fix because it deletes the term the cost model got wrong. With every column +/// the read touches in the key or INCLUDE, the plan is an Index Only Scan with no heap component to misprice, +/// at any random_page_cost and at any share of the fleet the busiest server grows into. That makes the +/// INCLUDE list load-bearing in a way most index definitions are not: a column added to the read and not to +/// the index does not fail anything \u2014 it silently hands the read back to the fleet-wide plan. The ungated pins +/// below hold the two against each other from source; the gated one asks the planner. +/// +/* Live-fixture tests share one Postgres store; the collection serializes them so cross-test row churn + cannot race another class's assertions. */ +[Collection("live-postgres")] +public sealed class ForcePlanFailuresAccessPathTests +{ + /// Distinctive fake ids \u2014 a real server_id is a storage-name hash, never these. The target is + /// one of six so its rows are ~17% of the seeded chunk, a share at which a seq scan is not competitive. + private const int TargetServerId = -735730; + private static readonly int[] OtherServerIds = { -735731, -735732, -735733, -735734, -735735 }; + private const string TestServerName = "force-plan-access-path-e2e"; + + /// + /// Every qs.<column> the shipped statement references, read from the statement itself. The + /// alias is fixed by the SQL, so this is the complete set of columns the scan must produce. + /// + private static IReadOnlyCollection ColumnsTheReadReferences() + { + return Regex.Matches(DarlingAlertReadAdapter.ForcePlanFailuresSql, @"\bqs\.([a-z_]+)") + .Select(m => m.Groups[1].Value) + .Distinct(StringComparer.Ordinal) + .OrderBy(c => c, StringComparer.Ordinal) + .ToList(); + } + + /// + /// The read's column references and the index's column list are the SAME set, both ways. A column the + /// read touches that the index lacks degrades the Index Only Scan to the heap plan it replaced, silently; + /// a column the index carries that the read no longer touches is dead weight on every insert into the + /// largest table in the store (INCLUDE disables deduplication, so each is real bytes per row). + /// + [Fact] + public void TheCoveringIndex_CarriesExactlyTheColumnsTheReadReferences() + { + var referenced = ColumnsTheReadReferences(); + var carried = PgTableTuning.ForcePlanFailuresIndexColumns.OrderBy(c => c, StringComparer.Ordinal).ToList(); + + Assert.Equal(carried, referenced); + + /* The predicate columns are the KEY, in predicate order: the equality column first so one server's + rows are one contiguous index range, the range column second. Everything else is INCLUDE. */ + Assert.Equal("server_id", PgTableTuning.ForcePlanFailuresIndexColumns[0]); + Assert.Equal("collection_time", PgTableTuning.ForcePlanFailuresIndexColumns[1]); + } + + /// + /// The statement text is BUILT from the same column list the pin above holds, so the two cannot drift: the + /// literal SQL carries the key as (server_id, collection_time DESC) and the remaining columns, in + /// order, as INCLUDE. Also pins the decisions the rig measured (see ): idempotent + /// IF NOT EXISTS; NOT CONCURRENTLY, which hypertables refuse; NOT + /// timescaledb.transaction_per_chunk, whose mid-build cancel leaves an invalid parent index that the + /// idempotent re-run then skips forever; and collect.-qualified like every neighbour. + /// + [Fact] + public void TheStatement_IsBuiltFromTheColumnList_AndTakesNeitherMeasuredTrap() + { + var statements = PgTableTuning.Statements + .Where(s => s.Contains(PgTableTuning.ForcePlanFailuresIndexName, StringComparison.Ordinal)) + .ToList(); + var statement = Assert.Single(statements); + + var columns = PgTableTuning.ForcePlanFailuresIndexColumns; + var expected = + "CREATE INDEX IF NOT EXISTS " + PgTableTuning.ForcePlanFailuresIndexName + + " ON collect.query_store_stats (" + columns[0] + ", " + columns[1] + " DESC)" + + " INCLUDE (" + string.Join(", ", columns.Skip(2)) + ")"; + Assert.Equal(expected, statement); + + Assert.DoesNotContain("CONCURRENTLY", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("transaction_per_chunk", statement, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(" WHERE ", statement, StringComparison.Ordinal); /* not partial \u2014 see the statement's remarks */ + } + + /// + /// #3579: the observation stamp is the LAST column of the shipped read and is n.collection_time — the + /// newer sighting's collector clock — not a new qs. reference. Last, because the reader binds ordinals + /// 0–6 to the seven pre-#3579 columns and an inserted column would silently shift every one of them onto + /// its neighbour's type (a string read as a bigint fails; a bigint read as a bigint from the wrong column + /// does not). Not a qs. reference, because the covering pin above re-derives the index list from + /// exactly those references and a new one would demand a new INCLUDE column on the largest table in the + /// store; collection_time is already in the key. + /// + [Fact] + public void TheObservationStamp_IsTheLastColumn_AndIsTheNewerSightingsCollectionTime() + { + var sql = DarlingAlertReadAdapter.ForcePlanFailuresSql; + var selectList = sql[sql.LastIndexOf("SELECT", StringComparison.Ordinal)..sql.IndexOf("FROM ranked AS n", StringComparison.Ordinal)]; + var columns = selectList.Replace("SELECT", "", StringComparison.Ordinal) + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + + Assert.Equal(8, columns.Length); + Assert.Equal("n.failures AS total_failures", columns[6]); + Assert.Equal("n.collection_time AS observed_at", columns[7]); + + /* The set of scan columns did not grow — the same nine the index carried before #3579. */ + Assert.Equal(PgTableTuning.ForcePlanFailuresIndexColumns.Count, ColumnsTheReadReferences().Count); + } + + /// + /// The evidence no string pin can give: that the planner TAKES the index for the shipped statement. The + /// production failure was a plan choice, not a missing object \u2014 the right composite was in the catalog and + /// the plan walked past it \u2014 so a test that only checked pg_indexes would have passed on the broken + /// store. This builds the store the way the service does (ladder, then the hypertable conversion where + /// TimescaleDB is present, then ), seeds six servers' rows in the + /// collector's per-pass contiguous batches, and EXPLAINs the exact shipped SQL with its real bound + /// parameters: the plan must be an Index Only Scan on the covering index and must NOT be the time-index + /// scan filtering on server_id that the issue's plan showed. + /// + [Fact] + public async Task TheShippedRead_PlansAsAnIndexOnlyScanOnTheCoveringIndex_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live access-path test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + /* The service's own order: hypertables first (where the extension exists), so the index is created on + a hypertable and propagates to chunks, then the tuning list. On a plain-PostgreSQL store the index is + an ordinary btree and the plan assertion below holds the same way. #1922: probe on its own connection. */ + var timescaleEnabled = await LiveTimescaleProbe.TryEnableAsync(connectionString!, ct); + if (timescaleEnabled) + { + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + } + + await PgTableTuning.ApplyAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + await DeleteTestRowsAsync(connection, ct); + + /* The index exists on the hypertable with the shipped definition. */ + using (var indexDef = new NpgsqlCommand( + "SELECT indexdef FROM pg_indexes WHERE schemaname = 'collect' AND tablename = 'query_store_stats' AND indexname = $1", connection)) + { + indexDef.Parameters.AddWithValue(PgTableTuning.ForcePlanFailuresIndexName); + var def = await indexDef.ExecuteScalarAsync(ct) as string; + Assert.NotNull(def); + Assert.Contains("(server_id, collection_time DESC)", def, StringComparison.Ordinal); + Assert.Contains("INCLUDE (" + string.Join(", ", PgTableTuning.ForcePlanFailuresIndexColumns.Skip(2)) + ")", def, StringComparison.Ordinal); + } + + /* Eight passes fifteen minutes apart, all inside the read's two-hour window; each pass writes the six + servers in turn, each server's batch contiguous \u2014 the write pattern that makes server_id's + correlation near zero, which is the condition the production plan was chosen under. All + Kind-Unspecified: naive-UTC storage, see PgCollectorRowWriter. */ + /* Floored to whole microseconds: PostgreSQL timestamp is microsecond-resolution and .NET ticks are + 100 ns, so a raw UtcNow does not survive the round trip and the #3579 stamp assertion below + (tick-equality against what was seeded) would fail on any clock that is not itself + microsecond-aligned — Windows' is not; the first CI run proved it by three ticks. */ + var rawNow = DateTime.UtcNow; + var utcNow = DateTime.SpecifyKind(new DateTime(rawNow.Ticks - (rawNow.Ticks % 10)), DateTimeKind.Unspecified); + for (var pass = 7; pass >= 0; pass--) + { + var collectionTime = utcNow.AddMinutes(-2 - pass * 15); + foreach (var serverId in OtherServerIds.Take(3).Append(TargetServerId).Concat(OtherServerIds.Skip(3))) + { + /* The target's forced plan climbs one failure per pass (pass 7 = 0 ... pass 0 = 7). */ + var forcedFailures = serverId == TargetServerId ? 7L - pass : (long?)null; + await SeedPassAsync(connection, serverId, collectionTime, rows: 400, forcedFailures, ct); + } + } + + /* The planner needs the visibility map current (the product keeps it so with the insert-autovacuum + override; a test cannot wait for autovacuum) and statistics for the rows just written. */ + using (var vacuum = new NpgsqlCommand("VACUUM ANALYZE collect.query_store_stats", connection)) + { + await vacuum.ExecuteNonQueryAsync(ct); + } + + var plan = await ExplainShippedReadAsync(connection, utcNow - DarlingAlertReadAdapter.ForcePlanFailureWindow, ct); + + /* The chunk copy of a hypertable index is named _, truncated to 63 characters, so match + on the name's stable prefix rather than its whole. */ + Assert.Contains("Index Only Scan", plan, StringComparison.Ordinal); + Assert.Contains("idx_query_store_stats_server_time", plan, StringComparison.Ordinal); + Assert.DoesNotContain("collection_time_idx", plan, StringComparison.Ordinal); + /* The issue's signature: server_id applied as a post-scan Filter (any alias, any parenthesisation) + rather than inside the Index Cond. */ + Assert.False(Regex.IsMatch(plan, @"Filter: \(+(qs(_\d+)?\.)?server_id"), + "server_id is being applied as a Filter after the scan — the fleet-wide plan is back:\n" + plan); + + /* And the read still answers correctly through the new path: the target's forced plan's counter + rose between its two newest sightings (pass 1 -> pass 0), and nothing else did. */ + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var adapter = new DarlingAlertReadAdapter(postgres); + var failures = await adapter.GetForcePlanFailuresAsync(TargetServerId.ToString(CultureInfo.InvariantCulture), ct); + var failure = Assert.Single(failures); + Assert.Equal("ForcedDb", failure.DatabaseName); + Assert.Equal(1L, failure.QueryId); + Assert.Equal(10L, failure.PlanId); + Assert.Equal(1L, failure.FailureDelta); + Assert.Equal(7L, failure.TotalFailures); + /* #3579: the observation's identity is the NEWEST sighting's collection_time (pass 0, two minutes + ago), read back through the real Npgsql path and stamped Utc. Ticks-equal to what was seeded: + the store holds naive UTC and the adapter only names the Kind, never shifts the value. */ + Assert.Equal((DateTime?)utcNow.AddMinutes(-2), failure.ObservedAtUtc); + Assert.Equal(DateTimeKind.Utc, failure.ObservedAtUtc!.Value.Kind); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteTestRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// One server's batch for one pass: ordinary rows plus, when + /// is given, one forced plan carrying that force_failure_count, so + /// the caller can make the newest two sightings differ by exactly one. Written as a single multi-row INSERT + /// so the batch lands as one contiguous run of heap pages, the way the collector's COPY does. + /// + private static async Task SeedPassAsync(NpgsqlConnection connection, int serverId, DateTime collectionTime, int rows, long? forcedFailures, CancellationToken ct) + { + var sql = new StringBuilder( + "INSERT INTO collect.query_store_stats (collection_id, collection_time, server_id, server_name, database_name, query_id, plan_id, " + + "execution_count, avg_duration_us, plan_forcing_type, is_forced_plan, force_failure_count, last_force_failure_reason) " + + "SELECT $1, $2, $3, $4, 'db_' || (g % 4), 1000 + g, (1000 + g) * 10, 10 + g, 500 + g, 'NONE', FALSE, 0, NULL " + + "FROM generate_series(1, $5) AS g"); + using (var insert = new NpgsqlCommand(sql.ToString(), connection)) + { + insert.Parameters.AddWithValue(1L); + insert.Parameters.AddWithValue(collectionTime); + insert.Parameters.AddWithValue(serverId); + insert.Parameters.AddWithValue(TestServerName); + insert.Parameters.AddWithValue(rows); + await insert.ExecuteNonQueryAsync(ct); + } + + if (forcedFailures is null) + { + return; + } + + using var forced = new NpgsqlCommand( + "INSERT INTO collect.query_store_stats (collection_id, collection_time, server_id, server_name, database_name, query_id, plan_id, " + + "execution_count, avg_duration_us, plan_forcing_type, is_forced_plan, force_failure_count, last_force_failure_reason) " + + "VALUES ($1, $2, $3, $4, 'ForcedDb', 1, 10, 5, 900, 'MANUAL', TRUE, $5, 'GENERAL_FAILURE')", connection); + forced.Parameters.AddWithValue(1L); + forced.Parameters.AddWithValue(collectionTime); + forced.Parameters.AddWithValue(serverId); + forced.Parameters.AddWithValue(TestServerName); + forced.Parameters.AddWithValue(forcedFailures.Value); + await forced.ExecuteNonQueryAsync(ct); + } + + private static async Task ExplainShippedReadAsync(NpgsqlConnection connection, DateTime windowStart, CancellationToken ct) + { + using var explain = new NpgsqlCommand("EXPLAIN (COSTS OFF) " + DarlingAlertReadAdapter.ForcePlanFailuresSql, connection); + explain.Parameters.AddWithValue(TargetServerId); + explain.Parameters.AddWithValue(DateTime.SpecifyKind(windowStart, DateTimeKind.Unspecified)); + var plan = new StringBuilder(); + using var reader = await explain.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + plan.AppendLine(reader.GetString(0)); + } + + return plan.ToString(); + } + + private static async Task DeleteTestRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + var ids = string.Join(", ", OtherServerIds.Append(TargetServerId).Select(id => id.ToString(CultureInfo.InvariantCulture))); + using var cleanup = new NpgsqlCommand($"DELETE FROM collect.query_store_stats WHERE server_id IN ({ids})", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/IntervalDedupMaterializationIndexesTests.cs b/Darling/Darling.Tests/IntervalDedupMaterializationIndexesTests.cs new file mode 100644 index 000000000..ad7dcbe81 --- /dev/null +++ b/Darling/Darling.Tests/IntervalDedupMaterializationIndexesTests.cs @@ -0,0 +1,266 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3597: the interval-dedup L1 aggregate () +/// carries no per-GROUP-BY-column index on its materialization, and an existing store is brought to that +/// shape at startup. +/// +/// What the rig measured, restated here because the pins below only make sense against it. +/// TimescaleDB's default create_group_indexes built eleven (column, bucket DESC) btrees on +/// L1's materialization beside the bucket index. Nothing reads L1 by any of those columns — its three +/// child aggregates refresh over it by bucket range, the coverage probe and the arming gate read +/// min(bucket), retention drops chunks — but every hourly refresh re-materializes a bucket by +/// DELETE + INSERT, and each inserted row cost twelve index inserts. EXPLAIN (ANALYZE, BUFFERS, WAL) +/// of one bucket's materialization INSERT at one tenth of the largest store's scale: 45.5 MB of WAL and +/// 1.64 M buffer touches with the group indexes, 10.7 MB and 447 K with the bucket index alone. The +/// refresh, the child refreshes, compress_chunk and decompress_chunk all ran unchanged +/// without them. +/// +/// Why the pins are scoped to L1 and only L1. The option is earned by that measurement, not +/// applied for symmetry: the composer-grain rollups ARE read through their group indexes (by +/// server_id, by query_hash), and the day-grain L2 refreshes once a day and was not measured. +/// A second aggregate wanting create_group_indexes = false should arrive with its own rig figures and +/// move the scope pin deliberately. +/// +/// #1776 own-store — the gated arm mints a scratch database (it creates the continuous +/// aggregates the shared fixture deliberately leaves to the tests, plants and drops an index on one of +/// their materializations, and refreshes over seeded rows), so it cannot race the shared store and is not +/// serialized against the live-postgres collection. The same shape as +/// and QueryStoreCorrectedRollupLiveTests. +/// +public sealed class IntervalDedupMaterializationIndexesTests +{ + private const string NoGroupIndexes = "timescaledb.create_group_indexes = false"; + + [Fact] + public void L1_IsCreatedWithoutGroupIndexes() + { + var sql = TimescaleSupport.CreateQueryStoreStatsIntervalHourlySql; + + Assert.Contains("WITH (timescaledb.continuous, " + NoGroupIndexes + ") AS", sql, StringComparison.Ordinal); + /* Still materialized-only (#1759): the option rides beside `continuous`, it does not displace the + absence TimescaleContinuousAggregateTests pins for the query-acceleration tier. */ + Assert.DoesNotContain("materialized_only", sql, StringComparison.Ordinal); + } + + /// + /// The scope pin: every OTHER registered aggregate keeps TimescaleDB's default. Enumerated from the three + /// registries the ensure sweep builds from, so a new aggregate is covered the day it is registered. + /// + [Fact] + public void EveryOtherAggregate_KeepsTheDefaultGroupIndexes_UntilMeasured() + { + var others = TimescaleSupport.HourlyAggregates + .Concat(TimescaleSupport.DailyAggregates) + .Concat(TimescaleSupport.BaselineAggregates) + .Where(a => !string.Equals(a.View, TimescaleSupport.QueryStoreStatsIntervalHourlyView, StringComparison.Ordinal)) + .ToList(); + + Assert.NotEmpty(others); + foreach (var (createSql, view) in others) + { + Assert.DoesNotContain("create_group_indexes", createSql, StringComparison.Ordinal); + Assert.True(view.Length > 0); + } + + /* And L1 is in the hourly registry, so the converge below finds a materialization to act on. */ + Assert.Contains(TimescaleSupport.HourlyAggregates, a => string.Equals(a.View, TimescaleSupport.QueryStoreStatsIntervalHourlyView, StringComparison.Ordinal)); + } + + /// + /// The converge's catalog read selects by SHAPE — one column then bucket DESC — on L1's + /// materialization only, so the bucket index ((bucket DESC) alone) can never match, and a + /// materialization other than L1's is never touched. Pinned as text because the regex runs in + /// PostgreSQL; the gated test below asks the server. + /// + [Fact] + public void TheGroupIndexRead_SelectsByShape_OnL1Only() + { + var sql = TimescaleSupport.IntervalDedupMaterializationGroupIndexesSql; + + Assert.Contains($"ca.view_name = '{TimescaleSupport.QueryStoreStatsIntervalHourlyView}'", sql, StringComparison.Ordinal); + Assert.Contains(@"i.indexdef ~ 'USING btree \([a-z_]+, bucket DESC\)$'", sql, StringComparison.Ordinal); + Assert.Contains("i.tablename = ca.materialization_hypertable_name", sql, StringComparison.Ordinal); + /* Resolved from the view name, never a hard-coded _materialized_hypertable_N. */ + Assert.DoesNotContain("_materialized_hypertable_", sql, StringComparison.Ordinal); + } + + /// + /// The drop yields to a refresh in flight rather than queueing behind it (the queued-exclusive convoy + /// HourlyRefreshStartOffset documents): a bounded lock timeout, short against the grid's hour and + /// long against an idle lock. + /// + [Fact] + public void TheIndexDrop_WaitsABoundedTimeForItsLock() + { + var timeout = TimescaleSupport.IntervalDedupIndexDropLockTimeout; + + Assert.EndsWith("s", timeout, StringComparison.Ordinal); + var seconds = int.Parse(timeout.TrimEnd('s'), System.Globalization.CultureInfo.InvariantCulture); + Assert.InRange(seconds, 1, 60); + } + + /// + /// The evidence no string pin can give, on a scratch database with TimescaleDB: (1) TimescaleDB honours the + /// option — a freshly created L1 materialization carries exactly one index, on (bucket DESC); + /// (2) the converge finds a pre-#3597 store's group index (planted by hand in TimescaleDB's own shape and + /// name), drops it, reports one, and reports zero on the next call; (3) the bucket index is never + /// selected; (4) the refresh still materializes rows afterwards, through the same policy window the + /// product uses, and the child corrected hourly still reads them. + /// + [Fact] + public async Task OnAFreshStore_L1HasOnlyTheBucketIndex_AndTheConvergeDropsAPlantedGroupIndex_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #3597 materialization-index test (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + /* Manual refreshes below assert on exact ranges; strip the policies so a background first run + cannot race them (the QueryStoreTrendRoutingLiveTests discipline). */ + foreach (var (view, _, _, _, _) in TimescaleSupport.RollupViews) + { + await using var remove = new NpgsqlCommand( + $"SELECT remove_continuous_aggregate_policy('collect.{view}', if_exists => true)", connection); + await remove.ExecuteNonQueryAsync(ct); + } + + var (matSchema, matName) = await MaterializationOfAsync(connection, TimescaleSupport.QueryStoreStatsIntervalHourlyView, ct); + + /* (1) A fresh L1 honours the option: the bucket index and nothing else. */ + var fresh = await IndexDefinitionsAsync(connection, matSchema, matName, ct); + var only = Assert.Single(fresh); + Assert.EndsWith("USING btree (bucket DESC)", only, StringComparison.Ordinal); + + /* A settled store: nothing to find, nothing dropped. */ + Assert.Equal(0, await TimescaleSupport.EnsureIntervalDedupMaterializationIndexesAsync(connection, null, ct)); + + /* (2) Plant what an earlier build's CREATE left behind, in TimescaleDB's own name and shape. */ + var planted = matName + "_server_id_bucket_idx"; + await using (var plant = new NpgsqlCommand( + $"CREATE INDEX \"{planted}\" ON \"{matSchema}\".\"{matName}\" (server_id, bucket DESC)", connection)) + { + await plant.ExecuteNonQueryAsync(ct); + } + + Assert.Equal(2, (await IndexDefinitionsAsync(connection, matSchema, matName, ct)).Count); + + Assert.Equal(1, await TimescaleSupport.EnsureIntervalDedupMaterializationIndexesAsync(connection, null, ct)); + var afterDrop = await IndexDefinitionsAsync(connection, matSchema, matName, ct); + var survivor = Assert.Single(afterDrop); + /* (3) The bucket index survived, by shape. */ + Assert.EndsWith("USING btree (bucket DESC)", survivor, StringComparison.Ordinal); + + Assert.Equal(0, await TimescaleSupport.EnsureIntervalDedupMaterializationIndexesAsync(connection, null, ct)); + + /* (4) The aggregate still materializes without the group indexes. Fixed instants, not now-relative. */ + var hour = new DateTime(2026, 3, 4, 10, 0, 0, DateTimeKind.Unspecified); + await using (var seed = new NpgsqlCommand(@" +INSERT INTO collect.query_store_stats (collection_id, collection_time, server_id, server_name, database_name, query_id, plan_id, + execution_type_desc, first_execution_time, module_name, query_hash, execution_count, avg_duration_us, avg_cpu_time_us, + max_duration_us, max_cpu_time_us, replica_role, runtime_stats_interval_id, interval_start_time_utc) +SELECT 1, $1 + (k * interval '10 minutes'), -735970, 'interval-dedup-index-e2e', 'db', 100 + i, 1000 + i, + 'Regular', $1, 'mod', md5('q' || i), 10 * (k + 1), 500, 300, 900, 700, 'PRIMARY', 77, $1 +FROM generate_series(0, 9) AS i CROSS JOIN generate_series(0, 2) AS k", connection)) + { + seed.Parameters.AddWithValue(hour); + await seed.ExecuteNonQueryAsync(ct); + } + + await RefreshAsync(connection, TimescaleSupport.QueryStoreStatsIntervalHourlyView, hour, hour.AddHours(1), ct); + await RefreshAsync(connection, TimescaleSupport.QueryStoreStatsCorrectedHourlyView, hour, hour.AddHours(1), ct); + + await using (var l1 = new NpgsqlCommand( + $"SELECT count(*), sum(execution_count) FROM collect.{TimescaleSupport.QueryStoreStatsIntervalHourlyView} WHERE server_id = -735970", connection)) + await using (var reader = await l1.ExecuteReaderAsync(ct)) + { + Assert.True(await reader.ReadAsync(ct)); + /* Ten interval identities, each deduped to its LAST snapshot (30 executions). */ + Assert.Equal(10L, reader.GetInt64(0)); + Assert.Equal(300L, Convert.ToInt64(reader.GetValue(1), System.Globalization.CultureInfo.InvariantCulture)); + } + + await using (var corrected = new NpgsqlCommand( + $"SELECT sum(execution_count_sum) FROM collect.{TimescaleSupport.QueryStoreStatsCorrectedHourlyView} WHERE server_id = -735970", connection)) + { + /* Ten query_hash groups, one per identity; the child sums L1's deduped counts, never the raw snapshots. */ + Assert.Equal(300L, Convert.ToInt64(await corrected.ExecuteScalarAsync(ct), System.Globalization.CultureInfo.InvariantCulture)); + } + } + + private static async Task<(string Schema, string Name)> MaterializationOfAsync(NpgsqlConnection connection, string view, CancellationToken ct) + { + await using var command = new NpgsqlCommand( + "SELECT materialization_hypertable_schema, materialization_hypertable_name FROM timescaledb_information.continuous_aggregates WHERE view_schema = 'collect' AND view_name = $1", connection); + command.Parameters.AddWithValue(view); + await using var reader = await command.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), $"{view} was not created"); + return (reader.GetString(0), reader.GetString(1)); + } + + private static async Task> IndexDefinitionsAsync(NpgsqlConnection connection, string schema, string table, CancellationToken ct) + { + var definitions = new List(); + await using var command = new NpgsqlCommand( + "SELECT indexdef FROM pg_indexes WHERE schemaname = $1 AND tablename = $2 ORDER BY indexname", connection); + command.Parameters.AddWithValue(schema); + command.Parameters.AddWithValue(table); + await using var reader = await command.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + definitions.Add(reader.GetString(0)); + } + + return definitions; + } + + /// Bounded retry on 55P03 — the same reason QueryStoreTrendRoutingLiveTests retries: a policy's + /// creation-time first run can still be finishing when the manual refresh lands. + private static async Task RefreshAsync(NpgsqlConnection connection, string view, DateTime from, DateTime to, CancellationToken ct) + { + for (var attempt = 1; ; attempt++) + { + try + { + await using var refresh = new NpgsqlCommand( + $"CALL refresh_continuous_aggregate('collect.{view}', $1::timestamp, $2::timestamp)", connection); + refresh.Parameters.AddWithValue(from); + refresh.Parameters.AddWithValue(to); + await refresh.ExecuteNonQueryAsync(ct); + return; + } + catch (PostgresException ex) when (ex.SqlState == "55P03" && attempt < 12) + { + await Task.Delay(TimeSpan.FromSeconds(1), ct); + } + } + } +} diff --git a/Darling/Darling.Tests/McpFilterSemanticsLivePostgresTests.cs b/Darling/Darling.Tests/McpFilterSemanticsLivePostgresTests.cs new file mode 100644 index 000000000..6a1f17614 --- /dev/null +++ b/Darling/Darling.Tests/McpFilterSemanticsLivePostgresTests.cs @@ -0,0 +1,369 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3541 A13 / A9 against live PostgreSQL: the properties only the SQL can prove. +/// +/// A filter is part of the query (A13). get_top_queries_by_cpu applied +/// parallel_only / min_dop in C# over the top-N page the SQL had already cut, so a box whose +/// hottest N plans were all serial answered an EMPTY page under parallel_only while the window held a +/// parallel plan just past the cut. The fixture is exactly that shape: three serial groups hotter than one +/// parallel group, read at top = 2. The old code returned nothing; the fixed read must return the +/// parallel group, and the unfiltered read at the same cap must still return the two hottest serial ones — +/// the pair is what separates "filter in the query" from "filter the page". +/// +/// get_active_queries read the whole window, filtered in C#, published the pre-filter +/// rows.Count as total_snapshots, and its WAITFOR trim dropped the head blocker its victims +/// pointed at. One capture is seeded with the three blocker situations the tool now names: a victim whose +/// blocker is a WAITFOR shell in the same capture (kept, flagged), a victim whose blocker was never captured +/// (an idle open transaction), and a victim in one database whose blocker is in another (present unfiltered, +/// filtered under database_name). The count beside the page must be the FILTERED population's, +/// and truncation must be observed on it. +/// +/// Retention ghosts (A9). A collector run 45 days back sits inside the collection log's 60-day +/// horizon and outside the signals' 30-day one — the exact stretch that used to band Healthy on COALESCEd +/// zeros. It must come back purged / NoData with the horizon stated; today's run must stay +/// collected / Healthy beside it; the single-day read of the purged day must refuse a verdict; +/// and a fleet-wide retention override on ONE signal collector must move the horizon, because the horizon is +/// the store's effective retention rather than the shipped default. +/// +[Collection("live-postgres")] +public sealed class McpFilterSemanticsLivePostgresTests +{ + private const string ServerName = "darling-mcp-filter-semantics-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private const string Db = "StackOverflow"; + private const string OtherDb = "AdventureWorks"; + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task ParallelFilter_RanksTheFilteredPopulation_NotTheFilteredPage() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live filter-semantics test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); + + /* Three serial groups, hottest first, then ONE parallel group cooler than all three. */ + await PlantQueryAsync(connection, ct, now, "0xSERIAL1", "SELECT 1", cpuUs: 900_000L, maxDop: 1); + await PlantQueryAsync(connection, ct, now, "0xSERIAL2", "SELECT 2", cpuUs: 800_000L, maxDop: 1); + await PlantQueryAsync(connection, ct, now, "0xSERIAL3", "SELECT 3", cpuUs: 700_000L, maxDop: 1); + await PlantQueryAsync(connection, ct, now, "0xPARALLEL", "SELECT 4", cpuUs: 100_000L, maxDop: 8); + + /* Unfiltered at top = 2: the two hottest, both serial, and no filter stated. */ + var unfiltered = JsonDocument.Parse(await DarlingMcpDataTools.GetTopQueriesByCpu(postgres, ServerName, 1, 2)).RootElement; + Assert.Equal(new[] { "0xSERIAL1", "0xSERIAL2" }, Hashes(unfiltered)); + Assert.Equal(JsonValueKind.Null, unfiltered.GetProperty("filter_applied").ValueKind); + + /* parallel_only at top = 2: the OLD code cut the two serial rows and then filtered them away — + an empty page over a window that holds a parallel plan. The fixed read ranks the parallel + population and returns it. */ + var parallel = JsonDocument.Parse(await DarlingMcpDataTools.GetTopQueriesByCpu(postgres, ServerName, 1, 2, parallel_only: true)).RootElement; + Assert.Equal(new[] { "0xPARALLEL" }, Hashes(parallel)); + Assert.Contains("max_dop >= 2", parallel.GetProperty("filter_applied").GetString(), StringComparison.Ordinal); + Assert.True(parallel.GetProperty("queries")[0].GetProperty("is_parallel").GetBoolean()); + + /* min_dop above the seeded DOP: an empty FILTERED page is the window's answer, not a collection miss. */ + var tooHigh = JsonDocument.Parse(await DarlingMcpDataTools.GetTopQueriesByCpu(postgres, ServerName, 1, 2, min_dop: 16)).RootElement; + Assert.Equal("empty", tooHigh.GetProperty("status").GetString()); + Assert.Contains("max_dop >= 16", tooHigh.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Contains("applied in SQL over the whole window", tooHigh.GetProperty("message").GetString(), StringComparison.Ordinal); + + /* The rollup read carries the same floor. */ + var rolled = JsonDocument.Parse(await DarlingMcpDataTools.GetTopQueriesByCpu(postgres, ServerName, 1, 2, parallel_only: true, group_by: "host_object")).RootElement; + Assert.Equal(new[] { "0xPARALLEL" }, Hashes(rolled)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task ActiveQueries_FiltersInTheQuery_KeepsHeadBlockers_AndNamesAbsentOnes() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live filter-semantics test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var t = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); + + /* One capture: + 55 (Db) blocked by 60 — a WAITFOR shell in the same capture: the classic head blocker. + 60 (Db) WAITFOR DELAY, blocking 55. The old read stripped it. + 56 (Db) blocked by 61 — 61 never captured (idle open transaction). + 57 (OtherDb) blocked by 62 — 62 captured, in Db. + 62 (Db) running, blocking 57. + 70 (Db) running, not involved in blocking. + A SECOND capture two minutes earlier holds an unrelated session 60 so a cross-capture match + would wrongly resurrect it as a head blocker: same id, different capture, must NOT be kept. */ + await PlantSnapshotAsync(connection, ct, t, 55, Db, "UPDATE Posts SET Score = 1", blockingSessionId: 60, cpuMs: 500); + await PlantSnapshotAsync(connection, ct, t, 60, Db, "WAITFOR DELAY '00:05'", blockingSessionId: 0, cpuMs: 1); + await PlantSnapshotAsync(connection, ct, t, 56, Db, "DELETE FROM Votes", blockingSessionId: 61, cpuMs: 400); + await PlantSnapshotAsync(connection, ct, t, 57, OtherDb, "SELECT * FROM Sales", blockingSessionId: 62, cpuMs: 300); + await PlantSnapshotAsync(connection, ct, t, 62, Db, "UPDATE Users SET Reputation = 0", blockingSessionId: 0, cpuMs: 900); + await PlantSnapshotAsync(connection, ct, t, 70, Db, "SELECT COUNT(*) FROM Comments", blockingSessionId: 0, cpuMs: 200); + await PlantSnapshotAsync(connection, ct, t.AddMinutes(-2), 60, Db, "WAITFOR DELAY '00:05'", blockingSessionId: 0, cpuMs: 1); + + /* Unfiltered: the WAITFOR head blocker is on the page, flagged; the stale WAITFOR in the other + capture is not; the never-captured blocker is named as such; the cross-database blocker is present. */ + var all = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, limit: 50)).RootElement; + var rows = all.GetProperty("queries").EnumerateArray().ToArray(); + Assert.Equal(6, rows.Length); + Assert.Equal(6, all.GetProperty("total_snapshots").GetInt64()); + Assert.Equal(6, all.GetProperty("snapshots_returned").GetInt32()); + Assert.False(all.GetProperty("truncated").GetBoolean()); + Assert.Equal("collection_time_desc", all.GetProperty("order").GetString()); + + var head = Assert.Single(rows, r => r.GetProperty("session_id").GetInt32() == 60); + Assert.True(head.GetProperty("is_head_blocker").GetBoolean()); + Assert.StartsWith("WAITFOR", head.GetProperty("query_text").GetString(), StringComparison.Ordinal); + + Assert.Equal(JsonValueKind.Null, Row(rows, 55).GetProperty("blocker_not_shown").ValueKind); + Assert.Equal("not_captured", Row(rows, 56).GetProperty("blocker_not_shown").GetString()); + Assert.Equal(JsonValueKind.Null, Row(rows, 57).GetProperty("blocker_not_shown").ValueKind); + Assert.Equal(JsonValueKind.Null, Row(rows, 70).GetProperty("is_head_blocker").ValueKind); + + /* database_name filter, IN the query: the population is OtherDb's one victim, its blocker is + in Db and therefore filtered — the caller asked for that database, and the row says so. */ + var other = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, OtherDb)).RootElement; + Assert.Equal(1, other.GetProperty("total_snapshots").GetInt64()); + Assert.Equal("filtered", Assert.Single(other.GetProperty("queries").EnumerateArray()).GetProperty("blocker_not_shown").GetString()); + Assert.Equal(OtherDb, other.GetProperty("filters_applied").GetProperty("database_name").GetString()); + + /* blocking_only, IN the query: victims 55/56/57 + heads 60/62 = 5; 70 is out. The count is the + blocking population's, not the window's. */ + var blocking = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, blocking_only: true)).RootElement; + Assert.Equal(5, blocking.GetProperty("total_snapshots").GetInt64()); + Assert.DoesNotContain(blocking.GetProperty("queries").EnumerateArray(), r => r.GetProperty("session_id").GetInt32() == 70); + + /* Truncation is observed on the FILTERED population: limit = 4 over 5 blocking rows is truncated, + limit = 5 is not, and the total stays the population's under both. The page is CPU-descending + within the capture, so the WAITFOR head (1 ms) falls past a 4-row page and its victim says so. */ + var cut = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, blocking_only: true, limit: 4)).RootElement; + Assert.True(cut.GetProperty("truncated").GetBoolean()); + Assert.Equal(4, cut.GetProperty("snapshots_returned").GetInt32()); + Assert.Equal(5, cut.GetProperty("total_snapshots").GetInt64()); + Assert.Equal("past_page", Row(cut.GetProperty("queries").EnumerateArray().ToArray(), 55).GetProperty("blocker_not_shown").GetString()); + var whole = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, blocking_only: true, limit: 5)).RootElement; + Assert.False(whole.GetProperty("truncated").GetBoolean()); + + /* A filtered miss names the filter rather than calling the window empty. */ + var miss = JsonDocument.Parse(await DarlingMcpSessionTools.GetActiveQueries(postgres, ServerName, 1, "NoSuchDb")).RootElement; + Assert.Equal("empty", miss.GetProperty("status").GetString()); + Assert.Contains("database_name 'NoSuchDb'", miss.GetProperty("message").GetString(), StringComparison.Ordinal); + + /* A13's third item, on the same fixture: the uncapped reads refuse a negative span. */ + Assert.StartsWith("Invalid hours_back value '-24'", await DarlingMcpDataTools.GetCollectionLog(postgres, ServerName, -24), StringComparison.Ordinal); + Assert.StartsWith("Invalid hours_back value '0'", await DarlingMcpDataTools.GetCurrentWaitsTrend(postgres, ServerName, 0), StringComparison.Ordinal); + Assert.StartsWith("Invalid hours_back value '-1'", await DarlingMcpDataTools.GetBlockingStats(postgres, ServerName, -1), StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task DailySummary_StopsPaintingPurgedDaysGreen_AndPublishesTheHorizon() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live retention-ghost test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var today = DateTime.UtcNow.Date; + /* Inside the collection log's 60-day horizon, outside the signals' 30 — the ghost stretch. */ + var ghostDay = today.AddDays(-45); + /* Inside every default horizon; becomes a ghost once ONE signal's retention is overridden to 10. */ + var nearDay = today.AddDays(-20); + var expectedHorizon = DailySummaryRetention.HorizonFor(DateTime.UtcNow, DarlingRetention.DataRetentionBaseDays); + + await SeedRunAsync(connection, ct, DateTime.UtcNow.AddMinutes(-2), "SUCCESS"); + await SeedRunAsync(connection, ct, ghostDay.AddHours(12), "SUCCESS"); + await SeedRunAsync(connection, ct, nearDay.AddHours(12), "SUCCESS"); + + var range = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummaryRange(postgres, ServerName, 60)).RootElement; + Assert.Equal(expectedHorizon.ToString("yyyy-MM-dd"), range.GetProperty("retention_horizon").GetString()); + Assert.Equal(3, range.GetProperty("day_count").GetInt32()); + Assert.Equal(1, range.GetProperty("days_before_horizon").GetInt32()); + Assert.Equal(1, range.GetProperty("purged_day_count").GetInt32()); + Assert.Equal(2, range.GetProperty("collected_day_count").GetInt32()); + + var days = range.GetProperty("days").EnumerateArray().ToArray(); + var ghost = Assert.Single(days, d => d.GetProperty("summary_date").GetString() == ghostDay.ToString("yyyy-MM-dd")); + /* The day the run record kept on the spine: runs = 1, every signal a COALESCEd zero — and it is + NOT Healthy. */ + Assert.Equal(1, ghost.GetProperty("collection_runs").GetInt64()); + Assert.Equal("purged", ghost.GetProperty("data_state").GetString()); + Assert.Equal("NoData", ghost.GetProperty("health_band").GetString()); + Assert.Equal("No Data", ghost.GetProperty("overall_health").GetString()); + Assert.Contains("PURGED", ghost.GetProperty("data_note").GetString(), StringComparison.Ordinal); + Assert.Contains(expectedHorizon.ToString("yyyy-MM-dd"), ghost.GetProperty("data_note").GetString(), StringComparison.Ordinal); + + var live = Assert.Single(days, d => d.GetProperty("summary_date").GetString() == today.ToString("yyyy-MM-dd")); + Assert.Equal("collected", live.GetProperty("data_state").GetString()); + Assert.Equal("Healthy", live.GetProperty("overall_health").GetString()); + Assert.Equal(JsonValueKind.Null, live.GetProperty("data_note").ValueKind); + + /* The single-day read of the ghost refuses a verdict, in the miss vocabulary's own word for it. */ + var single = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummary(postgres, ServerName, ghostDay.ToString("yyyy-MM-dd"))).RootElement; + Assert.Equal("unavailable", single.GetProperty("status").GetString()); + Assert.Contains("retention_horizon", single.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Equal("purged", single.GetProperty("hints").GetProperty("data_state").GetString()); + Assert.Equal(1, single.GetProperty("hints").GetProperty("collection_runs").GetInt64()); + + /* And of a collected day, the verdict with its state. */ + var todayRow = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummary(postgres, ServerName)).RootElement; + Assert.Equal("collected", todayRow.GetProperty("data_state").GetString()); + Assert.Equal(expectedHorizon.ToString("yyyy-MM-dd"), todayRow.GetProperty("retention_horizon").GetString()); + + /* summary_date is exact ISO-8601: the ambiguous spelling is refused, not guessed at. */ + Assert.StartsWith("Invalid summary_date value '01/02/2026'", await DarlingMcpHealthTools.GetDailySummary(postgres, ServerName, "01/02/2026"), StringComparison.Ordinal); + + /* A signal row surviving before the horizon (the purge has not reached it — gate-held, paused, + or simply not yet run) turns the shell into past_horizon: the row is real, the verdict is + still withheld, and the day is NOT called purged because that would be false. */ + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO deadlocks (deadlock_id, collection_time, server_id, server_name, deadlock_time, victim_process_id, victim_sql_text, deadlock_graph_xml) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + CollectionIdGenerator.Next(), DarlingMcpTestData.Naive(ghostDay.AddHours(6)), ServerId, ServerName, + DarlingMcpTestData.Naive(ghostDay.AddHours(6)), "process1", "DELETE FROM Posts", ""); + var survived = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummaryRange(postgres, ServerName, 60)).RootElement; + Assert.Equal(1, survived.GetProperty("days_before_horizon").GetInt32()); + Assert.Equal(0, survived.GetProperty("purged_day_count").GetInt32()); + var pastHorizon = Assert.Single(survived.GetProperty("days").EnumerateArray(), d => d.GetProperty("summary_date").GetString() == ghostDay.ToString("yyyy-MM-dd")); + Assert.Equal("past_horizon", pastHorizon.GetProperty("data_state").GetString()); + Assert.Equal("NoData", pastHorizon.GetProperty("health_band").GetString()); + Assert.Equal(1, pastHorizon.GetProperty("deadlock_count").GetInt64()); + Assert.Contains("1 of 7 signal sources", pastHorizon.GetProperty("data_note").GetString(), StringComparison.Ordinal); + /* The single-day read of a past-horizon day is a DATA payload (the rows are there), not unavailable. */ + var singlePast = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummary(postgres, ServerName, ghostDay.ToString("yyyy-MM-dd"))).RootElement; + Assert.Equal("past_horizon", singlePast.GetProperty("data_state").GetString()); + Assert.Equal("No Data", singlePast.GetProperty("overall_health").GetString()); + await DarlingMcpTestData.ExecAsync(connection, ct, $"DELETE FROM deadlocks WHERE server_id = {ServerId}"); + + /* The horizon is the store's EFFECTIVE retention: a fleet-wide override shortening one signal + collector to 10 days moves it, and the 20-day-old run becomes a ghost too. */ + await DarlingMcpTestData.ExecAsync(connection, ct, + "INSERT INTO config_collector_schedules (server_id, collector_name, retention_days, enabled) VALUES (NULL, 'deadlocks', 10, TRUE)"); + var shortened = JsonDocument.Parse(await DarlingMcpHealthTools.GetDailySummaryRange(postgres, ServerName, 60)).RootElement; + Assert.Equal(DailySummaryRetention.HorizonFor(DateTime.UtcNow, 10).ToString("yyyy-MM-dd"), shortened.GetProperty("retention_horizon").GetString()); + Assert.Equal(2, shortened.GetProperty("days_before_horizon").GetInt32()); + var near = Assert.Single(shortened.GetProperty("days").EnumerateArray(), d => d.GetProperty("summary_date").GetString() == nearDay.ToString("yyyy-MM-dd")); + Assert.Equal("purged", near.GetProperty("data_state").GetString()); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static string[] Hashes(JsonElement root) => + root.GetProperty("queries").EnumerateArray().Select(q => q.GetProperty("query_hash").GetString()!).ToArray(); + + private static JsonElement Row(JsonElement[] rows, int sessionId) => + Assert.Single(rows, r => r.GetProperty("session_id").GetInt32() == sessionId); + + private static async Task PlantQueryAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, string queryHash, string queryText, long cpuUs, int maxDop) + { + var digest = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(queryText)); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, + query_hash, query_plan_hash, sql_handle, plan_handle, query_text, + query_text_digest, delta_execution_count, delta_worker_time, + delta_elapsed_time, delta_logical_reads, min_dop, max_dop) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)", + CollectionIdGenerator.Next(), at, ServerId, ServerName, Db, + queryHash, "0xPLANHASH", "0xSQLH" + queryHash, "0xPLANH", queryText, + digest, 10L, cpuUs, cpuUs, 100L, 1, maxDop); + } + + private static Task PlantSnapshotAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime at, int sessionId, string database, string text, int blockingSessionId, long cpuMs) => + DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO query_snapshots (collection_id, collection_time, server_id, server_name, session_id, database_name, query_text, status, blocking_session_id, wait_type, cpu_time_ms, total_elapsed_time_ms, request_id) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)", + CollectionIdGenerator.Next(), at, ServerId, ServerName, sessionId, database, text, + blockingSessionId > 0 ? "suspended" : "running", blockingSessionId, blockingSessionId > 0 ? "LCK_M_X" : null, cpuMs, cpuMs * 2, 0); + + private static Task SeedRunAsync(NpgsqlConnection connection, CancellationToken ct, DateTime collectionTimeUtc, string status) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO collection_log + (log_id, server_id, server_name, collector_name, collection_time, + duration_ms, status, error_message, rows_collected, sql_duration_ms, duckdb_duration_ms) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + CollectionIdGenerator.Next(), ServerId, ServerName, "wait_stats", + DarlingMcpTestData.Naive(collectionTimeUtc), 100, status, null, 10, 80, 20); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + var sql = string.Join(" ", new[] { "query_stats", "query_snapshots", "collection_log", "deadlocks" } + .Select(tbl => $"DELETE FROM {tbl} WHERE server_id = {ServerId};")); + sql += " DELETE FROM config_collector_schedules WHERE server_id IS NULL AND collector_name = 'deadlocks' AND retention_days = 10;"; + sql += $" DELETE FROM servers WHERE server_id = {ServerId};"; + sql += $" DELETE FROM config_monitored_servers WHERE server_id = {ServerId};"; + using var cleanup = new NpgsqlCommand(sql, connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs b/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs new file mode 100644 index 000000000..8be610f3c --- /dev/null +++ b/Darling/Darling.Tests/McpLatestSnapshotStampTests.cs @@ -0,0 +1,894 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// #3541 A10 — latest is a time. A latest-snapshot MCP read (WHERE collection_time = MAX(...), +/// ORDER BY collection_time DESC LIMIT 1) answers with the newest row a store holds, and until this lane +/// most of them answered with nothing that said WHEN that row was collected: a memory-clerk list, a file-I/O +/// table, a perfmon page, the whole configuration family (captured ON CONNECT, so a "current" setting could be +/// weeks old) — an agent that cannot see the code read every one of them as "now". Two of them accepted +/// hours_back and read only the newest row in it, so a grant storm three hours ago was invisible while +/// the parameter read as a window; one CRITICAL-verdict tool took server_name alone on one SKU and +/// (server_name, hours_back, as_of) on the other; and get_server_summary published ONE clock (the newest +/// collection of ANY collector) beside two figures it did not stamp. +/// +/// This file pins the rule and the three honest shapes a latest read may take, on BOTH SKUs, as one +/// dialect: every latest read publishes captured_at (the snapshot's own stamp) and its description says +/// "LATEST IS A TIME" or names its two reads; a tool that takes NO window carries no hours_back; a tool +/// whose hours_back is the span SEARCHED for the newest snapshot says so in those words and publishes +/// age_seconds against its anchor; a tool whose hours_back is READ publishes a window block +/// beside the snapshot. The two SKUs' descriptions of the two aligned tools are pinned byte-equal. Darling's +/// half is reflected off the assembly; Lite's is read from source, as does. +/// The discriminators are witnessed against literals so a matcher that stops matching cannot report clean. +/// Lite.Tests/McpLatestSnapshotStampTests executes the Lite tools against a real DuckDB; +/// executes the Darling ones against live Postgres. +/// +public sealed class McpLatestSnapshotStampTests +{ + /* ───────────────────────── the roster ───────────────────────── */ + + /// The three honest shapes. Named rather than inferred so a tool's disposition is a decision + /// written down here, and moving one is a visible edit. + public enum Shape + { + /// Reads one snapshot, takes no window: captured_at only. + Stamped, + + /// hours_back bounds the SEARCH for the newest snapshot (described in those words) and + /// as_of anchors it: captured_at + age_seconds. + SearchBound, + + /// hours_back is READ: the newest snapshot (captured_at + age_seconds) + /// beside a window aggregate over every snapshot in it. + Windowed, + } + + /// Every latest-snapshot tool this lane stamped, by SKU file, with its shape. The Lite file and + /// name are given per row because Lite hosts the same tool names in differently-named files. + public static readonly (Type Tools, string ToolName, string LiteFile, Shape Shape)[] LatestTools = + [ + (typeof(DarlingMcpDataTools), "get_memory_stats", "Lite/Mcp/McpMemoryTools.cs", Shape.Stamped), + (typeof(DarlingMcpDataTools), "get_memory_clerks", "Lite/Mcp/McpMemoryTools.cs", Shape.Stamped), + (typeof(DarlingMcpDataTools), "get_file_io_stats", "Lite/Mcp/McpIoTools.cs", Shape.Stamped), + (typeof(DarlingMcpDataTools), "get_perfmon_stats", "Lite/Mcp/McpPerfmonTools.cs", Shape.Stamped), + (typeof(DarlingMcpConfigTools), "get_server_config", "Lite/Mcp/McpConfigTools.cs", Shape.Stamped), + (typeof(DarlingMcpConfigTools), "get_database_config", "Lite/Mcp/McpConfigTools.cs", Shape.Stamped), + (typeof(DarlingMcpConfigTools), "get_trace_flags", "Lite/Mcp/McpConfigTools.cs", Shape.Stamped), + (typeof(DarlingMcpConfigHistoryTools), "get_database_scoped_config", "Lite/Mcp/McpConfigTools.cs", Shape.Stamped), + (typeof(DarlingMcpConfigHistoryTools), "get_query_store_health", "Lite/Mcp/McpConfigTools.cs", Shape.Stamped), + (typeof(DarlingMcpPlanCacheSchedulerTools), "get_plan_cache_bloat", "Lite/Mcp/McpPlanCacheSchedulerTools.cs", Shape.SearchBound), + (typeof(DarlingMcpPlanCacheSchedulerTools), "get_cpu_scheduler_pressure", "Lite/Mcp/McpPlanCacheSchedulerTools.cs", Shape.SearchBound), + (typeof(DarlingMcpMemoryGrantTools), "get_resource_semaphore", "Lite/Mcp/McpMemoryTools.cs", Shape.Windowed), + (typeof(DarlingMcpMemoryGrantTools), "get_memory_grants", "Lite/Mcp/McpMemoryTools.cs", Shape.Windowed), + ]; + + /// + /// Latest reads that stamped themselves BEFORE this lane, under the top-level key collection_time + /// rather than captured_at. The stamp is true; only the spelling predates the vocabulary, and the + /// Darling web surface reads one of them (get_session_stats, server-tabs.js' SESSION_STATS) + /// by that key, which sits outside this lane's boundary. Carried as a stated allowance with the control + /// below rather than silently: the day one of them is renamed, the allowance must shrink (#3541 A15/A16 is + /// the vocabulary lane). + /// + public static readonly string[] StampedUnderCollectionTime = + [ + "get_database_sizes", "get_running_jobs", "get_server_properties", "get_session_stats", + ]; + + /// + /// The latest reads this lane did NOT reach: the object-stats family reads the latest DAILY snapshot per + /// database (DarlingObjectStatsReader.IndexLockingSql takes MAX per database, not one instant) and + /// publishes no stamp at all on either SKU. Named here so the census fails the day one of them gains + /// captured_at without leaving this list, and so the gap is on the record rather than invisible. + /// Reported to #3541 as the A10 residual. + /// + public static readonly string[] UnstampedLatestReadsPendingA10 = + [ + "get_index_usage", "get_object_locking", "get_table_index_sizes", + ]; + + /// + /// Tools the reader-call sweep sees because they call a *Latest*Async reader as an INPUT to a read + /// that is not itself a latest snapshot: the two CPU rankings read the newest server_properties row for the + /// core count their attribution divides by; get_plan_corrections reads the newest automatic-tuning settings + /// beside its paged correction history; get_sweep_reports reads the newest fleet sweep, a worklist rather than + /// a per-server snapshot; get_pvs_stats mixes a latest per-database snapshot (stamped per row) with a trend, + /// and is the named exclusion carries for that reason. None of them is a + /// latest-snapshot tool, and none gets a captured_at here — the control below fails if one gains it, + /// so the decision is revisited rather than drifted into. + /// + public static readonly string[] LatestLookupInsideAnotherRead = + [ + "get_plan_corrections", "get_pvs_stats", "get_sweep_reports", "get_top_procedures_by_cpu", "get_top_queries_by_cpu", + ]; + + /// get_server_summary carries THREE clocks by name (cpu_captured_at, memory_captured_at, + /// last_collection) rather than one captured_at, because its two latest figures come from two + /// collectors that can be arbitrarily far apart. + /// holds the shape; the sweep only needs to know it is accounted for. + public static readonly string[] ThreeClockTools = ["get_server_summary"]; + + /// + /// Stamped latest reads that exist on ONE SKU. is a roster of PAIRS — every entry + /// names the Lite file its twin lives in and reads both — so a Darling-only + /// tool cannot sit in it without a Lite file to point at. These are held to the Stamped dialect by + /// below with the SAME three assertions the + /// roster's Stamped entries get (a captured_at on the payload, LATEST IS A TIME in the description, + /// neither knob in the signature) — only the Lite half is absent, because the SKU is. + /// + /// get_pg_logging_audit (#3607) judges seven logging GUCs from the newest + /// pg_server_config snapshot; Lite has no PostgreSQL target, so there is no twin for the roster to + /// pair it with. The same architectural boundary CrossAppMcpToolInventoryPinTests records for every + /// get_pg_* read. + /// + public static readonly (Type Tools, string ToolName)[] DarlingOnlyStamped = + [ + (typeof(DarlingMcpPgLoggingAuditTools), "get_pg_logging_audit"), + ]; + + /* ───────────────────────── the discriminators ───────────────────────── */ + + /// The stamp, as a payload key. + private static readonly Regex CapturedAtKey = new(@"\bcaptured_at\s*=", RegexOptions.Compiled); + + /// The anchored distance. + private static readonly Regex AgeSecondsKey = new(@"\bage_seconds\s*=", RegexOptions.Compiled); + + /// The window half of a Windowed tool. + private static readonly Regex WindowKey = new(@"\bwindow\s*=\s*window\.Select\(", RegexOptions.Compiled); + + /// A pre-lane stamp under the old spelling, as a TOP-LEVEL key (a per-row collection_time + /// inside a Select(r => new { ... }) is a series column, not the snapshot's stamp). + private static readonly Regex TopLevelCollectionTimeKey = new(@"\n\s{16}collection_time\s*=", RegexOptions.Compiled); + + /// The words a SearchBound tool must use for hours_back. + private const string SearchBoundWords = "search for the latest snapshot"; + + /// The words a Windowed tool must use for hours_back. + private const string WindowedWords = "window[] aggregates every snapshot in these hours"; + + /* ───────────────────────── both SKUs, from source ───────────────────────── */ + + [Fact] + public void EveryLatestSnapshotTool_PublishesCapturedAt_OnBothSkus() + { + foreach (var (label, body, _) in LatestToolBodies()) + { + Assert.True(CapturedAtKey.IsMatch(Strip(body)), + $"{label}: no `captured_at =` on the payload — a latest read that never says when it was captured"); + } + } + + [Fact] + public void EveryLatestSnapshotTool_SaysLatestIsATime_InItsDescription_OnBothSkus() + { + foreach (var (label, body, shape) in LatestToolBodies()) + { + var description = DescriptionOf(body, label); + Assert.Contains("captured_at", description, StringComparison.Ordinal); + if (shape == Shape.Windowed) + { + Assert.Contains("TWO READS UNDER ONE WINDOW", description, StringComparison.Ordinal); + Assert.Contains("window[]", description, StringComparison.Ordinal); + } + else + { + Assert.Contains("LATEST IS A TIME", description, StringComparison.Ordinal); + } + } + } + + /// A parameter that does nothing is a lie: a Stamped tool takes neither knob. + [Fact] + public void StampedTools_TakeNoWindowAndNoAnchor_OnBothSkus() + { + foreach (var (type, name, liteFile, shape) in LatestTools.Where(t => t.Shape == Shape.Stamped)) + { + var darling = ToolMethod(type, name).GetParameters().Select(p => p.Name).ToArray(); + Assert.DoesNotContain("hours_back", darling); + Assert.DoesNotContain("as_of", darling); + + var lite = LiteParamNames(liteFile, name); + Assert.DoesNotContain("hours_back", lite); + Assert.DoesNotContain("as_of", lite); + } + } + + /// + /// The Stamped dialect, held on the Darling-only reads with the roster's own three assertions — the stamp + /// on the payload, LATEST IS A TIME in the description, neither knob in the signature. Read from the + /// Darling source only, because there is no Lite half to read. + /// + [Fact] + public void DarlingOnlyStampedTools_KeepTheStampedDialect() + { + Assert.NotEmpty(DarlingOnlyStamped); + + foreach (var (type, name) in DarlingOnlyStamped) + { + var body = ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), name); + Assert.True(CapturedAtKey.IsMatch(Strip(body)), + $"Darling {name}: no `captured_at =` on the payload — a latest read that never says when it was captured"); + + var description = ToolMethod(type, name).GetCustomAttribute()!.Description; + Assert.Contains("captured_at", description, StringComparison.Ordinal); + Assert.Contains("LATEST IS A TIME", description, StringComparison.Ordinal); + + var parameters = ToolMethod(type, name).GetParameters().Select(p => p.Name).ToArray(); + Assert.DoesNotContain("hours_back", parameters); + Assert.DoesNotContain("as_of", parameters); + } + } + + /// + /// A SearchBound tool's hours_back says it is the SEARCH span in the same words on both SKUs, the + /// tool takes the anchor, and the payload carries the anchored distance — the three things that make a + /// latest read with a window parameter honest. + /// + [Fact] + public void SearchBoundTools_DescribeHoursBackAsTheSearchSpan_AndPublishAge_OnBothSkus() + { + foreach (var (type, name, liteFile, _) in LatestTools.Where(t => t.Shape == Shape.SearchBound)) + { + var method = ToolMethod(type, name); + var hours = method.GetParameters().Single(p => p.Name == "hours_back"); + Assert.Contains(SearchBoundWords, hours.GetCustomAttribute()!.Description, StringComparison.Ordinal); + Assert.Contains("as_of", method.GetParameters().Select(p => p.Name)); + Assert.Matches(AgeSecondsKey, Strip(ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), name))); + + var liteBody = ToolBody(ReadRepoFileLf(liteFile.Split('/')), name); + Assert.Contains(SearchBoundWords, liteBody, StringComparison.Ordinal); + Assert.Contains("as_of", LiteParamNames(liteFile, name)); + Assert.Matches(AgeSecondsKey, Strip(liteBody)); + } + } + + /// A Windowed tool reads its window: the payload carries the window block, the anchored + /// distance, and the window's own bounds, and hours_back says which half is which. + [Fact] + public void WindowedTools_PublishTheWindowBesideTheSnapshot_OnBothSkus() + { + foreach (var (type, name, liteFile, _) in LatestTools.Where(t => t.Shape == Shape.Windowed)) + { + var hours = ToolMethod(type, name).GetParameters().Single(p => p.Name == "hours_back"); + Assert.Contains(WindowedWords, hours.GetCustomAttribute()!.Description, StringComparison.Ordinal); + + foreach (var body in new[] { ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), name), ToolBody(ReadRepoFileLf(liteFile.Split('/')), name) }) + { + var text = Strip(body); + Assert.Matches(WindowKey, text); + Assert.Matches(AgeSecondsKey, text); + Assert.Contains("window_start =", text, StringComparison.Ordinal); + Assert.Contains("window_end =", text, StringComparison.Ordinal); + Assert.Contains(WindowedWords, body, StringComparison.Ordinal); + } + } + } + + /// The same tool name takes the same parameters on both SKUs — the drift this lane closed on + /// get_cpu_scheduler_pressure, pinned for every tool in the roster so it cannot reopen on another. + [Fact] + public void TheSameToolName_TakesTheSameParameters_OnBothSkus() + { + foreach (var (type, name, liteFile, _) in LatestTools) + { + var darling = ToolMethod(type, name).GetParameters() + .Where(p => p.GetCustomAttribute() is not null) + .Select(p => p.Name!) + .ToArray(); + Assert.Equal(darling, LiteParamNames(liteFile, name)); + } + } + + /// + /// The two tools whose descriptions were rewritten on both SKUs are pinned byte-equal: a shared const + /// cannot cross the two assemblies, so the census holds the two literals together instead. Darling's is + /// reflected; Lite's is the const its attribute names, read from source. + /// + [Theory] + [InlineData(typeof(DarlingMcpPlanCacheSchedulerTools), "get_cpu_scheduler_pressure", "Lite/Mcp/McpPlanCacheSchedulerTools.cs", "CpuSchedulerPressureDescription")] + [InlineData(typeof(DarlingMcpHealthTools), "get_server_summary", "Lite/Mcp/McpHealthTools.cs", "ServerSummaryDescription")] + public void TheAlignedTools_AreDescribedIdentically_OnBothSkus(Type type, string toolName, string liteFile, string liteConst) + { + var darling = ToolMethod(type, toolName).GetCustomAttribute()!.Description; + var lite = LiteConstLiteral(liteFile, liteConst); + Assert.Equal(darling, lite); + + /* And the attribute really does name the const — a literal pasted beside an unused const would pass + the equality above while drifting the day either is edited. */ + var liteBody = ToolBody(ReadRepoFileLf(liteFile.Split('/')), toolName); + Assert.Contains($"Description({liteConst})", liteBody, StringComparison.Ordinal); + } + + /// The verdict Darling always published, now on Lite too: the same tool name answers with a + /// banded pressure_level on both SKUs, from the SHARED classifier. + [Fact] + public void CpuSchedulerPressure_PublishesTheVerdict_OnBothSkus() + { + var lite = Strip(ToolBody(ReadRepoFileLf("Lite", "Mcp", "McpPlanCacheSchedulerTools.cs"), "get_cpu_scheduler_pressure")); + Assert.Contains("CpuSchedulerMetrics.ClassifyCpuPressure(", lite, StringComparison.Ordinal); + Assert.Contains("pressure_level = pressure.Level", lite, StringComparison.Ordinal); + Assert.Contains("recommendation = pressure.Recommendation", lite, StringComparison.Ordinal); + + var darling = Strip(ToolBody(ReadRepoFileLf(DarlingFileOf(typeof(DarlingMcpPlanCacheSchedulerTools)).Split('/')), "get_cpu_scheduler_pressure")); + Assert.Contains("pressure_level = pressureLevel", darling, StringComparison.Ordinal); + } + + /// get_server_summary names its three clocks on both SKUs, and no longer publishes two figures + /// under one stamp. + [Fact] + public void ServerSummary_NamesItsThreeClocks_OnBothSkus() + { + foreach (var body in new[] + { + ToolBody(ReadRepoFileLf(DarlingFileOf(typeof(DarlingMcpHealthTools)).Split('/')), "get_server_summary"), + ToolBody(ReadRepoFileLf("Lite", "Mcp", "McpHealthTools.cs"), "get_server_summary"), + }) + { + var text = Strip(body); + Assert.Contains("cpu_captured_at =", text, StringComparison.Ordinal); + Assert.Contains("memory_captured_at =", text, StringComparison.Ordinal); + Assert.Contains("counts_window_hours =", text, StringComparison.Ordinal); + Assert.Contains("last_collection =", text, StringComparison.Ordinal); + } + } + + /// The latch band names the interval it came from, beside the window totals it did not. + [Fact] + public void LatchSeverity_NamesTheIntervalItWasBandedFrom() + { + var body = Strip(ToolBody(ReadRepoFileLf(DarlingFileOf(typeof(DarlingMcpLatchSpinlockTools)).Split('/')), "get_latch_stats")); + Assert.Contains("severity_banded_from = new", body, StringComparison.Ordinal); + Assert.Contains("delta_wait_time_ms = r.LatestDeltaWaitTimeMs", body, StringComparison.Ordinal); + Assert.Contains("interval_seconds =", body, StringComparison.Ordinal); + Assert.Contains("captured_at = r.LatestCollectionTime", body, StringComparison.Ordinal); + + var sql = DarlingLatchSpinlockReader.LatchStatsTopNSql; + Assert.Contains("AS latest_interval_seconds", sql, StringComparison.Ordinal); + Assert.Contains("l.latest_interval_seconds", sql, StringComparison.Ordinal); + + var description = ToolMethod(typeof(DarlingMcpLatchSpinlockTools), "get_latch_stats").GetCustomAttribute()!.Description; + Assert.Contains("severity_banded_from", description, StringComparison.Ordinal); + Assert.Contains("LATEST interval only", description, StringComparison.Ordinal); + } + + /* ───────────────────────── the sweep: no latest read escapes the roster ───────────────────────── */ + + /// + /// Every Darling tool whose body calls a *Latest*Async / *Snapshot*Async / Current-family + /// reader, or whose reader const is a latest-snapshot read, is in the roster, the pre-lane allowance, or the + /// named residual — and every allowance entry is still needed. A new latest read must pick a shape here + /// rather than ship unstamped. + /// + [Fact] + public void EveryLatestReadTool_IsInTheRoster_OrANamedAllowance_AndEveryAllowanceIsStillNeeded() + { + var rostered = LatestTools.Select(t => t.ToolName).ToHashSet(StringComparer.Ordinal); + var allowancesUsed = new HashSet(StringComparer.Ordinal); + var residualsSeen = new HashSet(StringComparer.Ordinal); + var lookupsSeen = new HashSet(StringComparer.Ordinal); + var threeClocksSeen = new HashSet(StringComparer.Ordinal); + var darlingOnlySeen = new HashSet(StringComparer.Ordinal); + var examined = 0; + + foreach (var (file, source) in AllDarlingToolSources()) + { + var marks = Regex.Matches(source, @"\[McpServerTool\(Name = ""([a-z_0-9]+)"""); + for (var i = 0; i < marks.Count; i++) + { + var end = i + 1 < marks.Count ? marks[i + 1].Index : source.Length; + var body = Strip(source[marks[i].Index..end]); + var toolName = marks[i].Groups[1].Value; + + if (!LatestReaderCall.IsMatch(body)) + { + continue; + } + + examined++; + if (rostered.Contains(toolName)) + { + continue; + } + + if (StampedUnderCollectionTime.Contains(toolName, StringComparer.Ordinal)) + { + Assert.True(TopLevelCollectionTimeKey.IsMatch(body), + $"{file} {toolName}: listed as stamped under collection_time but publishes no top-level collection_time"); + Assert.False(CapturedAtKey.IsMatch(body), + $"{file} {toolName}: now publishes captured_at — move it into the roster and out of StampedUnderCollectionTime"); + allowancesUsed.Add(toolName); + continue; + } + + if (UnstampedLatestReadsPendingA10.Contains(toolName, StringComparer.Ordinal)) + { + Assert.False(CapturedAtKey.IsMatch(body), + $"{file} {toolName}: now publishes captured_at — move it into the roster and out of UnstampedLatestReadsPendingA10"); + residualsSeen.Add(toolName); + continue; + } + + if (LatestLookupInsideAnotherRead.Contains(toolName, StringComparer.Ordinal)) + { + Assert.False(CapturedAtKey.IsMatch(body), + $"{file} {toolName}: now publishes captured_at — it has become a latest-snapshot tool; give it a Shape and remove it from LatestLookupInsideAnotherRead"); + lookupsSeen.Add(toolName); + continue; + } + + if (ThreeClockTools.Contains(toolName, StringComparer.Ordinal)) + { + Assert.Contains("cpu_captured_at =", body, StringComparison.Ordinal); + Assert.Contains("memory_captured_at =", body, StringComparison.Ordinal); + threeClocksSeen.Add(toolName); + continue; + } + + if (DarlingOnlyStamped.Any(t => t.ToolName == toolName)) + { + Assert.True(CapturedAtKey.IsMatch(body), + $"{file} {toolName}: listed as a Darling-only STAMPED read but publishes no captured_at"); + darlingOnlySeen.Add(toolName); + continue; + } + + Assert.Fail($"{file} {toolName}: calls a latest-snapshot reader and is in no list here — give it a Shape in LatestTools (and stamp it) or name it as an allowance with its reason"); + } + } + + /* Population controls: a sweep that matched nothing passes for free, and an allowance nobody needs is a + widened exemption. 26 latest-reading tool bodies at the time of writing. */ + Assert.True(examined >= 24, $"only {examined} latest-reading tool bodies were found; the reader-call pattern has stopped matching"); + Assert.True(LatestLookupInsideAnotherRead.ToHashSet(StringComparer.Ordinal).SetEquals(lookupsSeen), + "LatestLookupInsideAnotherRead no longer matches what the sweep finds: " + string.Join(", ", LatestLookupInsideAnotherRead.Except(lookupsSeen))); + Assert.True(ThreeClockTools.ToHashSet(StringComparer.Ordinal).SetEquals(threeClocksSeen), + "ThreeClockTools no longer matches what the sweep finds: " + string.Join(", ", ThreeClockTools.Except(threeClocksSeen))); + Assert.True(StampedUnderCollectionTime.ToHashSet(StringComparer.Ordinal).SetEquals(allowancesUsed), + "StampedUnderCollectionTime no longer matches what the sweep finds: " + string.Join(", ", StampedUnderCollectionTime.Except(allowancesUsed))); + Assert.True(DarlingOnlyStamped.Select(t => t.ToolName).ToHashSet(StringComparer.Ordinal).SetEquals(darlingOnlySeen), + "DarlingOnlyStamped no longer matches what the sweep finds: " + string.Join(", ", DarlingOnlyStamped.Select(t => t.ToolName).Except(darlingOnlySeen))); + Assert.True(UnstampedLatestReadsPendingA10.ToHashSet(StringComparer.Ordinal).SetEquals(residualsSeen), + "UnstampedLatestReadsPendingA10 no longer matches what the sweep finds: " + string.Join(", ", UnstampedLatestReadsPendingA10.Except(residualsSeen))); + } + + /// + /// A tool body's call into a latest-snapshot reader. The readers name themselves: GetLatest*Async, + /// *LatestAsync, *SnapshotAsync, the plan-cache / scheduler pair, the server summary, and the + /// object-stats trio (GetIndexUsageAsync / GetIndexLockingAsync / GetObjectSizeGrowthAsync, + /// each keyed on a correlated MAX(collection_time)). + /// + private static readonly Regex LatestReaderCall = new( + @"\.(GetLatest\w+Async|Get\w+LatestAsync|Get\w+SnapshotAsync|GetPlanCacheBloatAsync|GetCpuSchedulerPressureAsync|GetServerSummaryAsync|GetIndexUsageAsync|GetIndexLockingAsync|GetObjectSizeGrowthAsync|GetRunningJobsAsync)\(", + RegexOptions.Compiled); + + /* ───────────────────────── the readers ───────────────────────── */ + + /// Every stamped Darling read carries its stamp column ON THE ROW STATEMENT — never a second + /// MAX() read that could stamp the next capture. + [Theory] + [InlineData(nameof(DarlingDataReader.LatestMemoryClerksSql), "collection_time")] + [InlineData(nameof(DarlingDataReader.LatestFileIoStatsSql), "collection_time")] + [InlineData(nameof(DarlingDataReader.LatestPerfmonStatsSql), "collection_time")] + [InlineData(nameof(DarlingCurrentConfigReader.ServerConfigSql), "capture_time")] + [InlineData(nameof(DarlingCurrentConfigReader.DatabaseConfigSql), "capture_time")] + [InlineData(nameof(DarlingCurrentConfigReader.TraceFlagsSql), "capture_time")] + [InlineData(nameof(DarlingConfigHistoryReader.DatabaseScopedConfigSql), "capture_time")] + [InlineData(nameof(DarlingConfigHistoryReader.QueryStoreHealthSql), "capture_time")] + [InlineData(nameof(DarlingPgLoggingAuditReader.NewestSnapshotSql), "collection_time")] + public void EveryStampedRead_SelectsItsStampColumn_OnTheRowStatement(string sqlName, string column) + { + var sql = ReaderSql(sqlName); + var select = sql[..sql.IndexOf("FROM", StringComparison.Ordinal)]; + Assert.Contains(column, select, StringComparison.Ordinal); + } + + /// The scheduler read is bounded the way Lite's is — the newest row IN THE WINDOW, so a week-old + /// snapshot from a dead collector is unavailable rather than served as current. + [Fact] + public void CpuSchedulerRead_IsBoundedByTheWindow() + { + var sql = DarlingPlanCacheSchedulerReader.CpuSchedulerPressureSql; + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); + Assert.Contains("ORDER BY collection_time DESC", sql, StringComparison.Ordinal); + Assert.Contains("LIMIT 1", sql, StringComparison.Ordinal); + } + + /// The two window reads: the peak's instant from a DISTINCT ON over the SAME windowed rows, + /// the deltas SUMmed (no interval arithmetic — the naked-family rung is a rate question), and no literal cap. + [Theory] + [InlineData(nameof(DarlingMemoryGrantReader.ResourceSemaphoreWindowSql))] + [InlineData(nameof(DarlingMemoryGrantReader.MemoryGrantsWindowSql))] + public void MemoryGrantWindowReads_AggregateEverySnapshot_AndNameThePeaksInstant(string sqlName) + { + var sql = ReaderSql(sqlName); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $3", sql, StringComparison.Ordinal); + Assert.Contains("COUNT(*) AS snapshots_in_window", sql, StringComparison.Ordinal); + Assert.Contains("MAX(waiter_count) AS bigint) AS peak_waiter_count", sql, StringComparison.Ordinal); + Assert.Contains("SUM(timeout_error_count_delta) AS bigint) AS timeout_errors_in_window", sql, StringComparison.Ordinal); + Assert.Contains("SUM(forced_grant_count_delta) AS bigint) AS forced_grants_in_window", sql, StringComparison.Ordinal); + Assert.Contains("SELECT DISTINCT ON", sql, StringComparison.Ordinal); + Assert.Contains("waiter_count DESC, collection_time DESC", sql, StringComparison.Ordinal); + Assert.DoesNotMatch(@"\bLIMIT\b", sql); + Assert.DoesNotContain("sample_interval_seconds", sql, StringComparison.Ordinal); + } + + /// The web catalogue advertises the two knobs the aligned scheduler tool now takes, and the + /// dispatch forwards them ( holds the general rule; this names the tool). + [Fact] + public void CpuSchedulerPressure_AdvertisesAndForwardsItsWindow_OnTheWebSurface() + { + var descriptor = DarlingWebEndpoints.CatalogDescriptors["get_cpu_scheduler_pressure"]; + Assert.Contains("hours", descriptor.Params.Select(p => p.Name)); + Assert.Contains("as_of", descriptor.Params.Select(p => p.Name)); + + var source = Strip(ReadRepoFileLf("Darling", "PerformanceMonitor.Darling.Service", "DarlingWebEndpoints.cs")); + Assert.Matches(@"\[""get_cpu_scheduler_pressure""\] = \(c, pg, an\) => DarlingMcpPlanCacheSchedulerTools\.GetCpuSchedulerPressure\(pg, Server\(c\), Hours\(c, 24\), as_of: AsOf\(c\)\)", source); + } + + /* ───────────────────────── the pure pieces, executed ───────────────────────── */ + + [Fact] + public void LatestSnapshot_RefusesRowsWithoutAStamp_AndIsEmptyWithoutRows() + { + Assert.Throws(() => new LatestSnapshot(null, new List { 1 })); + + var empty = LatestSnapshot.Empty; + Assert.True(empty.IsEmpty); + Assert.Null(empty.CapturedAt); + Assert.Equal(0, empty.Count); + + var stamped = new LatestSnapshot(new DateTime(2026, 9, 18, 12, 0, 0), new List { 1, 2 }); + Assert.False(stamped.IsEmpty); + Assert.Equal(2, stamped.Count); + } + + /// Whole seconds from stamp to anchor; never negative; Kind-blind (both instants are UTC). + [Fact] + public void AgeSeconds_IsTheWholeSecondDistanceToTheAnchor_AndNeverNegative() + { + var stamp = new DateTime(2026, 9, 18, 12, 0, 0, DateTimeKind.Unspecified); + var anchor = new DateTime(2026, 9, 18, 12, 5, 0, DateTimeKind.Utc); + Assert.Equal(300L, LatestSnapshotStamp.AgeSeconds(stamp, anchor)); + Assert.Equal(300L, LatestSnapshotStamp.AgeSeconds(stamp, anchor.AddTicks(4_000_000))); /* 0.4 s rounds down */ + Assert.Equal(301L, LatestSnapshotStamp.AgeSeconds(stamp, anchor.AddTicks(6_000_000))); /* 0.6 s rounds up */ + Assert.Equal(0L, LatestSnapshotStamp.AgeSeconds(anchor, stamp)); /* clamped */ + Assert.Equal(0L, LatestSnapshotStamp.AgeSeconds(stamp, stamp)); + } + + /* ───────────────────────── the matchers, witnessed ───────────────────────── */ + + [Fact] + public void TheDiscriminators_FlagTheDefectShapes_AndPassTheFixedOnes() + { + Assert.Matches(CapturedAtKey, " captured_at = snapshot.CapturedAt!.Value.ToString(\"o\"),"); + Assert.DoesNotMatch(CapturedAtKey, " last_captured_at = Stamp(c.LastCapturedAt),"); + Assert.DoesNotMatch(CapturedAtKey, " collection_time = stats.CollectionTime.ToString(\"o\"),"); + + Assert.Matches(AgeSecondsKey, " age_seconds = LatestSnapshotStamp.AgeSeconds(rows[0].CollectionTime, now),"); + Assert.Matches(WindowKey, " window = window.Select(WindowShape)"); + Assert.DoesNotMatch(WindowKey, " window_start = windowStart.ToString(\"o\"),"); + + Assert.Matches(TopLevelCollectionTimeKey, "\n collection_time = rows[0].CollectionTime.ToString(\"o\"),"); + Assert.DoesNotMatch(TopLevelCollectionTimeKey, "\n collection_time = r.CollectionTime.ToString(\"o\"),"); + + Assert.Matches(LatestReaderCall, " var snapshot = await DarlingDataReader.GetLatestMemoryClerksAsync(postgres, resolved.ServerId);"); + Assert.Matches(LatestReaderCall, " var rows = await DarlingMemoryGrantReader.GetResourceSemaphoreLatestAsync("); + Assert.Matches(LatestReaderCall, " var rows = await dataService.GetLatchStatsSnapshotAsync(resolved.ServerId, hours_back, asOfUtc: windowEnd);"); + Assert.DoesNotMatch(LatestReaderCall, " var rows = await DarlingDataReader.GetTempDbTrendAsync(postgres, resolved.ServerId, a, b);"); + } + + /* ───────────────────────── plumbing ───────────────────────── */ + + private static IEnumerable<(string Label, string Body, Shape Shape)> LatestToolBodies() + { + foreach (var (type, name, liteFile, shape) in LatestTools) + { + yield return ($"Darling {name}", ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), name), shape); + yield return ($"Lite {name}", ToolBody(ReadRepoFileLf(liteFile.Split('/')), name), shape); + } + } + + private static IEnumerable<(string File, string Source)> AllDarlingToolSources() + { + var root = RepoFile.PathTo("Darling/PerformanceMonitor.Darling.Service/Mcp"); + foreach (var file in System.IO.Directory.EnumerateFiles(root, "*.cs").Order(StringComparer.Ordinal)) + { + yield return (System.IO.Path.GetFileName(file), System.IO.File.ReadAllText(file).Replace("\r\n", "\n", StringComparison.Ordinal)); + } + } + + private static string ReaderSql(string sqlName) => sqlName switch + { + nameof(DarlingDataReader.LatestMemoryClerksSql) => DarlingDataReader.LatestMemoryClerksSql, + nameof(DarlingDataReader.LatestFileIoStatsSql) => DarlingDataReader.LatestFileIoStatsSql, + nameof(DarlingDataReader.LatestPerfmonStatsSql) => DarlingDataReader.LatestPerfmonStatsSql, + nameof(DarlingCurrentConfigReader.ServerConfigSql) => DarlingCurrentConfigReader.ServerConfigSql, + nameof(DarlingCurrentConfigReader.DatabaseConfigSql) => DarlingCurrentConfigReader.DatabaseConfigSql, + nameof(DarlingCurrentConfigReader.TraceFlagsSql) => DarlingCurrentConfigReader.TraceFlagsSql, + nameof(DarlingConfigHistoryReader.DatabaseScopedConfigSql) => DarlingConfigHistoryReader.DatabaseScopedConfigSql, + nameof(DarlingConfigHistoryReader.QueryStoreHealthSql) => DarlingConfigHistoryReader.QueryStoreHealthSql, + nameof(DarlingMemoryGrantReader.ResourceSemaphoreWindowSql) => DarlingMemoryGrantReader.ResourceSemaphoreWindowSql, + nameof(DarlingMemoryGrantReader.MemoryGrantsWindowSql) => DarlingMemoryGrantReader.MemoryGrantsWindowSql, + nameof(DarlingPgLoggingAuditReader.NewestSnapshotSql) => DarlingPgLoggingAuditReader.NewestSnapshotSql, + _ => throw new ArgumentOutOfRangeException(nameof(sqlName), sqlName, "not a read this census names"), + }; + + private static MethodInfo ToolMethod(Type type, string toolName) => type + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == toolName); + + private static string DarlingFileOf(Type type) => + $"Darling/PerformanceMonitor.Darling.Service/Mcp/{type.Name}.cs"; + + /// The source from one tool's [McpServerTool(Name = "…")] attribute to the next tool's, or + /// to the end of the file — the span that holds its description, parameters and payload. + private static string ToolBody(string source, string toolName) + { + var marker = $"[McpServerTool(Name = \"{toolName}\")"; + var start = source.IndexOf(marker, StringComparison.Ordinal); + Assert.True(start >= 0, $"no tool named {toolName} in the source"); + var next = source.IndexOf("[McpServerTool(", start + marker.Length, StringComparison.Ordinal); + return next < 0 ? source[start..] : source[start..next]; + } + + /// The tool's description: the attribute's literal, or the const it names (the aligned tools + /// describe themselves through a const so the two SKUs' texts can be pinned equal). + private static string DescriptionOf(string body, string label) + { + /* Anchored on the TOOL attribute the body starts with, so a parameter's [Description("Server name…")] + further down can never be read as the tool's — which is exactly what a bare `Description(` search + did on the const-described tools when this file was first executed. */ + var attribute = Regex.Match(body, @"\A\[McpServerTool\(Name = ""[a-z_0-9]+""\), Description\((?:\s*)(?:""((?:[^""\\]|\\.)*)""|(\w+))\)\]"); + Assert.True(attribute.Success, $"{label}: could not locate the tool's Description on its McpServerTool attribute"); + if (attribute.Groups[1].Success) + { + return attribute.Groups[1].Value; + } + + var constName = attribute.Groups[2].Value; + + /* The const lives in the same file, above the attribute; the body slice starts AT the attribute, so it + is not in the body — resolve it from the file the label points at. */ + var (type, toolName, liteFile, _) = LatestTools.Single(t => label.EndsWith(t.ToolName, StringComparison.Ordinal)); + return label.StartsWith("Darling", StringComparison.Ordinal) + ? (string)type.GetField(constName, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static)!.GetRawConstantValue()! + : LiteConstLiteral(liteFile, constName); + } + + /// A Lite const string's literal, read from source: internal const string Name =\n "…";. + private static string LiteConstLiteral(string liteFile, string constName) + { + var source = ReadRepoFileLf(liteFile.Split('/')); + var m = Regex.Match(source, $@"const string {Regex.Escape(constName)}\s*=\s*""((?:[^""\\]|\\.)*)"";"); + Assert.True(m.Success, $"{liteFile}: no `const string {constName} = \"…\";`"); + return Regex.Unescape(m.Groups[1].Value); + } + + /// Lite's advertised parameter names for a tool, read off the MASKED signature (comments and + /// string literals blanked, so prose inside a [Description] cannot read as a parameter) — the idiom + /// DarlingMcpDataToolsTests.LiteMcpParamNames established. Every MCP parameter carries a default + /// and the injected services do not, so the defaulted ones in order ARE the contract. + private static string[] LiteParamNames(string liteFile, string toolName) + { + var raw = ReadRepoFileLf(liteFile.Split('/')); + var attribute = raw.IndexOf($"Name = \"{toolName}\"", StringComparison.Ordinal); + Assert.True(attribute > 0, $"{liteFile}: no tool named {toolName}"); + + var masked = CSharpSourceWalker.StripCommentsAndStrings(raw); + Assert.Equal(raw.Length, masked.Length); + + var declaration = masked.IndexOf("public static", attribute, StringComparison.Ordinal); + var signature = masked.IndexOf('(', declaration); + var body = masked.IndexOf('{', signature); + Assert.True(body > signature, $"{liteFile}: could not find the end of {toolName}'s signature"); + + return Regex.Matches(masked[signature..body], @"(\w+)\s*=\s*[^,)]+") + .Select(m => m.Groups[1].Value) + .ToArray(); + } + + /// Comments removed, so a comment that NAMES a key is not read as the key. Line and block comments + /// only; string literals stay, because the payload keys under test are not in strings. + private static string Strip(string source) => + Regex.Replace(Regex.Replace(source, @"/\*.*?\*/", string.Empty, RegexOptions.Singleline), @"//[^\n]*", string.Empty); +} + +/// +/// Gated (DARLING_TEST_PG) live round-trips for the stamps: the seeded row's stamp comes back as +/// captured_at, age_seconds is measured against the anchor the caller sent (never the wall clock), +/// a SearchBound tool refuses a snapshot older than its search span, the memory-grant window sees a storm the +/// latest snapshot does not, and the latch band names its interval. Anchored in the past on purpose: the +/// assertions are equalities, and an anchor of "now" would make every age a race. +/// +[Collection("live-postgres")] +public sealed class McpLatestSnapshotStampLivePostgresTests +{ + private const string ServerName = "darling-mcp-latest-stamp-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static readonly string[] Tables = + [ + "memory_grant_stats", "cpu_scheduler_stats", "server_config", "trace_flags", "memory_clerks", "latch_stats", + "memory_stats", "cpu_utilization_stats", "collection_log", + ]; + + [Fact] + public async Task LatestReads_SayWhenTheyWereCaptured_AgainstLivePostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live latest-stamp test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + + /* Every row sits in the past; the anchor is base + 5 min, so every age below is exact. */ + var @base = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddHours(-2); + var anchor = @base.AddMinutes(5).ToString("o") + "Z"; + + /* ── memory grants: a storm 30 minutes before a calm latest snapshot ── */ + foreach (var (t, waiters, timeouts, granted) in new[] { (@base.AddMinutes(-30), 12, 3L, 6000m), (@base, 0, 0L, 500m) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO memory_grant_stats (collection_id, collection_time, server_id, server_name, resource_semaphore_id, pool_id, target_memory_mb, max_target_memory_mb, total_memory_mb, available_memory_mb, granted_memory_mb, used_memory_mb, grantee_count, waiter_count, timeout_error_count, forced_grant_count, timeout_error_count_delta, forced_grant_count_delta) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, (short)0, 2, 8000m, 12000m, 8000m, 8000m - granted, granted, granted, 3, waiters, 4L, 2L, timeouts, 0L); + } + + var semaphore = Parse(await DarlingMcpMemoryGrantTools.GetResourceSemaphore(postgres, ServerName, 1, as_of: anchor)); + Assert.Equal(Stamp(@base), semaphore.GetProperty("captured_at").GetString()); + Assert.Equal(300, semaphore.GetProperty("age_seconds").GetInt64()); + var latestRow = Assert.Single(semaphore.GetProperty("grants").EnumerateArray()); + Assert.Equal(0, latestRow.GetProperty("waiter_count").GetInt32()); + var windowRow = Assert.Single(semaphore.GetProperty("window").EnumerateArray()); + Assert.Equal(2, windowRow.GetProperty("snapshots_in_window").GetInt64()); + Assert.Equal(12, windowRow.GetProperty("peak_waiter_count").GetInt64()); + Assert.Equal(Stamp(@base.AddMinutes(-30)), windowRow.GetProperty("peak_waiters_at").GetString()); + Assert.Equal(3, windowRow.GetProperty("timeout_errors_in_window").GetInt64()); + Assert.Equal(6000d, windowRow.GetProperty("peak_granted_memory_mb").GetDouble()); + Assert.Equal(2000d, windowRow.GetProperty("min_available_memory_mb").GetDouble()); + Assert.Equal(Stamp(@base), windowRow.GetProperty("last_snapshot_at").GetString()); + + var grants = Parse(await DarlingMcpMemoryGrantTools.GetMemoryGrants(postgres, ServerName, 1, as_of: anchor)); + Assert.Equal(Stamp(@base), grants.GetProperty("captured_at").GetString()); + Assert.Equal(300, grants.GetProperty("age_seconds").GetInt64()); + var poolWindow = Assert.Single(grants.GetProperty("window").EnumerateArray()); + Assert.Equal(JsonValueKind.Null, poolWindow.GetProperty("resource_semaphore_id").ValueKind); + Assert.Equal(12, poolWindow.GetProperty("peak_waiter_count").GetInt64()); + + /* ── scheduler: one snapshot three hours before the anchor — inside a 4 h search, outside a 1 h one ── */ + var schedulerAt = @base.AddMinutes(5).AddHours(-3); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO cpu_scheduler_stats (collection_id, collection_time, server_id, server_name, max_workers_count, scheduler_count, cpu_count, total_runnable_tasks_count, total_work_queue_count, total_current_workers_count, avg_runnable_tasks_count, total_active_request_count, total_queued_request_count, total_blocked_task_count, total_active_parallel_thread_count, runnable_percent, worker_thread_exhaustion_warning, runnable_tasks_warning, blocked_tasks_warning, queued_requests_warning, total_physical_memory_kb, available_physical_memory_kb, physical_memory_pressure_warning, total_node_count, nodes_online_count, offline_cpu_count, offline_cpu_warning) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27)", + CollectionIdGenerator.Next(), schedulerAt, ServerId, ServerName, 512, 8, 8, 60, 5L, 100, 7.5m, 40, 12, 2, 20L, 12.5m, false, true, false, true, 65536000L, 32768000L, false, 1, 1, 0, false); + + var scheduler = Parse(await DarlingMcpPlanCacheSchedulerTools.GetCpuSchedulerPressure(postgres, ServerName, 4, as_of: anchor)); + Assert.Equal(Stamp(schedulerAt), scheduler.GetProperty("captured_at").GetString()); + Assert.Equal(3 * 3600, scheduler.GetProperty("age_seconds").GetInt64()); + Assert.StartsWith("CRITICAL", scheduler.GetProperty("pressure_level").GetString(), StringComparison.Ordinal); + Assert.Equal("unavailable", DarlingMcpTestData.StatusOf(await DarlingMcpPlanCacheSchedulerTools.GetCpuSchedulerPressure(postgres, ServerName, 1, as_of: anchor))); + + /* ── config: captured on connect, stamped with that connect ── */ + var connectAt = @base.AddDays(-3); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO server_config (config_id, capture_time, server_id, server_name, configuration_name, value_configured, value_in_use, is_dynamic, is_advanced) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + CollectionIdGenerator.Next(), connectAt, ServerId, ServerName, "max degree of parallelism", 4L, 4L, true, true); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO trace_flags (config_id, capture_time, server_id, server_name, trace_flag, status, is_global, is_session) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + CollectionIdGenerator.Next(), connectAt, ServerId, ServerName, 3226, true, true, false); + + Assert.Equal(Stamp(connectAt), Parse(await DarlingMcpConfigTools.GetServerConfig(postgres, ServerName)).GetProperty("captured_at").GetString()); + Assert.Equal(Stamp(connectAt), Parse(await DarlingMcpConfigTools.GetTraceFlags(postgres, ServerName)).GetProperty("captured_at").GetString()); + + /* ── clerks: the newest snapshot's stamp, not the older one's ── */ + foreach (var t in new[] { @base.AddMinutes(-10), @base }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO memory_clerks (collection_id, collection_time, server_id, server_name, clerk_type, memory_mb) +VALUES ($1,$2,$3,$4,$5,$6)", CollectionIdGenerator.Next(), t, ServerId, ServerName, "MEMORYCLERK_SQLBUFFERPOOL", 40000m); + } + Assert.Equal(Stamp(@base), Parse(await DarlingMcpDataTools.GetMemoryClerks(postgres, ServerName)).GetProperty("captured_at").GetString()); + + /* ── latch: hot earlier, quiet now — LOW severity beside a large window total, and the band says why ── */ + foreach (var (t, delta) in new[] { (@base.AddMinutes(-20), 20000L), (@base, 100L) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO latch_stats (collection_id, collection_time, server_id, server_name, latch_class, waiting_requests_count, wait_time_ms, max_wait_time_ms, delta_waiting_requests_count, delta_wait_time_ms, delta_max_wait_time_ms, sample_interval_seconds) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, "ACCESS_METHODS_DATASET_PARENT", 1000L, 20100L, 50L, 100L, delta, 5L, 60); + } + var latch = Assert.Single(Parse(await DarlingMcpLatchSpinlockTools.GetLatchStats(postgres, ServerName, 1, as_of: anchor)).GetProperty("latches").EnumerateArray()); + Assert.Equal(20100, latch.GetProperty("total_delta_wait_time_ms").GetInt64()); + Assert.Equal("LOW", latch.GetProperty("severity").GetString()); + var band = latch.GetProperty("severity_banded_from"); + Assert.Equal(100, band.GetProperty("delta_wait_time_ms").GetInt64()); + Assert.Equal(60d, band.GetProperty("interval_seconds").GetDouble()); + Assert.Equal(Stamp(@base), band.GetProperty("captured_at").GetString()); + + /* ── server summary: three clocks — a stale CPU row under a fresh collection log ── */ + var cpuAt = @base.AddDays(-1); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO cpu_utilization_stats (collection_id, collection_time, server_id, server_name, sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization) +VALUES ($1,$2,$3,$4,$5,$6,$7)", CollectionIdGenerator.Next(), cpuAt, ServerId, ServerName, cpuAt, 42, 3); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO memory_stats (collection_id, collection_time, server_id, server_name, total_physical_memory_mb, available_physical_memory_mb, total_server_memory_mb) +VALUES ($1,$2,$3,$4,$5,$6,$7)", CollectionIdGenerator.Next(), @base, ServerId, ServerName, 65536m, 8192m, 40000m); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, rows_collected) +VALUES ($1,$2,$3,$4,$5,120,'SUCCESS',7)", CollectionIdGenerator.Next(), ServerId, ServerName, "memory_stats", @base.AddMinutes(1)); + + var summary = Parse(await DarlingMcpHealthTools.GetServerSummary(postgres, ServerName)); + Assert.Equal(Stamp(cpuAt), summary.GetProperty("cpu_captured_at").GetString()); + Assert.Equal(Stamp(@base), summary.GetProperty("memory_captured_at").GetString()); + Assert.Equal(Stamp(@base.AddMinutes(1)), summary.GetProperty("last_collection").GetString()); + Assert.Equal(DarlingHealthReader.ServerSummaryCountsWindowHours, summary.GetProperty("counts_window_hours").GetInt32()); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + private static JsonElement Parse(string json) + { + Assert.False(json.StartsWith("Error during", StringComparison.Ordinal), $"tool returned an error: {json}"); + var root = JsonDocument.Parse(json).RootElement.Clone(); + Assert.False(root.TryGetProperty("status", out _), "expected a data-bearing payload, got a status envelope: " + json); + return root; + } + + /// A seeded naive-UTC instant as the tools emit it: ToString("o") on a Kind=Unspecified + /// value, so no Z. + private static string Stamp(DateTime naiveUtc) => DateTime.SpecifyKind(naiveUtc, DateTimeKind.Unspecified).ToString("o"); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) + { + var sql = string.Join(" ", Tables.Select(t => $"DELETE FROM {t} WHERE server_id = {ServerId};")) + + $" DELETE FROM servers WHERE server_id = {ServerId};"; + using var cleanup = new NpgsqlCommand(sql, connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/McpPageContractTests.cs b/Darling/Darling.Tests/McpPageContractTests.cs new file mode 100644 index 000000000..bcc624b63 --- /dev/null +++ b/Darling/Darling.Tests/McpPageContractTests.cs @@ -0,0 +1,1124 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// #3541 A3: six MCP tool groups published a HIDDEN reader cap as if it were the window — LIMIT 200 / +/// 50 / 500 under a tool that advertised limit, a Take(limit) on top, and the capped +/// count emitted under a total_* name. An agent that cannot see the code read a 200-row page of a +/// 5,000-event window as the window. #3287 fixed one instance (get_collection_log) and established the +/// pattern; this file pins that pattern across every tool the finding named, on BOTH SKUs, as ONE dialect: +/// +/// +/// the cap is the caller's limit, bound as a SQL parameter, never a literal; +/// truncation is OBSERVED by fetching limit + 1 and comparing Count > limit — never +/// inferred from Count >= limit, which cannot tell a window of exactly limit rows from a busier +/// one; +/// the page publishes its own time bounds under the oldest_returned_* / newest_returned_* names +/// get_collection_log established, and names its ordering; +/// no page count is called total_*; +/// a filter that shapes the population is stated and measured (get_alert_history's +/// dismissed = FALSE); +/// the same tool name emits the same page-contract keys on both SKUs. +/// +/// +/// Darling's half is reflected off the assembly; Lite's half is read from source, because this project +/// does not (and should not) reference the desktop app — the same arrangement +/// uses. The discriminators are exercised against literals +/// written for the purpose, because a census whose matcher has quietly stopped matching reports a clean bill +/// of health. Lite.Tests/McpPageContractTests executes the Lite tools against a real DuckDB; +/// executes the Darling ones against live Postgres. +/// +/// #3541 A7 extends the dialect to PERCENTS. Five PostgreSQL tools divided every row by the sum +/// of the rows they had fetched and published the result as pct_of_total_*, so a three-row page summed +/// to 100% of "total" by construction. The rule added here: a share's denominator is the WINDOW's figure, +/// computed on the same statement as the rows (SUM(...) OVER () over the grouped result, before +/// LIMIT), published as total_*; the page's own sum travels as returned_*; and every +/// description names which denominator its shares use. *_of_returned remains the ONE other spelling, +/// for a share that genuinely is of the page and says so (get_pg_database_stats' cache ratio). The +/// arithmetic through each projection is ' subject; this +/// file holds the census. +/// +public sealed class McpPageContractTests +{ + /* ───────────────────────── the census ───────────────────────── */ + + /// + /// Every tool the finding named, by SKU file. The Lite get_blocked_process_reports is Darling's + /// get_blocking under another name (the one pair CrossAppMcpToolInventoryPinTests records + /// as drift), so it is listed under Darling's name here and the twin-parity test maps it. + /// + public static readonly (Type Tools, string ToolName, string LiteFile, string LiteToolName)[] PagedTools = + [ + (typeof(DarlingMcpBlockingTools), "get_blocking", "Lite/Mcp/McpBlockingTools.cs", "get_blocked_process_reports"), + (typeof(DarlingMcpBlockingTools), "get_deadlocks", "Lite/Mcp/McpBlockingTools.cs", "get_deadlocks"), + (typeof(DarlingMcpBlockingTools), "get_deadlock_detail", "Lite/Mcp/McpBlockingTools.cs", "get_deadlock_detail"), + (typeof(DarlingMcpBlockingTools), "get_blocked_process_xml", "Lite/Mcp/McpBlockingTools.cs", "get_blocked_process_xml"), + (typeof(DarlingMcpAlertTools), "get_alert_history", "Lite/Mcp/McpAlertTools.cs", "get_alert_history"), + (typeof(DarlingMcpLongQueryTools), "get_long_query_completions", "Lite/Mcp/McpLongQueryTools.cs", "get_long_query_completions"), + (typeof(DarlingMcpPlanCorrectionTools), "get_plan_corrections", "Lite/Mcp/McpPlanCorrectionTools.cs", "get_plan_corrections"), + (typeof(DarlingMcpSessionTools), "get_waiting_tasks", "Lite/Mcp/McpWaitTools.cs", "get_waiting_tasks"), + (typeof(DarlingMcpDataTools), "get_wait_stats", "Lite/Mcp/McpWaitTools.cs", "get_wait_stats"), + /* #3541 A13: get_active_queries joined the dialect when its filters moved into the SQL — it now pages + the FILTERED population at limit + 1, publishes snapshots_returned / truncated / the page's bounds, + and its total_snapshots is the filtered population's COUNT(*) OVER () rather than rows.Count of an + unfiltered window read. */ + (typeof(DarlingMcpSessionTools), "get_active_queries", "Lite/Mcp/McpSessionTools.cs", "get_active_queries"), + ]; + + /// + /// The source span of every paged tool on both SKUs — the census walks these BODIES, not whole files, + /// for the total_* = .Count and >= limit shapes. Whole files would sweep in tools with a + /// different contract: get_mute_rules' total_count counts a whole set, which is not a hidden + /// cap, and a census that flagged it would be asserting a rule the finding did not state. + /// (get_active_queries used to be the other exclusion, for publishing an unbounded window count + /// beside its page; #3541 A13 made that count the filtered population's, computed in SQL, so it is a + /// member now.) + /// + private static IEnumerable<(string Label, string Body)> PagedToolBodies() + { + foreach (var (type, darlingName, liteFile, liteName) in PagedTools) + { + yield return ($"Darling {darlingName}", ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), darlingName)); + yield return ($"Lite {liteName}", ToolBody(ReadRepoFileLf(liteFile.Split('/')), liteName)); + } + } + + /// + /// The Darling reader consts behind the paged tools, each of which must bind its cap as a parameter. + /// + private static readonly (string Name, string Sql)[] PagedReads = + [ + (nameof(DarlingBlockingReader.BlockedProcessReportsSql), DarlingBlockingReader.BlockedProcessReportsSql), + (nameof(DarlingBlockingReader.BlockedProcessReportsWithXmlSql), DarlingBlockingReader.BlockedProcessReportsWithXmlSql), + (nameof(DarlingBlockingReader.DmvBlockingSnapshotsSql), DarlingBlockingReader.DmvBlockingSnapshotsSql), + (nameof(DarlingBlockingReader.RecentDeadlocksSql), DarlingBlockingReader.RecentDeadlocksSql), + (nameof(DarlingBlockingReader.RecentDeadlocksWithGraphSql), DarlingBlockingReader.RecentDeadlocksWithGraphSql), + (nameof(DarlingAlertReader.AlertHistorySql), DarlingAlertReader.AlertHistorySql), + (nameof(DarlingAlertReader.AlertHistoryAllServersSql), DarlingAlertReader.AlertHistoryAllServersSql), + (nameof(DarlingLongQueryReader.LongQueryCompletionsSql), DarlingLongQueryReader.LongQueryCompletionsSql), + (nameof(DarlingPlanCorrectionReader.PlanCorrectionsSql), DarlingPlanCorrectionReader.PlanCorrectionsSql), + (nameof(DarlingSessionReader.WaitingTasksSql), DarlingSessionReader.WaitingTasksSql), + (nameof(DarlingDataReader.WaitStatsSql), DarlingDataReader.WaitStatsSql), + (nameof(DarlingSessionReader.ActiveQueriesSql), DarlingSessionReader.ActiveQueriesSql), + ]; + + /* ───────────────────────── the discriminators ───────────────────────── */ + + /// A page count emitted under a window's name — the shape the finding was about. + private static readonly Regex TotalOfPageCount = new(@"\btotal_\w+\s*=\s*\w+\.Count\b", RegexOptions.Compiled); + + /// Truncation inferred from the cap rather than observed past it. + private static readonly Regex TruncationInferred = new(@"\btruncated\s*=\s*\w+\.Count\s*>=\s*", RegexOptions.Compiled); + + /// Truncation observed: the row past limit came back. + private static readonly Regex TruncationObserved = new(@"var truncated = \w+\.Count > limit;", RegexOptions.Compiled); + + /// A literal row cap in a paged read. LIMIT 1), 0) is the single-row offset CTE and is not a + /// page cap; it is stripped before the match rather than allow-listed by number, because LIMIT 1 + /// elsewhere WOULD be a defect. + private static readonly Regex LiteralLimit = new(@"\bLIMIT\s+\d+\b", RegexOptions.Compiled); + + private static readonly Regex ParameterLimit = new(@"\bLIMIT \$\d+\s*$", RegexOptions.Compiled); + + /// The page-contract keys a tool body emits: its count, its bounds, its ordering. + private static readonly Regex ContractKeys = new( + @"\b(\w+_returned|oldest_returned_\w+|newest_returned_\w+)\s*=|\border = ""([a-z_]+)""", + RegexOptions.Compiled); + + /* ───────────────────────── Darling, reflected ───────────────────────── */ + + [Fact] + public void EveryPagedTool_SaysWhatBoundsItsPage_InItsDescription() + { + foreach (var (type, name, _, _) in PagedTools) + { + var method = ToolMethod(type, name); + var description = method.GetCustomAttribute()!.Description; + Assert.True(description.Contains("truncated", StringComparison.Ordinal), + $"{name}: the description never mentions `truncated`, so an agent reads the page as the window"); + Assert.True(description.Contains("limit", StringComparison.Ordinal), + $"{name}: the description never says the page is bounded by limit"); + + var limit = method.GetParameters().Single(p => p.Name == "limit"); + Assert.Contains("truncated", limit.GetCustomAttribute()!.Description, StringComparison.Ordinal); + } + } + + /// The three incident readers that take a dedup_key promise (#2159) that the fingerprint + /// scan runs over the window before limit; with the scan now bounded by a stated ceiling, each must + /// say so and name the two fields that report it. + [Theory] + [InlineData("get_blocking")] + [InlineData("get_deadlocks")] + public void FingerprintReaders_NameTheScanCeilingFields(string toolName) + { + var method = ToolMethod(typeof(DarlingMcpBlockingTools), toolName); + var description = method.GetCustomAttribute()!.Description; + Assert.Contains("rows_examined", description, StringComparison.Ordinal); + Assert.Contains("scan_truncated", description, StringComparison.Ordinal); + + var key = method.GetParameters().Single(p => p.Name == "dedup_key"); + Assert.Contains("BEFORE limit", key.GetCustomAttribute()!.Description, StringComparison.Ordinal); + } + + [Fact] + public void EveryPagedRead_BindsItsCapAsAParameter_NeverALiteral() + { + foreach (var (name, sql) in PagedReads) + { + var body = sql.Replace("LIMIT 1), 0)", string.Empty, StringComparison.Ordinal); + Assert.False(LiteralLimit.IsMatch(body), + $"{name} caps with a literal the caller cannot see: {LiteralLimit.Match(body).Value}"); + Assert.True(ParameterLimit.IsMatch(sql.TrimEnd()), + $"{name} does not end in a parameterised LIMIT, so the tool's limit + 1 over-fetch has nothing to bind to"); + } + } + + /* ───────────────────────── both SKUs, from source ───────────────────────── */ + + [Fact] + public void NoPagedTool_PublishesAPageCountAsATotal_OnEitherSku() + { + foreach (var (label, body) in PagedToolBodies()) + { + var text = Strip(body); + var hit = TotalOfPageCount.Match(text); + Assert.False(hit.Success, + $"{label}: `{hit.Value}` publishes a page count under a window's name — rename it *_returned, or " + + "compute a real windowed COUNT and publish both, clearly named"); + } + } + + [Fact] + public void EveryPagedTool_ObservesTruncation_AndNeverInfersIt_OnEitherSku() + { + foreach (var (label, body) in PagedToolBodies()) + { + var text = Strip(body); + var inferred = TruncationInferred.Match(text); + Assert.False(inferred.Success, + $"{label}: `{inferred.Value}` infers truncation from the cap; fetch limit + 1 and compare Count > limit"); + Assert.True(TruncationObserved.IsMatch(text), + $"{label}: no `var truncated = x.Count > limit;` — a paged tool that never observes its own cap"); + /* The over-fetch that makes the observation possible: the reader is asked for one row past the cap. */ + Assert.Contains("limit + 1", text, StringComparison.Ordinal); + } + } + + /// + /// One dialect, not six: the same tool name emits the same page-contract keys on both SKUs. Compared per + /// TOOL BODY rather than per file, because the blocking files host tools with different bounds + /// (event_time for reports, deadlock_time for deadlocks). The naming-drift pair is compared + /// on bounds and ordering only, since its count key follows the tool's own noun. + /// + [Fact] + public void TheSameToolName_EmitsTheSamePageContractKeys_OnBothSkus() + { + foreach (var (type, darlingName, liteFile, liteName) in PagedTools) + { + var darlingFile = DarlingFileOf(type); + var darlingKeys = KeysOf(Strip(ToolBody(ReadRepoFileLf(darlingFile.Split('/')), darlingName))); + var liteKeys = KeysOf(Strip(ToolBody(ReadRepoFileLf(liteFile.Split('/')), liteName))); + + Assert.NotEmpty(darlingKeys); + + if (darlingName != liteName) + { + /* get_blocking ↔ get_blocked_process_reports: the noun differs, the bounds must not. */ + darlingKeys.RemoveWhere(k => k.EndsWith("_returned", StringComparison.Ordinal)); + liteKeys.RemoveWhere(k => k.EndsWith("_returned", StringComparison.Ordinal)); + } + + Assert.True(darlingKeys.SetEquals(liteKeys), + $"{darlingName} ↔ {liteName}: Darling emits [{string.Join(", ", darlingKeys.Order())}], " + + $"Lite emits [{string.Join(", ", liteKeys.Order())}] — same tool name, different page contract"); + } + } + + /// The Lite descriptions carry the same commitment as Darling's, read from source. + [Fact] + public void EveryLitePagedTool_SaysWhatBoundsItsPage_InItsDescription() + { + foreach (var (_, _, liteFile, liteName) in PagedTools) + { + var body = ToolBody(ReadRepoFileLf(liteFile.Split('/')), liteName); + var description = Regex.Match(body, @"Description\((?:\s*)""((?:[^""\\]|\\.)*)""\)\]").Groups[1].Value; + Assert.False(string.IsNullOrEmpty(description), $"{liteFile}: could not locate {liteName}'s Description"); + Assert.Contains("truncated", description, StringComparison.Ordinal); + Assert.Contains("limit", description, StringComparison.Ordinal); + } + } + + /* ───────────────────────── the matchers, witnessed ───────────────────────── */ + + [Fact] + public void TheDiscriminators_FlagTheDefectShapes_AndPassTheFixedOnes() + { + Assert.Matches(TotalOfPageCount, " total_events = rows.Count,"); + Assert.DoesNotMatch(TotalOfPageCount, " events_returned = page.Count,"); + /* A real windowed total is not a page count and must pass. */ + Assert.DoesNotMatch(TotalOfPageCount, " total_deadlocks = totalDeadlocks,"); + + Assert.Matches(TruncationInferred, " var truncated = rows.Count >= limit;"); + Assert.DoesNotMatch(TruncationInferred, " var truncated = rows.Count > limit;"); + Assert.Matches(TruncationObserved, " var truncated = candidates.Count > limit;"); + + Assert.Matches(LiteralLimit, " ORDER BY event_time DESC\n LIMIT 200\n"); + Assert.DoesNotMatch(LiteralLimit, " ORDER BY event_time DESC\n LIMIT $4\n"); + Assert.Matches(ParameterLimit, " LIMIT $4"); + + var keys = KeysOf("deadlocks_returned = page.Count,\n truncated,\n oldest_returned_deadlock_time = x,\n newest_returned_deadlock_time = y,\n order = \"deadlock_time_desc\","); + Assert.Equal( + new[] { "deadlocks_returned", "newest_returned_deadlock_time", "oldest_returned_deadlock_time", "order:deadlock_time_desc" }, + keys.Order().ToArray()); + } + + /* ───────────────────────── #3541 A7: percents name their denominator ───────────────────────── */ + + /// + /// The five tools whose shares were of the page, with the three keys each must now publish: the per-row + /// share, the WINDOW total it divides by, and the page's own sum under a name that says so. Darling-only — + /// Lite has no PostgreSQL tools — so there is no twin-parity arm; the cross-SKU sweep below is the + /// negative census over every paged tool body on BOTH SKUs instead. + /// + public static readonly (Type Tools, string ToolName, string ShareKey, string TotalKey, string ReturnedKey)[] PercentTools = + [ + (typeof(DarlingMcpPgStatementTools), "get_pg_top_queries", "pct_of_total_time", "total_exec_time_ms", "returned_exec_time_ms"), + (typeof(DarlingMcpPgWaitTools), "get_pg_wait_stats", "pct_of_total_wait", "total_wait_time_ms", "returned_wait_time_ms"), + (typeof(DarlingMcpPgWaitSamplingTools), "get_pg_wait_sampling", "pct_of_samples", "total_samples", "returned_samples"), + (typeof(DarlingMcpPgKernelStatsTools), "get_pg_kernel_stats", "pct_of_total_cpu", "total_cpu_ms", "returned_cpu_ms"), + (typeof(DarlingMcpPgIoTools), "get_pg_io_stats", "pct_of_total_reads", "total_reads", "returned_reads"), + ]; + + /// The reader consts behind them: each must carry its window total on the SAME statement as the + /// rows and bind its cap as a parameter. + private static readonly (string Name, string Sql)[] PercentReads = + [ + (nameof(DarlingPgStatementReader.PgTopQueriesSql), DarlingPgStatementReader.PgTopQueriesSql), + (nameof(DarlingPgWaitReader.PgWaitStatsSql), DarlingPgWaitReader.PgWaitStatsSql), + (nameof(DarlingPgWaitSamplingReader.PgWaitSamplingSql), DarlingPgWaitSamplingReader.PgWaitSamplingSql), + (nameof(DarlingPgKernelStatsReader.PgKernelStatsSql), DarlingPgKernelStatsReader.PgKernelStatsSql), + (nameof(DarlingPgIoReader.PgIoSql), DarlingPgIoReader.PgIoSql), + ]; + + /// A local that is the sum of a fetched collection — the page sum the defect divided by. + private static readonly Regex PageSumLocal = new(@"\bvar\s+(\w+)\s*=\s*\w+\s*\.Sum\(", RegexOptions.Compiled); + + /// A share key whose divisor is the named local. [^,;] keeps the match inside one + /// initializer entry: the first comma in every share expression here is the one inside + /// Math.Round(x, 1), and the division precedes it. + private static Regex ShareOverLocal(string local) => + new($@"\b\w*pct\w*\s*=[^,;]*?/\s*{Regex.Escape(local)}\b", RegexOptions.Compiled); + + /// A total_* key assigned from the named local, rounded or bare. + private static Regex TotalFromLocal(string local) => + new($@"\btotal_\w+\s*=\s*(?:Math\.Round\(\s*)?{Regex.Escape(local)}\b", RegexOptions.Compiled); + + /// The window total on the reader's statement: an OVER () aggregate aliased window_total_*. + private static readonly Regex WindowTotalColumn = new(@"\)\s+OVER \(\)(?:\s*/\s*1000\.0)?(?:\s+AS\s+bigint\))?\s+AS\s+window_total_\w+", RegexOptions.Compiled); + + /// The page-record read the fixed tools go through: the tool fetches limit + 1 and observes. + private static readonly Regex PageTruncationObserved = new(@"var truncated = page\.Rows\.Count > limit;", RegexOptions.Compiled); + + [Fact] + public void EveryPercentTool_NamesItsDenominator_AndItsPageBound_InItsDescription() + { + foreach (var (type, name, shareKey, totalKey, returnedKey) in PercentTools) + { + var method = ToolMethod(type, name); + var description = method.GetCustomAttribute()!.Description; + + Assert.Contains("truncated", description, StringComparison.Ordinal); + Assert.Contains("limit", description, StringComparison.Ordinal); + /* The sentence that makes the numbers readable: which denominator, and that it is not the page. */ + Assert.Contains("SHARES ARE OF THE WINDOW, NOT OF THE PAGE", description, StringComparison.Ordinal); + Assert.Contains(shareKey, description, StringComparison.Ordinal); + Assert.Contains(totalKey, description, StringComparison.Ordinal); + Assert.Contains(returnedKey, description, StringComparison.Ordinal); + Assert.Contains("does not sum to 100%", description, StringComparison.Ordinal); + + var limit = method.GetParameters().Single(p => p.Name == "limit"); + var limitDescription = limit.GetCustomAttribute()!.Description; + Assert.Contains("truncated", limitDescription, StringComparison.Ordinal); + Assert.Contains("whole window", limitDescription, StringComparison.Ordinal); + } + } + + /// + /// The denominator is on the SAME statement as the rows — a window aggregate over the grouped result, + /// which PostgreSQL evaluates before LIMIT — so it cannot drift from them and costs no second read. + /// And the cap is a parameter: get_pg_top_queries carried LIMIT 50 as a literal under a limit + /// the tool accepts up to 1,000, which is the A3 shape and the reason its share had TWO wrong denominators. + /// + [Fact] + public void EveryPercentRead_CarriesItsWindowTotalOnTheSameStatement_AndBindsItsCap() + { + foreach (var (name, sql) in PercentReads) + { + Assert.True(WindowTotalColumn.IsMatch(sql), + $"{name} has no `... OVER () AS window_total_*` column, so the tool has nothing but the page to divide by"); + Assert.False(LiteralLimit.IsMatch(sql), $"{name} caps with a literal: {LiteralLimit.Match(sql).Value}"); + Assert.True(ParameterLimit.IsMatch(sql.TrimEnd()), $"{name} does not end in a parameterised LIMIT"); + /* Before the cap, so it is the window's and not the page's: the OVER () sits above the LIMIT. */ + Assert.True(sql.IndexOf("OVER ()", StringComparison.Ordinal) < sql.LastIndexOf("LIMIT", StringComparison.Ordinal)); + } + } + + /// + /// The kernel read's ordering was ORDER BY 3 + 4 DESC — meant as two ordinals, read by PostgreSQL as + /// the constant 7 and dropped, so the "ranked by CPU" page was the alphabetically-first series. Found when + /// A7's three-row page came back 60 / 10 / 30 against live PostgreSQL 18. Pinned by SHAPE: the sort key + /// names the two differenced columns, and no integer-expression ordinal survives in any of these reads. + /// + [Fact] + public void TheKernelRead_RanksByCpu_NotByAConstant() + { + var sql = DarlingPgKernelStatsReader.PgKernelStatsSql; + Assert.Contains("ORDER BY user_ms + system_ms DESC", sql, StringComparison.Ordinal); + + foreach (var (name, read) in PercentReads) + { + Assert.False(Regex.IsMatch(read, @"ORDER BY\s+\d+\s*[-+*/]\s*\d+"), + $"{name} orders by an arithmetic expression on ordinals, which PostgreSQL folds to a constant and ignores"); + } + } + + /// + /// The sampling read cannot name its sample_count alias inside a window function of the same + /// level, so the differencing CASE is written twice — once as the row figure, once inside the + /// SUM(...) OVER (). Two copies of one expression drift; this holds them identical modulo + /// whitespace, so the window total is provably the sum of the very figure the rows report. + /// + [Fact] + public void TheSamplingRead_SumsTheSameCaseItReportsPerRow() + { + var cases = Regex.Matches(DarlingPgWaitSamplingReader.PgWaitSamplingSql, @"CASE WHEN n\.sample_count.*?\bEND\b", RegexOptions.Singleline) + .Select(m => Regex.Replace(m.Value, @"\s+", " ")) + .ToArray(); + + Assert.Equal(2, cases.Length); + Assert.Equal(cases[0], cases[1]); + Assert.Contains("coalesce(o.sample_count, 0)", cases[0], StringComparison.Ordinal); + } + + /// + /// The positive half over the five tool bodies: the share divides by the page record's window total, the + /// page's own sum reaches only a returned_* key, truncation is observed off the limit + 1 fetch. + /// + [Fact] + public void EveryPercentTool_DividesByTheWindowTotal_AndPublishesThePageSumAsReturned() + { + foreach (var (type, name, shareKey, totalKey, returnedKey) in PercentTools) + { + var body = Strip(ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), name)); + + Assert.Contains("page.WindowTotal", body, StringComparison.Ordinal); + Assert.True(PageTruncationObserved.IsMatch(body), $"{name}: truncation is not observed off the page record"); + Assert.Contains("limit + 1", body, StringComparison.Ordinal); + Assert.Contains($"{returnedKey} =", body, StringComparison.Ordinal); + Assert.Contains($"{totalKey} =", body, StringComparison.Ordinal); + Assert.Contains($"{shareKey} =", body, StringComparison.Ordinal); + + foreach (Match local in PageSumLocal.Matches(body)) + { + var sum = local.Groups[1].Value; + Assert.False(ShareOverLocal(sum).IsMatch(body), + $"{name}: a share divides by `{sum}`, which is a sum of the rows fetched — divide by the page's window total"); + Assert.False(TotalFromLocal(sum).IsMatch(body), + $"{name}: a total_* key is `{sum}`, a sum of the rows fetched — publish it as returned_* and the window's as total_*"); + } + } + } + + /// + /// The negative half, over EVERY tool body on both SKUs that takes a limit or top: no share + /// anywhere divides by a sum of the rows fetched, and no total_* key is one. Scoped to paged tools + /// because an unpaged read's sum over all its rows IS a total (get_session_summary, + /// get_plan_cache_stats) and a census that flagged those would be asserting a rule the finding did + /// not state. + /// + /// One stated allowance. get_pg_database_stats publishes total_temp_files / + /// total_temp_bytes / total_deadlocks summed over its top-N page — beside limit_reached, + /// a note that says the totals cover only the databases returned, a database_count the web tile + /// labels "Databases returned", and a share it already spells _of_returned. Honest by disclosure + /// rather than by name; moving it to the window idiom means relabelling the web tile's figures, which + /// sits outside the lane that added this census. Named here so it fails loudly the day the allowance is + /// no longer needed, rather than being carried silently. + /// + [Fact] + public void NoPagedTool_DividesByOrPublishesAPageSumAsATotal_OnEitherSku() + { + var allowedPageSummedTotals = new HashSet(StringComparer.Ordinal) + { + "get_pg_database_stats:total_temp_files", + "get_pg_database_stats:total_temp_bytes", + "get_pg_database_stats:total_deadlocks", + }; + var allowancesUsed = new HashSet(StringComparer.Ordinal); + var examined = 0; + + foreach (var (file, source) in AllMcpToolSources()) + { + var marks = Regex.Matches(source, @"\[McpServerTool\(Name = ""([a-z_0-9]+)"""); + for (var i = 0; i < marks.Count; i++) + { + var end = i + 1 < marks.Count ? marks[i + 1].Index : source.Length; + var body = source[marks[i].Index..end]; + if (!Regex.IsMatch(body, @"\bint\s+(limit|top)\b")) + { + continue; + } + + examined++; + var toolName = marks[i].Groups[1].Value; + var text = Strip(body); + + foreach (Match local in PageSumLocal.Matches(text)) + { + var sum = local.Groups[1].Value; + var share = ShareOverLocal(sum).Match(text); + Assert.False(share.Success, + $"{file} {toolName}: `{share.Value}` divides by a sum of the rows fetched — a page share under a share-of-total name"); + + var total = TotalFromLocal(sum).Match(text); + if (total.Success) + { + var key = toolName + ":" + Regex.Match(total.Value, @"\btotal_\w+").Value; + Assert.True(allowedPageSummedTotals.Contains(key), + $"{file} {toolName}: `{total.Value}` publishes a sum of the rows fetched under a total's name — publish the window's total, or name it returned_*"); + allowancesUsed.Add(key); + } + } + } + } + + /* Population controls: a sweep that parsed nothing passes for free, and an allowance nobody needs + is a widened exemption waiting for a defect to hide under. 82 bodies at the time of writing. */ + Assert.True(examined >= 60, $"only {examined} paged tool bodies were examined across both SKUs; the marker or the signature pattern has stopped matching"); + Assert.True(allowedPageSummedTotals.SetEquals(allowancesUsed), + "stated allowances no longer match what the sweep finds — remove the ones that are no longer needed: " + + string.Join(", ", allowedPageSummedTotals.Except(allowancesUsed))); + } + + /// The A7 matchers, witnessed against the defect as it shipped and the fix as it landed. + [Fact] + public void TheA7Discriminators_FlagTheDefectShapes_AndPassTheFixedOnes() + { + /* The defect, verbatim from the shipped statement tool. */ + const string defect = """ + var totalTimeMs = rows.Sum(r => r.TotalExecTimeMs); + var result = rows.Take(limit).Select(r => new + { + pct_of_total_time = totalTimeMs > 0 ? Math.Round((double)r.TotalExecTimeMs / totalTimeMs * 100, 1) : 0, + }); + return JsonSerializer.Serialize(new { total_exec_time_ms = totalTimeMs, }); + """; + var local = Assert.Single(PageSumLocal.Matches(defect)).Groups[1].Value; + Assert.Equal("totalTimeMs", local); + Assert.Matches(ShareOverLocal(local), defect); + Assert.Matches(TotalFromLocal(local), defect); + /* The rounded form the wait tool used. */ + Assert.Matches(TotalFromLocal("totalWaitMs"), "total_wait_time_ms = Math.Round(totalWaitMs, 1),"); + /* A multi-line share, the sampling tool's shape. */ + Assert.Matches(ShareOverLocal("totalSamples"), "pct_of_samples = totalSamples > 0\n ? Math.Round((double)r.SampleCount / totalSamples * 100, 1)\n : 0,"); + + /* The fix: the page sum feeds only returned_*, the share divides by the window total. */ + const string fixedShape = """ + var truncated = page.Rows.Count > limit; + var windowTotalMs = page.WindowTotalExecTimeMs; + var returnedMs = rows.Sum(r => r.TotalExecTimeMs); + pct_of_total_time = windowTotalMs > 0 ? Math.Round((double)r.TotalExecTimeMs / windowTotalMs * 100, 1) : 0, + total_exec_time_ms = windowTotalMs, + returned_exec_time_ms = returnedMs, + returned_pct_of_total = windowTotalMs > 0 ? Math.Round((double)returnedMs / windowTotalMs * 100, 1) : 0, + """; + var fixedLocal = Assert.Single(PageSumLocal.Matches(fixedShape)).Groups[1].Value; + Assert.Equal("returnedMs", fixedLocal); + Assert.DoesNotMatch(ShareOverLocal(fixedLocal), fixedShape); + Assert.DoesNotMatch(TotalFromLocal(fixedLocal), fixedShape); + Assert.Matches(PageTruncationObserved, fixedShape); + + Assert.Matches(WindowTotalColumn, "CAST(SUM(SUM(delta_total_exec_time_ms)) OVER () AS bigint) AS window_total_exec_time_ms"); + Assert.Matches(WindowTotalColumn, "SUM(SUM(delta_wait_time_us)) OVER () / 1000.0 AS window_total_wait_time_ms"); + Assert.Matches(WindowTotalColumn, "SUM(user_ms + system_ms) OVER () AS window_total_cpu_ms"); + Assert.DoesNotMatch(WindowTotalColumn, "GREATEST(reads - LAG(reads) OVER series, 0) AS d_reads"); + } + + /* ───────────────────────── #3541 A13: a filter is part of the query ───────────────────────── */ + + /// + /// The tools whose filters ran in C# AFTER the read — over a page the SQL had already cut, or over a + /// whole-window read whose count was then published beside the filtered page. Each body, on both SKUs, + /// must now hand every filter to its reader and never .Where( the rows between the read and the + /// emit: a filter applied after the cut makes the page the filtered remainder of an unfiltered top-N, + /// which can be EMPTY while the window holds matches, and a count taken before the filter is a total of + /// a different population from the rows beside it. + /// + public static readonly (Type Tools, string ToolName, string LiteFile, string LiteToolName)[] FilterInQueryTools = + [ + (typeof(DarlingMcpDataTools), "get_top_queries_by_cpu", "Lite/Mcp/McpQueryTools.cs", "get_top_queries_by_cpu"), + (typeof(DarlingMcpSessionTools), "get_active_queries", "Lite/Mcp/McpSessionTools.cs", "get_active_queries"), + ]; + + /// A LINQ filter over the rows a reader returned — the shape that puts the cut before the filter. + private static readonly Regex PostReadWhere = new(@"\.Where\(", RegexOptions.Compiled); + + /// A parameter read as its absolute value — the shape that answers a negative window with a + /// positive one and says nothing. + private static readonly Regex AbsOfParameter = new(@"\bMath\.Abs\(\s*(hours_back|hoursBack|days_back|daysBack|limit|top)\b", RegexOptions.Compiled); + + [Fact] + public void NoFilteredTool_FiltersItsRowsAfterTheRead_OnEitherSku() + { + foreach (var (type, darlingName, liteFile, liteName) in FilterInQueryTools) + { + foreach (var (label, body) in new[] + { + ($"Darling {darlingName}", ToolBody(ReadRepoFileLf(DarlingFileOf(type).Split('/')), darlingName)), + ($"Lite {liteName}", ToolBody(ReadRepoFileLf(liteFile.Split('/')), liteName)), + }) + { + var text = Strip(body); + var hit = PostReadWhere.Match(text); + Assert.False(hit.Success, + $"{label}: `{hit.Value}` filters the rows in C# after the read — push the predicate into the SQL so the page is the top-N of the filtered population, and the count beside it counts that population"); + /* And the filter's presence is STATED on the payload, so a stored result says what shaped it. */ + Assert.True( + text.Contains("filter_applied", StringComparison.Ordinal) || text.Contains("filters_applied", StringComparison.Ordinal), + $"{label}: the payload never names the filter that shaped its population"); + } + } + } + + /// + /// The reader statements behind them carry the predicates: the parallelism floor as a HAVING term on the + /// grouped population BEFORE the CPU ordering and the cap, and the session read's two filters as WHERE + /// terms with the population counted on the same statement above a parameterised LIMIT. + /// + [Fact] + public void TheFilteredReads_CarryTheirPredicates_BeforeTheOrderingAndTheCap() + { + foreach (var (name, sql) in new[] + { + (nameof(DarlingDataReader.TopQueriesSql), DarlingDataReader.TopQueriesSql), + (nameof(DarlingDataReader.TopQueriesByHostObjectSql), DarlingDataReader.TopQueriesByHostObjectSql), + }) + { + var floor = sql.IndexOf("COALESCE(MAX(max_dop), 0) >= $6", StringComparison.Ordinal); + var order = sql.IndexOf("ORDER BY SUM(delta_worker_time) DESC", StringComparison.Ordinal); + Assert.True(floor >= 0, $"{name}: the parallelism floor is not in the statement"); + Assert.True(order > floor, $"{name}: the parallelism floor sits after the ranking, so it filters a ranked page rather than ranking a filtered population"); + } + + var active = DarlingSessionReader.ActiveQueriesSql; + Assert.Contains("($5::text IS NULL OR w.database_name = $5)", active, StringComparison.Ordinal); + Assert.Contains("(NOT $6::boolean OR w.blocking_session_id > 0 OR h.session_id IS NOT NULL)", active, StringComparison.Ordinal); + Assert.Contains("COUNT(*) OVER () AS population_count", active, StringComparison.Ordinal); + /* The head-blocker keep: a WAITFOR row stays when a row in the SAME capture names it. */ + Assert.Contains("(w.query_text NOT LIKE 'WAITFOR%' OR h.session_id IS NOT NULL)", active, StringComparison.Ordinal); + Assert.Contains("h.collection_time = w.collection_time", active, StringComparison.Ordinal); + Assert.True(active.IndexOf("COUNT(*) OVER ()", StringComparison.Ordinal) < active.IndexOf("LIMIT $4", StringComparison.Ordinal), + "the population count must be computed above the cap, or it counts the page"); + } + + /// + /// No tool on either SKU reads a parameter as its absolute value. Three did (hours_back on the + /// uncapped reads), and a caller who sent -24 was answered about the last 24 hours with nothing to + /// say the sign had flipped. Every tool body on both SKUs is swept, comments stripped, because the fix's + /// own comments name the shape. + /// + [Fact] + public void NoTool_ReadsAParameterAsItsAbsoluteValue_OnEitherSku() + { + var examined = 0; + var offenders = new List(); + foreach (var (file, source) in AllMcpToolSources()) + { + var marks = Regex.Matches(source, @"\[McpServerTool\(Name = ""([a-z_0-9]+)"""); + for (var i = 0; i < marks.Count; i++) + { + var end = i + 1 < marks.Count ? marks[i + 1].Index : source.Length; + var body = Strip(source[marks[i].Index..end]); + examined++; + var hit = AbsOfParameter.Match(body); + if (hit.Success) + { + offenders.Add($"{file} {marks[i].Groups[1].Value}: {hit.Value}"); + } + } + } + + Assert.True(examined >= 150, $"only {examined} tool bodies were examined across both SKUs; the marker has stopped matching"); + Assert.True(offenders.Count == 0, + "these tools read a parameter as its absolute value — refuse the negative instead: " + string.Join("; ", offenders)); + } + + /// The three uncapped reads route their span through the shared refusal, on both SKUs. + [Theory] + [InlineData("Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs", "get_collection_log")] + [InlineData("Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs", "get_current_waits_trend")] + [InlineData("Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs", "get_blocking_stats")] + [InlineData("Lite/Mcp/McpHealthTools.cs", "get_collection_log")] + [InlineData("Lite/Mcp/McpHealthTools.cs", "get_current_waits_trend")] + [InlineData("Lite/Mcp/McpHealthTools.cs", "get_blocking_stats")] + public void TheUncappedReads_RefuseANonPositiveSpan_ThroughTheSharedValidator(string file, string toolName) + { + var body = Strip(ToolBody(ReadRepoFileLf(file.Split('/')), toolName)); + Assert.Contains("McpHelpers.ValidateUncappedWindow(hours_back, as_of, out var windowEnd)", body, StringComparison.Ordinal); + Assert.DoesNotContain("McpHelpers.ResolveAsOf(", body, StringComparison.Ordinal); + } + + /// The A13 matchers, witnessed against the defect as it shipped and the fix as it landed. + [Fact] + public void TheA13Discriminators_FlagTheDefectShapes_AndPassTheFixedOnes() + { + /* The defect, verbatim from the shipped queries tool: the page is cut in SQL, then filtered. */ + const string defectFilter = """ + var rows = await DarlingDataReader.GetTopQueriesByCpuAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name); + var filtered = rows + .Where(r => !(parallel_only || min_dop > 1) || (r.MaxDop > 1 && r.MaxDop >= (min_dop > 1 ? min_dop : 2))) + .ToList(); + """; + Assert.Matches(PostReadWhere, defectFilter); + /* The fix: the floor is an argument to the read. */ + const string fixedFilter = """ + var minMaxDop = min_dop > 1 ? min_dop : parallel_only ? 2 : 0; + var rows = await DarlingDataReader.GetTopQueriesByCpuAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name, rollUpByHostObject: rollUp, minMaxDop: minMaxDop); + var result = rows.Select(r => new { max_dop = r.MaxDop }); + """; + Assert.DoesNotMatch(PostReadWhere, fixedFilter); + + /* The defect, verbatim from the three uncapped reads. */ + Assert.Matches(AbsOfParameter, " var start = end.AddHours(-Math.Abs(hours_back));"); + Assert.Matches(AbsOfParameter, " var hours = Math.Abs(hours_back);"); + Assert.DoesNotMatch(AbsOfParameter, " var start = end.AddHours(-hours_back);"); + /* A genuine absolute value of a MEASUREMENT is not a parameter flip and must pass. */ + Assert.DoesNotMatch(AbsOfParameter, " var drift = Math.Abs(observed - expected);"); + } + + /// Every MCP tool source on both SKUs, LF-normalised, for the cross-SKU sweep. Through + /// so a worktree checkout resolves the same root every other pin uses. + private static IEnumerable<(string File, string Source)> AllMcpToolSources() + { + foreach (var directory in new[] { "Darling/PerformanceMonitor.Darling.Service/Mcp", "Lite/Mcp" }) + { + var root = RepoFile.PathTo(directory); + foreach (var file in System.IO.Directory.EnumerateFiles(root, "*.cs").Order(StringComparer.Ordinal)) + { + yield return (System.IO.Path.GetFileName(file), System.IO.File.ReadAllText(file).Replace("\r\n", "\n", StringComparison.Ordinal)); + } + } + } + + /* ───────────────────────── plumbing ───────────────────────── */ + + private static MethodInfo ToolMethod(Type type, string toolName) => type + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == toolName); + + private static string DarlingFileOf(Type type) => + $"Darling/PerformanceMonitor.Darling.Service/Mcp/{type.Name}.cs"; + + /// The source from one tool's [McpServerTool(Name = "…")] attribute to the next tool's, or + /// to the end of the file — the span that holds its description, parameters and payload. + private static string ToolBody(string source, string toolName) + { + var marker = $"[McpServerTool(Name = \"{toolName}\")"; + var start = source.IndexOf(marker, StringComparison.Ordinal); + Assert.True(start >= 0, $"no tool named {toolName} in the source"); + var next = source.IndexOf("[McpServerTool(", start + marker.Length, StringComparison.Ordinal); + return next < 0 ? source[start..] : source[start..next]; + } + + private static HashSet KeysOf(string body) + { + var keys = new HashSet(StringComparer.Ordinal); + foreach (Match m in ContractKeys.Matches(body)) + { + if (m.Groups[1].Success) keys.Add(m.Groups[1].Value); + else keys.Add("order:" + m.Groups[2].Value); + } + return keys; + } + + /// Comments removed, so a comment that NAMES the defect shape (as the fix's comments do) is not + /// read as an instance of it. Line and block comments only; string literals stay, because the payload + /// keys under test are not in strings. + private static string Strip(string source) => + Regex.Replace(Regex.Replace(source, @"/\*.*?\*/", string.Empty, RegexOptions.Singleline), @"//[^\n]*", string.Empty); +} + +/// +/// Gated (DARLING_TEST_PG) live round-trips for the page contract: the same boundary PAIRS +/// Lite.Tests/McpPageContractTests asserts against DuckDB, here against live Postgres through the real +/// tool methods. limit = N - 1 over N seeded rows must read truncated and limit = N must not, +/// because a window holding exactly limit rows is the case count >= limit gets wrong. +/// +[Collection("live-postgres")] +public sealed class McpPageContractLivePostgresTests +{ + private const string ServerName = "darling-mcp-page-contract-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static readonly string[] Tables = + [ + "blocked_process_reports", "dmv_blocking_snapshots", "deadlocks", "config_alert_log", + "long_query_completions", "plan_correction", "waiting_tasks", "wait_stats", + /* #3541 A7: the five PostgreSQL percent tools' tables. */ + "pg_statement_stats", "pg_wait_stats", "pg_wait_sampling", "pg_kernel_stats", "pg_io_stats", + ]; + + [Fact] + public async Task PagedTools_ObserveTruncationAtTheBoundary_AndDescribeThePage_AgainstDevPostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live page-contract test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); + const string Db = "StackOverflow"; + + /* Blocking: 3 XE rows on distinct pairs, plus 1 DMV row on a fourth pair — the merged population + is 4, which neither arm reaches alone. */ + for (var i = 0; i < 3; i++) + { + var t = now.AddMinutes(-10 * i); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, lock_mode, blocked_sql_text, blocking_sql_text, blocked_process_report_xml, contentious_object) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, t, Db, 50 + i, 90, 8000L, "X", "SELECT 1", "UPDATE Posts SET Score = Score + 1", + i == 0 ? null : "", "dbo.Posts"); + } + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, monitor_loop, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, lock_mode, blocking_status, contentious_object, blocked_sql_text, blocking_sql_text) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)", + CollectionIdGenerator.Next(), now.AddMinutes(-5), ServerId, ServerName, -1, now.AddMinutes(-5), Db, 70, 80, 3000L, "S", "suspended", "dbo.Users", "SELECT 2", "WAITFOR DELAY '00:01'"); + + var cut = await DarlingMcpBlockingTools.GetBlocking(postgres, ServerName, 24, 3); + JsonAssert.Contains("\"events_returned\": 3", cut); + JsonAssert.Contains("\"truncated\": true", cut); + var whole = await DarlingMcpBlockingTools.GetBlocking(postgres, ServerName, 24, 4); + JsonAssert.Contains("\"events_returned\": 4", whole); + JsonAssert.Contains("\"truncated\": false", whole); + Assert.DoesNotContain("total_events", whole, StringComparison.Ordinal); + + /* The XML page counts reports WITH XML: two of the three XE rows. */ + var xml = await DarlingMcpBlockingTools.GetBlockedProcessXml(postgres, ServerName, 24, 2); + JsonAssert.Contains("\"reports_returned\": 2", xml); + JsonAssert.Contains("\"truncated\": false", xml); + JsonAssert.Contains("\"truncated\": true", await DarlingMcpBlockingTools.GetBlockedProcessXml(postgres, ServerName, 24, 1)); + + /* Deadlocks: 3 rows, the newest without a graph. */ + for (var i = 0; i < 3; i++) + { + var t = now.AddMinutes(-15 * i); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO deadlocks (deadlock_id, collection_time, server_id, server_name, deadlock_time, victim_process_id, victim_sql_text, deadlock_graph_xml) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, t, "process" + i, "DELETE FROM Posts", + i == 0 ? null : "DELETE FROM Posts"); + } + JsonAssert.Contains("\"truncated\": true", await DarlingMcpBlockingTools.GetDeadlocks(postgres, ServerName, 24, 2)); + var deadlocks = await DarlingMcpBlockingTools.GetDeadlocks(postgres, ServerName, 24, 3); + JsonAssert.Contains("\"deadlocks_returned\": 3", deadlocks); + JsonAssert.Contains("\"truncated\": false", deadlocks); + var detail = await DarlingMcpBlockingTools.GetDeadlockDetail(postgres, ServerName, 24, 2); + JsonAssert.Contains("\"deadlocks_returned\": 2", detail); + JsonAssert.Contains("\"truncated\": false", detail); + + /* Alerts: 3 live + 2 dismissed. The default read hides two and says so; lifting the filter shows five. */ + for (var i = 0; i < 5; i++) + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO config_alert_log (alert_time, server_id, server_name, metric_name, current_value, threshold_value, alert_sent, notification_type, dismissed) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + now.AddMinutes(-5 * i), ServerId, ServerName, "High CPU", 92.5, 80.0, true, "email", i >= 3); + var alerts = await DarlingMcpAlertTools.GetAlertHistory(postgres, ServerName, 24, 3); + JsonAssert.Contains("\"alerts_returned\": 3", alerts); + JsonAssert.Contains("\"truncated\": false", alerts); + JsonAssert.Contains("\"dismissed_excluded\": true", alerts); + JsonAssert.Contains("\"dismissed_excluded_count\": 2", alerts); + JsonAssert.Contains("\"truncated\": true", await DarlingMcpAlertTools.GetAlertHistory(postgres, ServerName, 24, 2)); + var lifted = await DarlingMcpAlertTools.GetAlertHistory(postgres, ServerName, 24, 50, include_dismissed: true); + JsonAssert.Contains("\"alerts_returned\": 5", lifted); + JsonAssert.Contains("\"dismissed_excluded\": false", lifted); + JsonAssert.Contains("\"dismissed\": true", lifted); + + /* Long queries: the slowest run is the OLDEST; a limit = 1 page must still hold it. */ + for (var i = 0; i < 4; i++) + { + var t = now.AddMinutes(-20 * i); + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO long_query_completions (long_query_completion_id, collection_time, server_id, server_name, event_time, event_type, database_name, duration_microseconds, statement_text) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, t, "rpc_completed", Db, (long)(i + 1) * 1_000_000, "EXEC p" + i); + } + var slowest = await DarlingMcpLongQueryTools.GetLongQueryCompletions(postgres, ServerName, 24, 1); + JsonAssert.Contains("\"duration_ms\": 4000", slowest); + JsonAssert.Contains("\"truncated\": true", slowest); + JsonAssert.Contains("\"order\": \"duration_ms_desc\"", slowest); + JsonAssert.Contains("\"truncated\": false", await DarlingMcpLongQueryTools.GetLongQueryCompletions(postgres, ServerName, 24, 4)); + + /* Plan corrections: 3 re-captures. */ + for (var i = 0; i < 3; i++) + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO plan_correction (collection_id, collection_time, server_id, server_name, database_name, recommendation_name, recommendation_state, score) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", CollectionIdGenerator.Next(), now.AddMinutes(-5 * i), ServerId, ServerName, Db, "PR_1", "Active", 50); + JsonAssert.Contains("\"truncated\": true", await DarlingMcpPlanCorrectionTools.GetPlanCorrections(postgres, ServerName, 24, 2)); + var corrections = await DarlingMcpPlanCorrectionTools.GetPlanCorrections(postgres, ServerName, 24, 3); + JsonAssert.Contains("\"recommendations_returned\": 3", corrections); + JsonAssert.Contains("\"truncated\": false", corrections); + + /* Waiting tasks: 3 rows; the envelope now carries the window. */ + for (var i = 0; i < 3; i++) + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO waiting_tasks (collection_id, collection_time, server_id, server_name, session_id, wait_type, wait_duration_ms, blocking_session_id, resource_description, database_name) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)", CollectionIdGenerator.Next(), now.AddMinutes(-i), ServerId, ServerName, 55 + i, "LCK_M_X", 3000L, 60, null, Db); + var tasks = await DarlingMcpSessionTools.GetWaitingTasks(postgres, ServerName, 1, 2); + JsonAssert.Contains("\"hours_back\": 1", tasks); + JsonAssert.Contains("\"tasks_returned\": 2", tasks); + JsonAssert.Contains("\"truncated\": true", tasks); + JsonAssert.Contains("\"truncated\": false", await DarlingMcpSessionTools.GetWaitingTasks(postgres, ServerName, 1, 3)); + + /* Wait stats: 3 types; the cap binds to limit past the old 50 too, but 3 vs 2 is the boundary. */ + var i2 = 0; + foreach (var w in new[] { "CXPACKET", "PAGEIOLATCH_SH", "LCK_M_X" }) + await DarlingMcpTestData.ExecAsync(connection, ct, + @"INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8)", CollectionIdGenerator.Next(), now, ServerId, ServerName, w, 10L, 1000L * (3 - i2++), 100L); + var waits = await DarlingMcpDataTools.GetWaitStats(postgres, ServerName, 24, 2); + JsonAssert.Contains("\"wait_types_returned\": 2", waits); + JsonAssert.Contains("\"truncated\": true", waits); + JsonAssert.Contains("\"truncated\": false", await DarlingMcpDataTools.GetWaitStats(postgres, ServerName, 24, 3)); + + /* A window whose every alert was dismissed names the filter rather than calling itself quiet. */ + await DarlingMcpTestData.ExecAsync(connection, ct, $"UPDATE config_alert_log SET dismissed = TRUE WHERE server_id = {ServerId}"); + var allDismissed = await DarlingMcpAlertTools.GetAlertHistory(postgres, ServerName, 24, 50); + Assert.Equal("empty", DarlingMcpTestData.StatusOf(allDismissed)); + Assert.Contains("5 dismissed alert(s) were excluded", allDismissed, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// #3541 A7 against live PostgreSQL: the SQL that produces each window total is the thing the in-process + /// tests cannot see, so it is executed here through the real tool methods. Three series per table at + /// 600 / 300 / 100 — the whole window is 1,000 — read at limit = 1: the one row's share must be + /// 60, the published total must be the seeded window's 1,000, and truncated must be true; + /// at limit = 3 the shares must be 60 / 30 / 10, the page sum must equal the total, and + /// truncated must be false. The pair is what separates the fix from the defect: a page that IS the + /// window sums to 100 under either arithmetic. + /// + /// The cumulative tables (pg_wait_sampling, pg_kernel_stats, pg_io_stats) and the + /// block/WAL side of pg_statement_stats are seeded with TWO snapshots, because their reads difference + /// newest against oldest and a single sample has no interval. The kernel series are seeded so the HOTTEST + /// query has the HIGHEST query_id: under the read's old ORDER BY 3 + 4 — a constant, not two + /// ordinals — the page came back in query_id order and a limit = 1 page held the coolest + /// query; the assertion that the 60% row leads is what pins the ranking. + /// + [Fact] + public async Task PercentTools_ShareTheWindowNotThePage_AgainstDevPostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live percent-denominator test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var t1 = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow).AddMinutes(-2); + var t0 = t1.AddMinutes(-10); + long[] series = [600, 300, 100]; + /* Spread over the int8 range and NOT in share order: the hottest series has the middle id, the + coolest the smallest, so any read that falls back to id order shows it. */ + long[] queryIds = [42L, 7_000_000_000_000_000_001L, -4_185_925_123_159_566_327L]; + + for (var i = 0; i < 3; i++) + { + /* Statements: the rate columns are stored deltas, so only the second snapshot carries them. */ + foreach (var (t, delta) in new[] { (t0, 0L), (t1, series[i]) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_statement_stats + (collection_id, collection_time, server_id, server_name, queryid, database_id, user_id, toplevel, + calls, total_exec_time_ms, max_exec_time_ms, rows_returned, shared_blks_hit, shared_blks_read, storage_blks_read, orcache_blks_hit, + temp_blks_read, temp_blks_written, wal_bytes, max_exec_peakmem_bytes, delta_calls, delta_total_exec_time_ms, delta_rows) +VALUES ($1, $2, $3, $4, $5, 16384, 10, TRUE, 100, 5000, 91.5, 250, 10, 5, 3, 2, 0, 0, 1000, 2097152, $6, $7, $8)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, queryIds[i], delta > 0 ? 10L : 0L, delta, delta > 0 ? 100L : 0L); + } + + /* Aurora waits: stored deltas, one snapshot is enough. Microseconds in the store. */ + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_wait_stats (collection_id, collection_time, server_id, server_name, wait_type_id, wait_event_id, wait_type, wait_event, waits, wait_time_us, delta_waits, delta_wait_time_us) +VALUES ($1, $2, $3, $4, $5, $6, 'IO', $7, 100, 9999999, 10, $8)", + CollectionIdGenerator.Next(), t1, ServerId, ServerName, i + 1, (long)(i + 100), "DataFileRead" + i, series[i] * 1000); + + /* Sampled waits: cumulative, oldest 100 -> newest 100 + share. */ + foreach (var (t, count) in new[] { (t0, 100L), (t1, 100L + series[i]) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_wait_sampling (collection_id, collection_time, server_id, server_name, event_type, event, query_id, sample_count, profile_period_ms, backend_count) +VALUES ($1, $2, $3, $4, 'IO', 'DataFileRead', $5, $6, 10, 1)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, queryIds[i], count); + } + + /* Kernel: cumulative user + system, oldest (100, 0) -> newest (100 + half the share, half the + share). Halves, because every seeded share is even and the arithmetic stays exact in a double. */ + foreach (var (t, user, system) in new[] { (t0, 100.0, 0.0), (t1, 100.0 + series[i] / 2.0, series[i] / 2.0) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_kernel_stats (collection_id, collection_time, server_id, server_name, database_name, query_id, exec_user_time_ms, exec_system_time_ms, plan_cpu_time_ms, exec_read_bytes, exec_write_bytes, minor_faults, major_faults, stats_since) +VALUES ($1, $2, $3, $4, 'app', $5, $6, $7, 0, 8192, 0, 0, 0, $8)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, queryIds[i], user, system, t0.AddDays(-1)); + } + + /* I/O: cumulative reads and read time, three contexts of one backend type. */ + var context = i == 0 ? "normal" : i == 1 ? "vacuum" : "bulkread"; + foreach (var (t, reads, readTime) in new[] { (t0, 1000L, 100.0), (t1, 1000L + series[i], 100.0 + series[i] / 10.0) }) + { + await DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_io_stats (collection_id, collection_time, server_id, server_name, backend_type, object_type, context, reads, read_time_ms, writes, write_time_ms, writebacks, writeback_time_ms, extends, extend_time_ms, op_bytes, hits, evictions, reuses, fsyncs, fsync_time_ms, stats_reset, read_bytes, write_bytes, extend_bytes) +VALUES ($1, $2, $3, $4, 'client backend', 'relation', $5, $6, $7, 5, 1, 0, 0, 1, 0, 8192, 100, 0, 0, 0, 0, NULL, NULL, NULL, NULL)", + CollectionIdGenerator.Next(), t, ServerId, ServerName, context, reads, readTime); + } + } + + /* One assertion set, five tools. The hot row's id is asserted on the three per-query reads so a + read that still ranks by id rather than by weight fails on the ROW, not only on the share. */ + var hot = queryIds[0].ToString(CultureInfo.InvariantCulture); + + AssertPercentPair( + await DarlingMcpPgStatementTools.GetPgTopQueries(postgres, ServerName, 4, 1), + await DarlingMcpPgStatementTools.GetPgTopQueries(postgres, ServerName, 4, 3), + "queries_returned", "total_exec_time_ms", "returned_exec_time_ms", "returned_pct_of_total", "queries", "pct_of_total_time", hot); + + AssertPercentPair( + await DarlingMcpPgWaitTools.GetPgWaitStats(postgres, ServerName, 4, 1), + await DarlingMcpPgWaitTools.GetPgWaitStats(postgres, ServerName, 4, 3), + "wait_events_returned", "total_wait_time_ms", "returned_wait_time_ms", "returned_pct_of_total", "waits", "pct_of_total_wait", hotQueryId: null); + + AssertPercentPair( + await DarlingMcpPgWaitSamplingTools.GetPgWaitSampling(postgres, ServerName, 4, 1), + await DarlingMcpPgWaitSamplingTools.GetPgWaitSampling(postgres, ServerName, 4, 3), + "waits_returned", "total_samples", "returned_samples", "returned_pct_of_total", "waits", "pct_of_samples", hot); + + AssertPercentPair( + await DarlingMcpPgKernelStatsTools.GetPgKernelStats(postgres, ServerName, 4, 1), + await DarlingMcpPgKernelStatsTools.GetPgKernelStats(postgres, ServerName, 4, 3), + "queries_returned", "total_cpu_ms", "returned_cpu_ms", "returned_pct_of_total", "queries", "pct_of_total_cpu", hot); + + AssertPercentPair( + await DarlingMcpPgIoTools.GetPgIoStats(postgres, ServerName, 4, 1), + await DarlingMcpPgIoTools.GetPgIoStats(postgres, ServerName, 4, 3), + "combination_count", "total_reads", "returned_reads", "returned_pct_of_total_reads", "combinations", "pct_of_total_reads", hotQueryId: null); + + /* The second I/O denominator: read time, inferred as tracked because non-zero times are seeded + and no configuration row says otherwise. 100 ms across the window, 60 on the one-row page. */ + var ioCut = JsonDocument.Parse(await DarlingMcpPgIoTools.GetPgIoStats(postgres, ServerName, 4, 1)).RootElement; + Assert.True(ioCut.GetProperty("io_timing_tracked").GetBoolean()); + Assert.Equal(100.0, ioCut.GetProperty("total_read_time_ms").GetDouble()); + Assert.Equal(60.0, ioCut.GetProperty("returned_read_time_ms").GetDouble()); + Assert.Equal(60.0, ioCut.GetProperty("combinations")[0].GetProperty("pct_of_total_read_time").GetDouble()); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => + await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /// + /// The cut page and the whole page over one seeded window. Cut: one row, share 60, total 1,000, page sum + /// 600, truncated. Whole: three rows at 60 / 30 / 10, page sum equal to the total, not truncated. And on + /// the per-query reads, the cut page's row IS the hottest series by id. + /// + private static void AssertPercentPair( + string cutJson, string wholeJson, + string returnedCountKey, string totalKey, string returnedKey, string ratioKey, string rowsKey, string shareKey, + string? hotQueryId) + { + var cut = JsonDocument.Parse(cutJson).RootElement; + /* Data-bearing means the ROWS are there, not that `status` is absent: get_pg_io_stats carries + status = "io_activity" on its data payload, so a classifier keyed on the presence of `status` + called a correct 60% page a status envelope (first CI run of this test). */ + Assert.True(cut.TryGetProperty(rowsKey, out _), $"expected a data-bearing payload with `{rowsKey}`, got: " + cutJson); + Assert.Equal(1, cut.GetProperty(returnedCountKey).GetInt32()); + Assert.True(cut.GetProperty("truncated").GetBoolean()); + Assert.Equal(1000.0, cut.GetProperty(totalKey).GetDouble()); + Assert.Equal(600.0, cut.GetProperty(returnedKey).GetDouble()); + Assert.Equal(60.0, cut.GetProperty(ratioKey).GetDouble()); + var cutRow = Assert.Single(cut.GetProperty(rowsKey).EnumerateArray()); + Assert.Equal(60.0, cutRow.GetProperty(shareKey).GetDouble()); + if (hotQueryId is not null) + { + Assert.Equal(hotQueryId, cutRow.GetProperty("queryid").GetString()); + } + + var whole = JsonDocument.Parse(wholeJson).RootElement; + Assert.Equal(3, whole.GetProperty(returnedCountKey).GetInt32()); + Assert.False(whole.GetProperty("truncated").GetBoolean()); + Assert.Equal(1000.0, whole.GetProperty(totalKey).GetDouble()); + Assert.Equal(1000.0, whole.GetProperty(returnedKey).GetDouble()); + Assert.Equal(100.0, whole.GetProperty(ratioKey).GetDouble()); + Assert.Equal(new[] { 60.0, 30.0, 10.0 }, + whole.GetProperty(rowsKey).EnumerateArray().Select(r => r.GetProperty(shareKey).GetDouble()).ToArray()); + } + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) + { + var sql = string.Join(" ", Tables.Select(tbl => $"DELETE FROM {tbl} WHERE server_id = {ServerId};")) + + $" DELETE FROM servers WHERE server_id = {ServerId};"; + using var cleanup = new NpgsqlCommand(sql, connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/McpZeroIsAMeasurementTests.cs b/Darling/Darling.Tests/McpZeroIsAMeasurementTests.cs new file mode 100644 index 000000000..1071c1725 --- /dev/null +++ b/Darling/Darling.Tests/McpZeroIsAMeasurementTests.cs @@ -0,0 +1,767 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// #3541 A12 — contract rule 5, "zero is a measurement": a zero the product publishes must have been measured, +/// and an absence must be null / unknown WITH its reason. Six sites published a 0, an empty, or a +/// nominal-window label where nothing had been measured: +/// +/// +/// eight of the nine get_health_parser_* tools answered a dead system_health session (or a +/// collector that never ran) with the same empty a healthy quiet hour earns; +/// get_query_store_regressions coerced a NULL percent (baseline side 0 → no denominator) to 0, +/// publishing the largest possible I/O regression as "no change"; +/// the LAG-differenced duration trends rated the window's first collection 0.0 — a fabricated quiet +/// instant — on both SKUs and on the Query Store rollup route (#3540 A8's "first point of every differenced +/// series"); +/// get_pg_xmin_horizon divided a holder's wins by its OWN rows, so 2 wins in 2 holder-bearing +/// collections out of 288 captures read as 100% chronic; +/// get_pvs_stats published a measured 0 MB and an unmeasured NULL identically (both +/// pct_of_database: null); +/// get_table_index_sizes folded a missing 30-day baseline onto the 7-day one (and that onto the +/// oldest, and that onto current → growth 0) and labelled the result with the window asked for. +/// +/// +/// This file is the census: the discriminators are witnessed against the defect shapes as literals (a +/// matcher that quietly stopped matching reports a clean bill), the fixed shapes are pinned on BOTH SKUs from +/// source (Lite's assembly is not referenced here, the arrangement), the +/// SQL consts are pinned for the new columns, and the pure derivations (growth, PVS reasons) are executed. +/// runs the tools against live Postgres. +/// +public sealed class McpZeroIsAMeasurementTests +{ + private const string DarlingMcp = "Darling/PerformanceMonitor.Darling.Service/Mcp"; + private const string LiteMcp = "Lite/Mcp"; + + private static readonly string[] HealthParserTools = + [ + "get_health_parser_cpu_tasks", + "get_health_parser_io_issues", + "get_health_parser_memory_broker", + "get_health_parser_memory_conditions", + "get_health_parser_memory_node_oom", + "get_health_parser_scheduler_issues", + "get_health_parser_severe_errors", + "get_health_parser_significant_waits", + "get_health_parser_system_health", + ]; + + /* ───────────────────────── 1. the health-parser source witness ───────────────────────── */ + + /// + /// Every one of the nine, on both SKUs, publishes the witness on its data envelope and routes its + /// zero-row case through the shared four-rung ladder — a tool that kept a private + /// Status("empty", …) would be the defect returning under one name. + /// + [Theory] + [InlineData(DarlingMcp + "/DarlingMcpHealthParserTools.cs")] + [InlineData(LiteMcp + "/McpHealthParserTools.cs")] + public void EveryHealthParserTool_PublishesTheSourceWitness_AndClimbsTheSharedLadder(string file) + { + var source = ReadRepoFile(file.Split('/')); + + foreach (var tool in HealthParserTools) + { + var body = Strip(ToolBody(source, tool)); + Assert.Contains("source_observed = true", body, StringComparison.Ordinal); + Assert.Contains("last_captured_at = Stamp(", body, StringComparison.Ordinal); + Assert.Contains("EmptyAsync(", body, StringComparison.Ordinal); + Assert.DoesNotContain("McpHelpers.Status(\"empty\"", body, StringComparison.Ordinal); + + var description = DescriptionOf(source, tool); + Assert.Contains("source_observed", description, StringComparison.Ordinal); + Assert.Contains("last_captured_at", description, StringComparison.Ordinal); + } + } + + /// + /// The ladder itself: four rungs, three of them empty and exactly one unavailable — the + /// nothing-of-any-type-ever rung, with source_observed: false and the sentence the + /// EngineCapabilityMissTests pin (system_health session is started). Rung 3 — this type never, the + /// session alive — must be empty: a memory-node OOM that never happened is the healthy measurement. + /// + [Theory] + [InlineData(DarlingMcp + "/DarlingMcpHealthParserTools.cs")] + [InlineData(LiteMcp + "/McpHealthParserTools.cs")] + public void TheEmptyLadder_HasFourRungs_AndOnlyTheDeadSessionIsUnavailable(string file) + { + var source = ReadRepoFile(file.Split('/')); + var start = source.IndexOf("private static async Task EmptyAsync", StringComparison.Ordinal); + Assert.True(start > 0, $"{file} has no shared EmptyAsync ladder"); + var end = source.IndexOf("private static string WitnessStatus", start, StringComparison.Ordinal); + var ladder = Strip(source[start..end]); + + Assert.Equal(3, Regex.Matches(ladder, "WitnessStatus\\(\\s*\"empty\"").Count); + Assert.Single(Regex.Matches(ladder, "WitnessStatus\\(\\s*\"unavailable\"")); + Assert.Single(Regex.Matches(ladder, "sourceObserved: false")); + Assert.Contains("NOT an all-clear", ladder, StringComparison.Ordinal); + Assert.Contains("system_health session is started", ladder, StringComparison.Ordinal); + /* The engine-capability probe goes FIRST on the dead rung — the stronger claim. */ + Assert.True( + ladder.IndexOf("NotCollectedStatusAsync", StringComparison.Ordinal) < ladder.IndexOf("\"unavailable\"", StringComparison.Ordinal)); + /* The healthy rungs never reach for the dead rung's word, so a caller keying on it cannot be misled. */ + var rung3 = ladder[..ladder.IndexOf("NotCollectedStatusAsync", StringComparison.Ordinal)]; + Assert.DoesNotContain("EVER", rung3, StringComparison.Ordinal); + } + + /// + /// The witness reads the SAME view the tools read and is not the collection log — the log records a + /// SUCCESS for a run that read a dead session and stored nothing, which is the shape being fixed. + /// + [Fact] + public void TheWitness_IsTheEventsView_NotTheCollectionLog_OnBothSkus() + { + Assert.Contains("FROM v_system_health_events", DarlingSystemHealthReader.LastCaptureSql, StringComparison.Ordinal); + Assert.DoesNotContain("collection_log", DarlingSystemHealthReader.LastCaptureSql, StringComparison.Ordinal); + + var lite = ReadRepoFile("Lite", "Services", "LocalDataService.SystemEvents.cs"); + var probe = lite[lite.IndexOf("GetLastSystemHealthCaptureAsync", StringComparison.Ordinal)..]; + probe = probe[..probe.IndexOf("GetLastSystemHealthCaptureOfTypeAsync", StringComparison.Ordinal)]; + Assert.Contains("SELECT MAX(collection_time)", probe, StringComparison.Ordinal); + Assert.Contains("FROM v_system_health_events", probe, StringComparison.Ordinal); + Assert.DoesNotContain("collection_log", probe, StringComparison.Ordinal); + } + + /* ───────────────────────── 2. NULL percents stay NULL ───────────────────────── */ + + private static readonly Regex NullPercentCoerced = new(@"RegressionPercent = reader\.IsDBNull\(\d+\) \? 0 ", RegexOptions.Compiled); + + [Fact] + public void TheRegressionReaders_NeverCoerceANullPercentToZero_OnEitherSku() + { + /* Witness: the defect as it shipped on Lite. */ + Assert.Matches(NullPercentCoerced, "DurationRegressionPercent = reader.IsDBNull(4) ? 0 : ToDouble(reader.GetValue(4)),"); + + var lite = ReadRepoFile("Lite", "Services", "LocalDataService.QueryStoreRegressions.cs"); + Assert.DoesNotMatch(NullPercentCoerced, lite); + Assert.Equal(3, Regex.Matches(lite, @"RegressionPercent = reader\.IsDBNull\(\d+\) \? null ").Count); + Assert.Equal(3, Regex.Matches(lite, @"public double\? \w+RegressionPercent").Count); + + /* Darling's reader is positional; the three percent columns are 4, 7 and 10. */ + var darling = ReadRepoFile(DarlingMcp.Split('/').Append("DarlingQueryStoreRegressionReader.cs").ToArray()); + foreach (var column in new[] { 4, 7, 10 }) + { + Assert.Contains($"reader.IsDBNull({column}) ? null : Convert.ToDouble(reader.GetValue({column}))", darling, StringComparison.Ordinal); + } + + var percents = typeof(DarlingQueryStoreRegressionReader.RegressionRow).GetProperties() + .Where(p => p.Name.EndsWith("RegressionPercent", StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(3, percents.Length); + Assert.All(percents, p => Assert.Equal(typeof(double?), p.PropertyType)); + } + + /// Both tools publish the reason beside the null, and a null duration ratio nulls the + /// severity banded from it rather than letting the TVF's ELSE 'LOW' stand. + [Theory] + [InlineData(DarlingMcp + "/DarlingMcpQueryStoreRegressionTools.cs")] + [InlineData(LiteMcp + "/McpQueryTools.cs")] + public void TheRegressionTool_SaysWhyAPercentIsNull_AndDoesNotBandAMissingRatio(string file) + { + var body = Strip(ToolBody(ReadRepoFile(file.Split('/')), "get_query_store_regressions")); + Assert.Contains("undefined_percents = UndefinedPercentNotes(r)", body, StringComparison.Ordinal); + Assert.Contains("severity = r.DurationRegressionPercent is null ? null : r.Severity", body, StringComparison.Ordinal); + + var description = DescriptionOf(ReadRepoFile(file.Split('/')), "get_query_store_regressions"); + Assert.Contains("undefined_percents", description, StringComparison.Ordinal); + Assert.Contains("null percent never sorts as 0", description, StringComparison.Ordinal); + } + + /* ───────────────────────── 3. differenced trends: the first point is unrated ───────────────────────── */ + + private static readonly Regex FabricatedFirstPoint = new(@"ELSE 0 END AS \w+_per_second", RegexOptions.Compiled); + + private static IEnumerable<(string Name, string Sql)> DifferencedTrendSql() + { + yield return (nameof(DarlingTrendReader.QueryDurationTrendSql), DarlingTrendReader.QueryDurationTrendSql); + yield return (nameof(DarlingTrendReader.ProcedureDurationTrendSql), DarlingTrendReader.ProcedureDurationTrendSql); + yield return (nameof(DarlingTrendReader.QueryStoreDurationTrendSql), DarlingTrendReader.QueryStoreDurationTrendSql); + yield return ("BuildRollupTrendSql(false)", QueryStoreTrendRouting.BuildRollupTrendSql(withDatabaseFilter: false)); + yield return ("BuildRollupTrendSql(true)", QueryStoreTrendRouting.BuildRollupTrendSql(withDatabaseFilter: true)); + } + + [Fact] + public void EveryDifferencedTrend_LeavesTheFirstPointUnrated_NeverZero() + { + /* Witness: the shipped shape. */ + Assert.Matches(FabricatedFirstPoint, "CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds ELSE 0 END AS elapsed_ms_per_second,"); + + foreach (var (name, sql) in DifferencedTrendSql()) + { + Assert.Contains("LAG(", sql, StringComparison.Ordinal); + Assert.DoesNotMatch(FabricatedFirstPoint, sql); + /* Every rate column is a CASE with no ELSE — NULL where the denominator does not exist. */ + Assert.True( + Regex.Matches(sql, @"CASE WHEN interval_seconds > 0 THEN [^\n]*? END AS \w+_per_second").Count >= 2, + $"{name} no longer rates through a no-ELSE CASE"); + /* And the row is KEPT, not filtered: a lone collection is "no rate yet", not an empty window. */ + Assert.DoesNotContain("WHERE interval_seconds > 0", sql, StringComparison.Ordinal); + } + + /* Lite's four differenced trends, read from source: every `AS interval_seconds` statement rates + through a no-ELSE CASE and none carries the fabricated 0. */ + foreach (var file in new[] { "LocalDataService.QueryStats.cs", "LocalDataService.QueryStore.cs" }) + { + var lite = ReadRepoFile("Lite", "Services", file); + Assert.DoesNotMatch(FabricatedFirstPoint, lite); + var lagged = Regex.Matches(lite, @"\)\)\) AS interval_seconds").Count; + var rated = Regex.Matches(lite, @"CASE WHEN interval_seconds > 0 THEN [^\n]*? END AS \w+_per_second").Count; + Assert.True(lagged >= 1, $"{file}: the LAG idiom is gone, so this pin is looking at nothing"); + Assert.True(rated >= lagged, $"{file}: {lagged} differenced statement(s) but only {rated} no-ELSE rate column(s)"); + } + + /* The readers carry the null through instead of re-fabricating it. */ + var point = typeof(DarlingTrendReader.QueryDurationTrendPoint); + Assert.Equal(typeof(double?), point.GetProperty("Value")!.PropertyType); + Assert.Equal(typeof(long?), point.GetProperty("ExecutionCount")!.PropertyType); + Assert.Equal(typeof(double?), point.GetProperty("ExecutionsPerSecond")!.PropertyType); + var litePoint = ReadRepoFile("Lite", "Services", "LocalDataService.QueryStats.cs"); + Assert.Contains("public double? Value { get; set; }", litePoint, StringComparison.Ordinal); + Assert.Contains("public long? ExecutionCount { get; set; }", litePoint, StringComparison.Ordinal); + Assert.Contains("public double? ExecutionsPerSecond { get; set; }", litePoint, StringComparison.Ordinal); + } + + /// The three trend tools on both SKUs count and explain their unrated points with one sentence. + [Theory] + [InlineData(DarlingMcp + "/DarlingMcpTrendTools.cs")] + [InlineData(LiteMcp + "/McpQueryTools.cs")] + public void TheTrendTools_CountAndExplainUnratedPoints(string file) + { + var source = Strip(ReadRepoFile(file.Split('/'))); + Assert.Contains("envelope[\"unrated_points\"] = unrated;", source, StringComparison.Ordinal); + Assert.Contains("Unknowable is not 0", source, StringComparison.Ordinal); + foreach (var tool in new[] { "get_query_duration_trend", "get_procedure_duration_trend", "get_query_store_duration_trend" }) + { + Assert.Contains("unrated_points", DescriptionOf(ReadRepoFile(file.Split('/')), tool), StringComparison.Ordinal); + } + } + + /// + /// The Darling viewer shares the rollup builder, so its chart reader must tolerate the NULL first bucket — + /// by skipping it, since a chart has nowhere to draw "unknown" and coercing it to 0 would be the defect + /// plotted. (The viewer's OWN raw-route SQL copies still carry ELSE 0; that is the viewer lane's + /// A2 residual, out of this change's boundary, and not what this pin asserts.) + /// + [Fact] + public void TheViewerRollupReader_SkipsTheUnratedBucket_RatherThanPlottingZero() + { + var viewer = ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.QueryTrends.cs"); + var start = viewer.IndexOf("QueryStoreDurationTrendRollupSql);", StringComparison.Ordinal); + var rollupReader = viewer[start..viewer.IndexOf("return items;", start, StringComparison.Ordinal)]; + Assert.Contains("if (reader.IsDBNull(1))", rollupReader, StringComparison.Ordinal); + Assert.Contains("continue;", rollupReader, StringComparison.Ordinal); + Assert.DoesNotContain("IsDBNull(1) ? 0", rollupReader, StringComparison.Ordinal); + + /* Lite's charts do the same with the nullable point. */ + var charts = ReadRepoFile("Lite", "Controls", "ServerTab.Charts.cs"); + Assert.Equal(4, Regex.Matches(charts, @"var rated = data\.Where\(d => d\.HasRate\)\.ToList\(\);").Count); + } + + /* ───────────────────────── 4. xmin: the window's denominator ───────────────────────── */ + + /// + /// The MCP share and the alert evaluator's horizon arm fraction over the SAME denominator: the + /// collector's own SUCCESS rows in collection_log, every time it looked, held or not. + /// + [Fact] + public void TheXminShare_DividesByEveryCapture_AndAgreesWithTheAlertEvaluator() + { + foreach (var sql in new[] { DarlingPgXminReader.XminCapturesInWindowSql, DarlingPostgresAlertReadAdapter.XminSql }) + { + Assert.Contains("FROM collection_log", sql, StringComparison.Ordinal); + Assert.Contains("collector_name = 'pg_xmin_horizon'", sql, StringComparison.Ordinal); + Assert.Contains("status = 'SUCCESS'", sql, StringComparison.Ordinal); + } + + /* And it is not the holder table: that table has no rows for an unheld capture. */ + var captures = DarlingPgXminReader.XminCapturesInWindowSql; + Assert.DoesNotContain("pg_xmin_horizon\n", captures, StringComparison.Ordinal); + Assert.Contains("COUNT(*) AS captures_in_window", captures, StringComparison.Ordinal); + + var body = Strip(ToolBody(ReadRepoFile(DarlingMcp.Split('/').Append("DarlingMcpPgXminTools.cs").ToArray()), "get_pg_xmin_horizon")); + Assert.Contains("/ capturesInWindow * 100", body, StringComparison.Ordinal); + Assert.DoesNotContain("/ r.Samples", body, StringComparison.Ordinal); + Assert.Contains("captures_in_window = capturesInWindow", body, StringComparison.Ordinal); + /* No captures and no holders is not an all-clear. */ + Assert.Contains("if (capturesInWindow == 0)", body, StringComparison.Ordinal); + Assert.Contains("status = \"unavailable\"", body, StringComparison.Ordinal); + /* Null, not 0, when there is nothing to divide by. */ + Assert.Contains(": (double?)null", body, StringComparison.Ordinal); + } + + /* ───────────────────────── 5. PVS: a measured zero is a measurement ───────────────────────── */ + + [Theory] + [InlineData(DarlingMcp + "/DarlingMcpPvsTools.cs")] + [InlineData(LiteMcp + "/McpPvsTools.cs")] + public void ThePvsShare_DividesAnyMeasuredSize_AndSaysWhyWhenItCannot(string file) + { + var body = Strip(ToolBody(ReadRepoFile(file.Split('/')), "get_pvs_stats")); + /* The defect: only a POSITIVE size divided, so 0 MB fell through to the same null as unmeasured. */ + Assert.DoesNotContain("PvsSizeMb is > 0 &&", body, StringComparison.Ordinal); + Assert.Contains("r.PvsSizeMb is { } pvsMb && r.DatabaseDataSizeMb is > 0", body, StringComparison.Ordinal); + Assert.Contains("pvs_measured = r.PvsSizeMb.HasValue", body, StringComparison.Ordinal); + Assert.Contains("pct_of_database_reason = PctReason(", body, StringComparison.Ordinal); + } + + [Fact] + public void ThePvsReason_IsNullOnlyWhenTheShareIsDefined_IncludingADefinedZero() + { + Assert.Null(DarlingMcpPvsTools.PctReason(pvsMeasured: true, databaseDataSizeMb: 1280)); + Assert.Contains("unknown — not zero", DarlingMcpPvsTools.PctReason(pvsMeasured: false, databaseDataSizeMb: 1280), StringComparison.Ordinal); + Assert.Contains("no denominator", DarlingMcpPvsTools.PctReason(pvsMeasured: true, databaseDataSizeMb: null), StringComparison.Ordinal); + Assert.Contains("no denominator", DarlingMcpPvsTools.PctReason(pvsMeasured: true, databaseDataSizeMb: 0), StringComparison.Ordinal); + } + + /* ───────────────────────── 6. growth: only over history the store holds ───────────────────────── */ + + private static readonly Regex FoldedBaseline = new(@"COALESCE\(p30\.reserved_mb, p7\.reserved_mb|COALESCE\(p7\.reserved_mb, o\.reserved_mb", RegexOptions.Compiled); + + [Fact] + public void TheGrowthRead_NeverFoldsAMissingBaselineOntoANearerOne_OnEitherSku() + { + /* Witness: the shipped chain. */ + Assert.Matches(FoldedBaseline, "l.current_reserved_mb - COALESCE(p30.reserved_mb, p7.reserved_mb, o.reserved_mb, l.current_reserved_mb) AS growth_30d_mb,"); + + var darling = DarlingObjectStatsReader.ObjectSizeGrowthSql; + Assert.DoesNotMatch(FoldedBaseline, darling); + Assert.Contains("MAX(collection_time) FILTER (WHERE collection_time <= $2) AS snapshot_7d_time", darling, StringComparison.Ordinal); + Assert.Contains("MAX(collection_time) FILTER (WHERE collection_time <= $3) AS snapshot_30d_time", darling, StringComparison.Ordinal); + foreach (var column in new[] { "reserved_mb_7d_ago", "reserved_mb_30d_ago", "reserved_mb_oldest", "b.snapshot_7d_time", "b.snapshot_30d_time", "b.earliest_time", "b.latest_time", "b.days_of_data" }) + { + Assert.Contains(column, darling, StringComparison.Ordinal); + } + Assert.DoesNotContain("growth_7d_mb", darling, StringComparison.Ordinal); + + var lite = ReadRepoFile("Lite", "Services", "LocalDataService.FinOps.IndexObjects.cs"); + var read = lite[lite.IndexOf("GetObjectSizeGrowthAsync(int serverId", StringComparison.Ordinal)..]; + read = read[..read.IndexOf("return items;", StringComparison.Ordinal)]; + Assert.DoesNotMatch(FoldedBaseline, read); + Assert.Contains("MAX(collection_time) FILTER (WHERE collection_time <= $2) AS snapshot_7d_time", read, StringComparison.Ordinal); + Assert.Contains("ReservedMb30dAgo = reader.IsDBNull(8) ? null", read, StringComparison.Ordinal); + + foreach (var file in new[] { DarlingMcp + "/DarlingMcpObjectStatsTools.cs", LiteMcp + "/McpObjectStatsTools.cs" }) + { + var body = Strip(ToolBody(ReadRepoFile(file.Split('/')), "get_table_index_sizes")); + foreach (var key in new[] { "history_days_available", "covers_7d", "covers_30d", "growth_over_available_history_mb", "growth_over_available_history_pct", "growth_window_days", "growth_note = GrowthNote(r)", "tables_returned = page.Count", "truncated," }) + { + Assert.Contains(key, body, StringComparison.Ordinal); + } + /* Truncation observed by over-fetch, never inferred from a full page (A3's rule). */ + Assert.Contains("TableSizesTop + 1", body, StringComparison.Ordinal); + Assert.Contains("var truncated = rows.Count > TableSizesTop;", body, StringComparison.Ordinal); + } + } + + /// The derivations, executed: each figure comes from exactly the baseline it names or is null. + [Fact] + public void TheGrowthRow_RefusesEveryFigureItsBaselineCannotSupport() + { + var earliest = new DateTime(2026, 3, 1, 3, 0, 0); + var latest = new DateTime(2026, 3, 4, 3, 0, 0); + + /* Three days of history: no 7-day or 30-day snapshot; the oldest holds the table at 100 MB. */ + var threeDays = new DarlingObjectStatsReader.ObjectSizeGrowthRow( + "db", "dbo", "Posts", CurrentReservedMb: 160, CurrentUsedMb: 150, TotalRows: 10, IndexCount: 2, + ReservedMb7dAgo: null, ReservedMb30dAgo: null, ReservedMbOldest: 100, + Snapshot7dTime: null, Snapshot30dTime: null, EarliestSnapshotTime: earliest, LatestSnapshotTime: latest, DaysOfData: 3); + Assert.Null(threeDays.Growth7dMb); + Assert.Null(threeDays.Growth30dMb); + Assert.Null(threeDays.GrowthPct30d); + Assert.Equal(60, threeDays.GrowthOverAvailableHistoryMb); + Assert.Equal(60.0, threeDays.GrowthOverAvailableHistoryPct!.Value, 9); + Assert.Equal(20.0, threeDays.DailyGrowthRateMb!.Value, 9); + var note = DarlingMcpObjectStatsTools.GrowthNote(threeDays)!; + Assert.Contains("no snapshot 7+ days old exists", note, StringComparison.Ordinal); + Assert.Contains("no snapshot 30+ days old exists", note, StringComparison.Ordinal); + + /* The shipped chain would have said growth_30d = 60 (labelled 30d, measured over 3). Now the 30-day + figure is null and the 3-day figure carries its own name and span. */ + + /* A table created since the earliest snapshot: nothing but growth, and the shipped chain said 0. */ + var newTable = threeDays with { ReservedMbOldest = null }; + Assert.Null(newTable.GrowthOverAvailableHistoryMb); + Assert.Null(newTable.DailyGrowthRateMb); + Assert.Contains("not in the earliest snapshot", DarlingMcpObjectStatsTools.GrowthNote(newTable)!, StringComparison.Ordinal); + + /* A single day of snapshots: no span, so no growth is knowable — every figure null. */ + var oneDay = threeDays with { DaysOfData = 0, EarliestSnapshotTime = latest }; + Assert.Null(oneDay.GrowthOverAvailableHistoryMb); + Assert.Null(oneDay.DailyGrowthRateMb); + Assert.Contains("no growth is knowable yet", DarlingMcpObjectStatsTools.GrowthNote(oneDay)!, StringComparison.Ordinal); + + /* Full history, every baseline present: every figure defined and NO note. */ + var full = threeDays with + { + ReservedMb7dAgo = 140, ReservedMb30dAgo = 80, ReservedMbOldest = 50, + Snapshot7dTime = latest.AddDays(-7), Snapshot30dTime = latest.AddDays(-30), EarliestSnapshotTime = latest.AddDays(-40), DaysOfData = 40, + }; + Assert.Equal(20, full.Growth7dMb); + Assert.Equal(80, full.Growth30dMb); + Assert.Equal(100.0, full.GrowthPct30d!.Value, 9); + Assert.Equal(110, full.GrowthOverAvailableHistoryMb); + Assert.Null(DarlingMcpObjectStatsTools.GrowthNote(full)); + + /* A 0 baseline has no ratio — the absolute stands, the percent is null with its reason. */ + var wasEmpty = full with { ReservedMb30dAgo = 0 }; + Assert.Equal(160, wasEmpty.Growth30dMb); + Assert.Null(wasEmpty.GrowthPct30d); + Assert.Contains("no denominator", DarlingMcpObjectStatsTools.GrowthNote(wasEmpty)!, StringComparison.Ordinal); + } + + /* ───────────────────────── helpers ───────────────────────── */ + + private static string ToolBody(string source, string toolName) + { + var marker = $"[McpServerTool(Name = \"{toolName}\")"; + var start = source.IndexOf(marker, StringComparison.Ordinal); + Assert.True(start >= 0, $"no tool named {toolName} in the source"); + var next = source.IndexOf("[McpServerTool(", start + marker.Length, StringComparison.Ordinal); + return next < 0 ? source[start..] : source[start..next]; + } + + /// The Description literal of one tool, whichever side of a line break it sits on (Lite's + /// get_pvs_stats opens its string on the next line). + private static string DescriptionOf(string source, string toolName) + { + var body = ToolBody(source, toolName); + var match = Regex.Match(body, @"Description\(\s*""((?:[^""\\]|\\.)*)""", RegexOptions.Singleline); + Assert.True(match.Success, $"{toolName} has no Description literal"); + return match.Groups[1].Value; + } + + private static string Strip(string source) => + Regex.Replace(Regex.Replace(source, @"/\*.*?\*/", string.Empty, RegexOptions.Singleline), @"//[^\n]*", string.Empty); +} + +/// +/// Gated (DARLING_TEST_PG) live round-trips for #3541 A12, through the real tool methods: the Query Store +/// regression whose baseline read nothing, the xmin holder that won 2 of 6 captures, the PVS row measured at +/// 0 MB beside one not measured at all, and the growth read over three days of history. +/// +[Collection("live-postgres")] +public sealed class McpZeroIsAMeasurementLivePostgresTests +{ + private const string ServerName = "zero-is-a-measurement-e2e"; + private static readonly int ServerId = ServerIdHelper.GetDeterministicHashCode(ServerName); + private const string Db = "ZeroDb"; + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + [Fact] + public async Task ARegressionWithNoBaselineReads_StaysNull_AndSaysWhy() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live #3541 A12 regression test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + + /* Query 1: baseline read NOTHING (0 logical reads), recent reads 50,000 — the largest possible + I/O regression, and the row the shipped reader published as io_regression_percent 0. CPU + regressed 100% so it passes the gate. Query 2: every side defined. */ + await SeedQueryStoreAsync(connection, ct, HoursAgo(30), queryId: 1, intervalId: 1, executions: 10, avgDurationUs: 1_000, avgCpuUs: 500, avgReads: 0); + await SeedQueryStoreAsync(connection, ct, MinutesAgo(30), queryId: 1, intervalId: 2, executions: 10, avgDurationUs: 2_000, avgCpuUs: 1_000, avgReads: 50_000); + await SeedQueryStoreAsync(connection, ct, HoursAgo(30), queryId: 2, intervalId: 3, executions: 10, avgDurationUs: 1_000, avgCpuUs: 500, avgReads: 100); + await SeedQueryStoreAsync(connection, ct, MinutesAgo(30), queryId: 2, intervalId: 4, executions: 10, avgDurationUs: 3_000, avgCpuUs: 1_000, avgReads: 200); + + var root = JsonDocument.Parse(await DarlingMcpQueryStoreRegressionTools.GetQueryStoreRegressions(postgres, ServerName, hours_back: 24)).RootElement; + Assert.Equal(2, root.GetProperty("regression_count").GetInt32()); + + var rows = root.GetProperty("regressions").EnumerateArray().ToDictionary(r => r.GetProperty("query_id").GetInt64()); + + var noBaselineReads = rows[1]; + Assert.Equal(JsonValueKind.Null, noBaselineReads.GetProperty("io_regression_percent").ValueKind); + Assert.Equal(0, noBaselineReads.GetProperty("baseline_reads").GetDouble()); + Assert.Equal(50_000, noBaselineReads.GetProperty("recent_reads").GetDouble()); + var notes = noBaselineReads.GetProperty("undefined_percents").EnumerateArray().Select(n => n.GetString()!).ToArray(); + Assert.Single(notes); + Assert.Contains("io_regression_percent is null: no_baseline", notes[0], StringComparison.Ordinal); + Assert.Contains("NOT 0% change", notes[0], StringComparison.Ordinal); + /* The other two ratios exist and are unaffected; severity is banded from the duration ratio. */ + Assert.Equal(100.0, noBaselineReads.GetProperty("duration_regression_percent").GetDouble(), 6); + Assert.Equal(100.0, noBaselineReads.GetProperty("cpu_regression_percent").GetDouble(), 6); + Assert.Equal("HIGH", noBaselineReads.GetProperty("severity").GetString()); /* 100% duration: > 50, not > 100 */ + + var defined = rows[2]; + Assert.Equal(100.0, defined.GetProperty("io_regression_percent").GetDouble(), 6); + Assert.Equal(JsonValueKind.Null, defined.GetProperty("undefined_percents").ValueKind); + + /* The ranking key is the absolute delta, defined for both — query 2's 20 ms × 10 outranks + query 1's 10 ms × 10, whatever their ratios. */ + var order = root.GetProperty("regressions").EnumerateArray().Select(r => r.GetProperty("query_id").GetInt64()).ToArray(); + Assert.Equal(new long[] { 2, 1 }, order); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task TheXminShare_IsOverEveryCapture_AndAgreesWithTheAlertAdapter() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live #3541 A12 xmin test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await DarlingMcpTestData.ExecAsync(connection, ct, "UPDATE servers SET engine_kind = $2 WHERE server_id = $1", ServerId, MonitoredEngineKind.AuroraPostgres); + + /* No holders and no captures: the collector did not look, so this is NOT "nothing holds the + horizon". */ + var blind = JsonDocument.Parse(await DarlingMcpPgXminTools.GetPgXminHorizon(postgres, ServerName, hours_back: 4)).RootElement; + Assert.Equal("unavailable", blind.GetProperty("status").GetString()); + Assert.Equal(0, blind.GetProperty("captures_in_window").GetInt32()); + + /* Six successful captures, one failed one (does not count), two of which recorded pid 104 as the + winner. The shipped payload divided 2 by the source's OWN 2 rows: 100%, "chronic". */ + for (var i = 1; i <= 6; i++) + await SeedLogAsync(connection, ct, MinutesAgo(i * 10), "SUCCESS"); + await SeedLogAsync(connection, ct, MinutesAgo(70), "ERROR"); + + /* Captures only, no holder: the healthy answer, and it names the denominator it rests on. */ + var clear = JsonDocument.Parse(await DarlingMcpPgXminTools.GetPgXminHorizon(postgres, ServerName, hours_back: 4)).RootElement; + Assert.Equal("no_holder", clear.GetProperty("status").GetString()); + Assert.Equal(6, clear.GetProperty("captures_in_window").GetInt32()); + Assert.Contains("captured 6 time(s)", clear.GetProperty("finding").GetString()!, StringComparison.Ordinal); + + await SeedHolderAsync(connection, ct, MinutesAgo(20), "session", 80_000_000, "104", "state=idle in transaction", isWinner: true); + await SeedHolderAsync(connection, ct, MinutesAgo(10), "session", 81_000_000, "104", "state=idle in transaction", isWinner: true); + + var held = JsonDocument.Parse(await DarlingMcpPgXminTools.GetPgXminHorizon(postgres, ServerName, hours_back: 4)).RootElement; + Assert.Equal("holder_present", held.GetProperty("status").GetString()); + Assert.Equal(6, held.GetProperty("captures_in_window").GetInt32()); + var session = held.GetProperty("holders").EnumerateArray().Single(h => h.GetProperty("source").GetString() == "session"); + Assert.Equal(2, session.GetProperty("samples_as_winner").GetInt32()); + Assert.Equal(2, session.GetProperty("captures_recording_this_source").GetInt32()); + Assert.Equal(33.3, session.GetProperty("pct_of_window_winning").GetDouble(), 1); + + /* The alert evaluator's horizon arm counts the same six captures — one denominator, two surfaces. */ + var adapter = new DarlingPostgresAlertReadAdapter(postgres); + var info = await adapter.GetXminHorizonAsync(ServerId, ct); + Assert.NotNull(info); + Assert.Equal(6, info!.CapturesInWindow); + Assert.Equal(6L, await DarlingPgXminReader.GetXminCapturesInWindowAsync(postgres, ServerId, HoursAgo(4), DateTime.UtcNow, ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task AMeasuredZeroPvs_IsAMeasurement_AndAnUnmeasuredOneSaysSo() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live #3541 A12 PVS test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var t = MinutesAgo(5); + await SeedPvsAsync(connection, ct, t, "CleanDb", pvsSizeMb: 0m, dataSizeMb: 1280m); + await SeedPvsAsync(connection, ct, t, "UnreadDb", pvsSizeMb: null, dataSizeMb: 1280m); + await SeedPvsAsync(connection, ct, t, "BusyDb", pvsSizeMb: 912.82m, dataSizeMb: 1280m); + + var root = JsonDocument.Parse(await DarlingMcpPvsTools.GetPvsStats(postgres, ServerName)).RootElement; + var byDb = root.GetProperty("databases").EnumerateArray().ToDictionary(d => d.GetProperty("database_name").GetString()!); + + /* 0 MB of 1,280 MB is 0.00% — measured, and said so. Before: pct_of_database null, same as unread. */ + var clean = byDb["CleanDb"]; + Assert.True(clean.GetProperty("pvs_measured").GetBoolean()); + Assert.Equal(0, clean.GetProperty("pvs_size_mb").GetDouble()); + Assert.Equal(0.0, clean.GetProperty("pct_of_database").GetDouble()); + Assert.Equal(JsonValueKind.Null, clean.GetProperty("pct_of_database_reason").ValueKind); + + var unread = byDb["UnreadDb"]; + Assert.False(unread.GetProperty("pvs_measured").GetBoolean()); + Assert.Equal(JsonValueKind.Null, unread.GetProperty("pvs_size_mb").ValueKind); + Assert.Equal(JsonValueKind.Null, unread.GetProperty("pct_of_database").ValueKind); + Assert.Contains("not zero", unread.GetProperty("pct_of_database_reason").GetString()!, StringComparison.Ordinal); + + var busy = byDb["BusyDb"]; + Assert.True(busy.GetProperty("pvs_measured").GetBoolean()); + Assert.Equal(71.31, busy.GetProperty("pct_of_database").GetDouble(), 2); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + [Fact] + public async Task GrowthOverThreeDaysOfHistory_IsLabelledAsThreeDays_NotThirty() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to a Postgres connection string to run the live #3541 A12 growth test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + var now = MinutesAgo(5); + var threeDaysAgo = now.AddDays(-3); + + /* Posts: 100 MB three days ago, 160 MB now. Comments: created since — only in the latest snapshot. */ + await SeedIndexAsync(connection, ct, threeDaysAgo, "Posts", reservedMb: 100m); + await SeedIndexAsync(connection, ct, now, "Posts", reservedMb: 160m); + await SeedIndexAsync(connection, ct, now, "Comments", reservedMb: 40m); + + var root = JsonDocument.Parse(await DarlingMcpObjectStatsTools.GetTableIndexSizes(postgres, ServerName)).RootElement; + + var history = root.GetProperty("history"); + Assert.Equal(3, history.GetProperty("history_days_available").GetInt32()); + Assert.False(history.GetProperty("covers_7d").GetBoolean()); + Assert.False(history.GetProperty("covers_30d").GetBoolean()); + Assert.Contains("3 day(s) of index snapshots", history.GetProperty("note").GetString()!, StringComparison.Ordinal); + Assert.Equal(2, root.GetProperty("tables_returned").GetInt32()); + Assert.False(root.GetProperty("truncated").GetBoolean()); + + var byTable = root.GetProperty("tables").EnumerateArray().ToDictionary(t => t.GetProperty("table_name").GetString()!); + + /* The shipped read said growth_30d_mb = 60 for Posts — measured over three days, labelled thirty. */ + var posts = byTable["Posts"]; + Assert.Equal(JsonValueKind.Null, posts.GetProperty("growth_7d_mb").ValueKind); + Assert.Equal(JsonValueKind.Null, posts.GetProperty("growth_30d_mb").ValueKind); + Assert.Equal(JsonValueKind.Null, posts.GetProperty("growth_pct_30d").ValueKind); + Assert.Equal(60, posts.GetProperty("growth_over_available_history_mb").GetDouble()); + Assert.Equal(60.0, posts.GetProperty("growth_over_available_history_pct").GetDouble(), 6); + Assert.Equal(3, posts.GetProperty("growth_window_days").GetInt32()); + Assert.Equal(20.0, posts.GetProperty("daily_growth_rate_mb").GetDouble(), 6); + Assert.Contains("no snapshot 30+ days old exists", posts.GetProperty("growth_note").GetString()!, StringComparison.Ordinal); + + /* And the shipped read said Comments grew 0 MB — for a table that is nothing but growth. */ + var comments = byTable["Comments"]; + Assert.Equal(JsonValueKind.Null, comments.GetProperty("growth_over_available_history_mb").ValueKind); + Assert.Equal(JsonValueKind.Null, comments.GetProperty("daily_growth_rate_mb").ValueKind); + Assert.Contains("not in the earliest snapshot", comments.GetProperty("growth_note").GetString()!, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) => await DeleteRowsAsync(cleanup, cleanupCt)); + } + } + + /* ───────────────────────── seeds ───────────────────────── */ + + private static DateTime MinutesAgo(int minutes) => DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); + private static DateTime HoursAgo(int hours) => DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddHours(-hours)); + + private static Task SeedQueryStoreAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTime, long queryId, long intervalId, + long executions, long avgDurationUs, long avgCpuUs, long avgReads) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO query_store_stats + (collection_id, collection_time, server_id, server_name, database_name, query_id, plan_id, + execution_type_desc, execution_count, avg_duration_us, avg_cpu_time_us, avg_logical_io_reads, + runtime_stats_interval_id, query_text, last_execution_time) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)", + CollectionIdGenerator.Next(), collectionTime, ServerId, ServerName, Db, queryId, 9L, "Regular", + executions, avgDurationUs, avgCpuUs, avgReads, intervalId, $"SELECT {queryId}", collectionTime); + + private static Task SeedHolderAsync( + NpgsqlConnection connection, CancellationToken ct, DateTime collectionTime, string source, long xminAge, string holder, string? detail, bool isWinner) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_xmin_horizon + (collection_id, collection_time, server_id, server_name, source, xmin_age, holder, detail, is_winner) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + CollectionIdGenerator.Next(), collectionTime, ServerId, ServerName, source, xminAge, holder, detail, isWinner); + + private static Task SeedLogAsync(NpgsqlConnection connection, CancellationToken ct, DateTime collectionTime, string status) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO collection_log + (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, error_message, rows_collected, sql_duration_ms, duckdb_duration_ms) +VALUES ($1, $2, $3, 'pg_xmin_horizon', $4, 50, $5, NULL, 0, 40, 10)", + CollectionIdGenerator.Next(), ServerId, ServerName, collectionTime, status); + + private static Task SeedPvsAsync(NpgsqlConnection connection, CancellationToken ct, DateTime collectionTime, string database, decimal? pvsSizeMb, decimal dataSizeMb) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO collect.pvs_stats + (collection_id, collection_time, server_id, server_name, database_name, database_id, is_accelerated_database_recovery_on, + persistent_version_store_size_mb, online_index_version_store_size_mb, database_data_size_mb, current_aborted_transaction_count) +VALUES ($1, $2, $3, $4, $5, $6, TRUE, $7, 0, $8, 0)", + CollectionIdGenerator.Next(), collectionTime, ServerId, ServerName, database, 7, pvsSizeMb, dataSizeMb); + + private static Task SeedIndexAsync(NpgsqlConnection connection, CancellationToken ct, DateTime collectionTime, string table, decimal reservedMb) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO index_object_stats + (collection_id, collection_time, server_id, server_name, database_name, schema_name, object_id, table_name, index_id, index_name, index_type_desc, reserved_mb, used_mb, total_rows) +VALUES ($1, $2, $3, $4, $5, 'dbo', $6, $7, 1, $8, 'CLUSTERED', $9, $9, 1000)", + CollectionIdGenerator.Next(), collectionTime, ServerId, ServerName, Db, table.GetHashCode(StringComparison.Ordinal), table, "PK_" + table, reservedMb); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + using var cleanup = new NpgsqlCommand( + $"DELETE FROM query_store_stats WHERE server_id = {ServerId}; DELETE FROM pg_xmin_horizon WHERE server_id = {ServerId}; " + + $"DELETE FROM collection_log WHERE server_id = {ServerId}; DELETE FROM collect.pvs_stats WHERE server_id = {ServerId}; " + + $"DELETE FROM index_object_stats WHERE server_id = {ServerId}; DELETE FROM servers WHERE server_id = {ServerId};", connection); + await cleanup.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs b/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs index bbf7c745d..dbd7dafd9 100644 --- a/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs +++ b/Darling/Darling.Tests/NpgsqlRootCertificateValidationTests.cs @@ -39,11 +39,12 @@ public async Task ChainShape_VerifyFullWithPrintedRoot_CompletesTheHandshake_OnE { var generated = StoreTlsCertificates.Create("localhost", IPAddress.Loopback, validityYears: 2); - var completed = await HandshakeCompletesAsync(generated.ServerCertChainPem, generated.ServerKeyPem, generated.RootCertPem); + var (completed, serverFailure) = await HandshakeCompletesAsync(generated.ServerCertChainPem, generated.ServerKeyPem, generated.RootCertPem); Assert.True(completed, "VerifyFull with the printed root must survive Npgsql's certificate validation on this platform — " + - "this is the exact remote-viewer path #2117 exists to fix."); + "this is the exact remote-viewer path #2117 exists to fix." + + (serverFailure is null ? string.Empty : $" Server side of the harness failed: {serverFailure}")); } [Fact] @@ -69,17 +70,22 @@ public async Task LegacySelfSignedShape_VerifyFullWithItselfAsRoot_TheFieldConfi using var legacy = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddYears(2)); var pem = legacy.ExportCertificatePem(); - var completed = await HandshakeCompletesAsync(pem, rsa.ExportPkcs8PrivateKeyPem(), pem); + var (completed, serverFailure) = await HandshakeCompletesAsync(pem, rsa.ExportPkcs8PrivateKeyPem(), pem); /* Recorded, not required: the CHAIN shape's test above is the guarantee. The dynamic skip puts the platform fact in every CI log without inventing a requirement that the legacy shape fail — the first cut asserted that and Windows CI refuted it. */ - Assert.Skip($"legacy self-signed shape at VerifyFull: handshake completed = {completed} on {Environment.OSVersion.Platform}"); + Assert.Skip($"legacy self-signed shape at VerifyFull: handshake completed = {completed} on {Environment.OSVersion.Platform}" + + (serverFailure is null ? string.Empty : $"; server-side failure: {serverFailure.GetType().Name}: {serverFailure.Message}")); } - /// Runs the fake server + a VerifyFull Npgsql connect; true when the server-side TLS - /// handshake completed (the client accepted the certificate). - private static async Task HandshakeCompletesAsync(string serverCertChainPem, string serverKeyPem, string rootPem) + /// Runs the fake server + a VerifyFull Npgsql connect. Completed is true when the + /// server-side TLS handshake completed (the client accepted the certificate); ServerFailure + /// is whatever the server task threw, because "completed = false" alone cannot distinguish an + /// Npgsql rejection from the server never reaching TLS at all — #3557 was exactly that, a + /// chain-build failure swallowed here and misread as a + /// certificate-validation verdict for days. + private static async Task<(bool Completed, Exception? ServerFailure)> HandshakeCompletesAsync(string serverCertChainPem, string serverKeyPem, string rootPem) { var rootPath = Path.Combine(Path.GetTempPath(), $"darling-test-root-{Guid.NewGuid():N}.crt"); await File.WriteAllTextAsync(rootPath, rootPem); @@ -168,9 +174,10 @@ depends on a complete read. */ is handshakeCompleted, not the exception. */ } - try { await serverTask; } catch { /* aborted handshakes land here; the flag says enough */ } + Exception? serverFailure = null; + try { await serverTask; } catch (Exception ex) { serverFailure = ex; /* client-rejection aborts AND pre-TLS failures both land here — return it so the caller can tell them apart */ } try { File.Delete(rootPath); } catch { /* temp file, best-effort */ } - return handshakeCompleted; + return (handshakeCompleted, serverFailure); } } diff --git a/Darling/Darling.Tests/OversizedPlanBacklogPins.cs b/Darling/Darling.Tests/OversizedPlanBacklogPins.cs index d0bd089bf..80ec51f7e 100644 --- a/Darling/Darling.Tests/OversizedPlanBacklogPins.cs +++ b/Darling/Darling.Tests/OversizedPlanBacklogPins.cs @@ -204,28 +204,36 @@ from the constants the backlog's own reads filter on rather than as fresh litera } [Fact] - public void BothCollectors_DeclareTheSizeAsTheirLastPayloadColumn() + public void BothCollectors_DeclareTheSizeAtTheOrdinalItWasAppendedAt() { - /* Appended LAST on both, which is what keeps every earlier ordinal — and therefore every existing - store column's position, the positional binary COPY and the positional DuckDB appender — stable. - BigInt because DATALENGTH over an nvarchar(max) expression returns bigint, and a narrower store - column would silently overflow on the megabyte-scale plans this exists to describe. + /* Appended LAST on both when #3392 landed, which is what keeps every earlier ordinal — and therefore + every existing store column's position, the positional binary COPY and the positional DuckDB + appender — stable. BigInt because DATALENGTH over an nvarchar(max) expression returns bigint, and + a narrower store column would silently overflow on the megabyte-scale plans this exists to + describe. + + No longer the LAST column: V128 / Lite v61 (#3540) appended behind it — the interval on + procedure_stats, the two statement offsets on query_stats — by the same append-only rule. The + claim that outlives that is the one the stores depend on: this column's ORDINAL never moved. + Pinned as the ordinal (51 and 35), which is what "appended last at #3392" means once later rungs + exist; a `names[^1]` pin here would assert #3392 is still the newest appender, which is how the + next rung's build goes red. This is the declaration half. The SELECT-ordinal-to-payload-slot agreement is driven through the real shredder in Lite.Tests' two collector-definition suites, which own the reader fakes. */ - Assert.Equal(52, QueryStatsCollector.Instance.PayloadColumns.Count); - Assert.Equal(36, ProcedureStatsCollector.Instance.PayloadColumns.Count); + Assert.Equal(54, QueryStatsCollector.Instance.PayloadColumns.Count); + Assert.Equal(37, ProcedureStatsCollector.Instance.PayloadColumns.Count); - foreach (ICollectorSchemaInfo collector in new ICollectorSchemaInfo[] + foreach (var (collector, ordinal) in new (ICollectorSchemaInfo, int)[] { - QueryStatsCollector.Instance, - ProcedureStatsCollector.Instance, + (QueryStatsCollector.Instance, 51), + (ProcedureStatsCollector.Instance, 35), }) { var names = collector.PayloadColumns.Select(c => c.Name).ToArray(); - Assert.Equal("query_plan_xml_bytes", names[^1]); - Assert.Equal(CollectorColumnType.BigInt, collector.PayloadColumns[^1].Type); + Assert.Equal("query_plan_xml_bytes", names[ordinal]); + Assert.Equal(CollectorColumnType.BigInt, collector.PayloadColumns[ordinal].Type); /* The gated content column is still there and still AHEAD of the size, which is the pair a reader tests as "measured, not captured". query_stats keeps other columns between them, so @@ -895,19 +903,23 @@ comparison alone passes when the primary arm is gone entirely. */ } [Fact] - public void TheQueryStatsFallback_KeysOnTheHash_BecauseTheFactRowCarriesNoOffsets() + public void TheQueryStatsFallback_KeysOnTheHash_TheGrainItsReadersAskAt() { - /* This is the REASON, pinned. query_stats reads the statement offsets for its delta key and never - stores them, so a join from a stored fact row could only match plan_handle + sql_handle — which - for a multi-statement plan is several backlog rows describing DIFFERENT statements' plans. Serving - one of those as "the plan for this query" is worse than serving nothing. If the offsets ever DO - become stored columns, this pin fails and the fallback can become an exact join. */ + /* This was "…BecauseTheFactRowCarriesNoOffsets" and pinned the absence of the two offset columns, + with the note that the day they became stored columns the pin would fail and the fallback could + become an exact join. V128 (#3540) stored them, the pin failed as designed, and the decision is + recorded here rather than taken silently: the fallback STAYS keyed on query_hash. Two reasons, + both in the SQL's own doc — the readers ask at the query_hash grain, which the backlog row serves + directly; and every row written before V128 carries NULL offsets, so an exact join would go dark + on an upgraded store for a raw retention's worth of history. The offsets are asserted PRESENT and + trailing so this record cannot drift back into the old claim. */ var stored = QueryStatsCollector.Instance.PayloadColumns.Select(c => c.Name).ToArray(); - Assert.DoesNotContain("statement_start_offset", stored); - Assert.DoesNotContain("statement_end_offset", stored); + Assert.Equal("statement_start_offset", stored[^2]); + Assert.Equal("statement_end_offset", stored[^1]); Assert.Contains("query_hash = $2", OversizedPlanBacklog.QueryStatsFallbackSql, StringComparison.Ordinal); Assert.DoesNotContain("plan_handle", OversizedPlanBacklog.QueryStatsFallbackSql, StringComparison.Ordinal); + Assert.DoesNotContain("statement_start_offset", OversizedPlanBacklog.QueryStatsFallbackSql, StringComparison.Ordinal); /* procedure_stats CAN join exactly, and does: its three DMVs expose no offsets at all, so the plan apply passes fixed literals and every backlog row for it carries that same pair. */ diff --git a/Darling/Darling.Tests/PayloadDimensionTests.cs b/Darling/Darling.Tests/PayloadDimensionTests.cs index 4186d0fe7..48a19ecad 100644 --- a/Darling/Darling.Tests/PayloadDimensionTests.cs +++ b/Darling/Darling.Tests/PayloadDimensionTests.cs @@ -239,11 +239,13 @@ public void CopyCommandFor_ProcedureStats_DivertsOnlyThePlanColumn() [Fact] public void CopyCommandFor_NonDivertingCollector_IsUnchangedByTheDimensions() { - /* Every collector that declares no dimension keeps the pre-#1767 command verbatim, byte for byte. */ + /* Every collector that declares no dimension keeps the pre-#1767 command verbatim, byte for byte — + plus the trailing sample_interval_seconds V127 appended (#3540), which lands last because the + COPY column list is the PayloadColumns order and the column was appended there. */ Assert.Equal( "COPY wait_stats (collection_id, collection_time, server_id, server_name, wait_type, " + "waiting_tasks_count, wait_time_ms, signal_wait_time_ms, delta_waiting_tasks, delta_wait_time_ms, " + - "delta_signal_wait_time_ms) FROM STDIN (FORMAT BINARY)", + "delta_signal_wait_time_ms, sample_interval_seconds) FROM STDIN (FORMAT BINARY)", PgCollectorRowWriter.CopyCommandFor(WaitStatsCollector.Instance)); } @@ -765,10 +767,12 @@ must define it as the RESOLVING one. */ exactly this tripwire's regression in review: its first cut was a SELECT * passthrough. The shipped V51 DROPs the view (the new column lands mid-list, which CREATE OR REPLACE refuses) and re-emits the generator's resolving definition. V121 re-defines it again for - query_plan_xml_bytes, by the same DROP-then-re-emit route and for the same reason. - The literal is deliberate: a rung that redefines this view has to change this line, which - is what brings a human to the paragraph above. */ - Assert.Equal(121, definers[^1].Version); + query_plan_xml_bytes, by the same DROP-then-re-emit route and for the same reason. V128 + (#3540) re-defines it a fourth time for statement_start_offset / statement_end_offset — the + delta key's two halves — again DROP-then-re-emit, because the offsets land ahead of the digest + columns. The literal is deliberate: a rung that redefines this view has to change this line, + which is what brings a human to the paragraph above. */ + Assert.Equal(128, definers[^1].Version); Assert.Contains( "COALESCE(f.query_text, qtd.query_text) AS query_text", definers[^1].Sql, diff --git a/Darling/Darling.Tests/Pg18IoBytesTests.cs b/Darling/Darling.Tests/Pg18IoBytesTests.cs index 019212a85..af10652d2 100644 --- a/Darling/Darling.Tests/Pg18IoBytesTests.cs +++ b/Darling/Darling.Tests/Pg18IoBytesTests.cs @@ -131,7 +131,9 @@ common indentation and the runtime string is not indented the way the source fil .Count(l => l.Contains(" AS ", StringComparison.Ordinal) || l.TrimEnd(',') is "backend_type" or "object_type" or "context"); - /* 15 before this rung, plus read_bytes, write_bytes, extend_bytes and byte_counters_tracked. */ - Assert.Equal(19, items); + /* 15 before this rung, plus read_bytes, write_bytes, extend_bytes and byte_counters_tracked, plus the + two window totals #3541 A7 appended (window_total_reads, window_total_read_time_ms) - which the + positional reader lifts onto the PAGE at ordinals 19 and 20 rather than onto the row. */ + Assert.Equal(21, items); } } diff --git a/Darling/Darling.Tests/PgAlertCountKnobRungTests.cs b/Darling/Darling.Tests/PgAlertCountKnobRungTests.cs index 2eba2622c..180f52ebb 100644 --- a/Darling/Darling.Tests/PgAlertCountKnobRungTests.cs +++ b/Darling/Darling.Tests/PgAlertCountKnobRungTests.cs @@ -230,17 +230,20 @@ public void TheMcpWriteBound_TheSettingsClamp_AndTheViewerGate_AreTheSameConstan var window = RepoFile.ReadRepoFile( "Darling", "PerformanceMonitor.Darling.Viewer", "SettingsWindow.xaml.cs"); - /* The floor appears TWICE per surface — once per knob. ONE is what a half-migration looks like: - the deadlock knob bounded by the constant and the blocking knob by a literal beside it. The - surface name rides in the failure message so a count mismatch does not send the reader to - three files. */ - foreach (var (what, text) in new[] + /* The floor appears once per knob per surface — FEWER is what a half-migration looks like: one + knob bounded by the constant and its sibling by a literal beside it. #3528 floored the two SQL + Server twins with the same constant, so the engine clamp and the MCP write bound now reach it + four times (pg + SQL, blocking + deadlocks) while the Settings window still gates its SQL boxes + with the numerically-identical `> 0` — the viewer pass is deliberately deferred (backend-first), + and the window's two constant references remain the PG boxes'. The surface name rides in the + failure message so a count mismatch does not send the reader to three files. */ + foreach (var (what, text, perSurface) in new[] { - ("engine clamp", settings), ("mcp write bound", tools), ("settings window gate", window), + ("engine clamp", settings, 4), ("mcp write bound", tools, 4), ("settings window gate", window, 2), }) { Assert.True( - CountOf(text, "PostgresAlertEvaluator.CountThresholdFloor") == 2, + CountOf(text, "PostgresAlertEvaluator.CountThresholdFloor") == perSurface, $"the {what} must reach the shared floor constant once per knob, not a literal"); } @@ -367,8 +370,8 @@ public void TheShippedDefaults_ReproduceTheFireAtOneBehaviour() /// /// The clamp floors a hand-edited store row, so the gate cannot fire on a count of zero — the failure /// the floor exists for, stated on 's own doc. - /// The SQL Server twins take the same write bound and have no read-side floor; that asymmetry is named - /// on DarlingAlertSettings rather than copied here. + /// Since #3528 the SQL Server twins floor at the same constant, so all four count gates are asserted + /// together — the asymmetry this doc used to record is gone. /// [Theory] [InlineData(0)] @@ -378,10 +381,16 @@ public void AHandEditedRowBelowTheFloor_ReadsAsTheFloor_SoNothingFiresOnNothing( var config = new DarlingConfig(); config.Alerts.PgDeadlockCountThreshold = stored; config.Alerts.PgBlockingCountThreshold = stored; + config.Alerts.DeadlockCountThreshold = stored; + config.Alerts.BlockingCountThreshold = stored; var settings = new DarlingAlertSettings(config); Assert.Equal(PostgresAlertEvaluator.CountThresholdFloor, settings.PgDeadlockCountThreshold); Assert.Equal(PostgresAlertEvaluator.CountThresholdFloor, settings.PgBlockingCountThreshold); + /* #3528: the SQL Server twins, floored at the same constant — a store row hand-edited to 0 no + longer makes count >= threshold true on a deadlock-free (or blocking-free) server. */ + Assert.Equal(PostgresAlertEvaluator.CountThresholdFloor, settings.DeadlockCountThreshold); + Assert.Equal(PostgresAlertEvaluator.CountThresholdFloor, settings.BlockingCountThreshold); Assert.False(RollingCountAlertGate.Evaluate( 0, settings.PgDeadlockCountThreshold, watermark: 0, cooldownElapsed: true, suppressed: false).Fire); diff --git a/Darling/Darling.Tests/PgCpuCapacityHeadroomTests.cs b/Darling/Darling.Tests/PgCpuCapacityHeadroomTests.cs index 87f1ec762..769fe18c2 100644 --- a/Darling/Darling.Tests/PgCpuCapacityHeadroomTests.cs +++ b/Darling/Darling.Tests/PgCpuCapacityHeadroomTests.cs @@ -72,6 +72,7 @@ private static FleetServerCard Card( default, default, default, + default, Now.AddSeconds(-30), default, null, @@ -187,6 +188,7 @@ public void TheRingBufferArmBandsOnItsOwnReading(double sqlCpu, double otherCpu, default, default, default, + default, Now.AddSeconds(-30), default, null, @@ -519,7 +521,7 @@ public void BothReasonLinesNameTheCapacityFigure_InTheSameWords() var sqlServer = DarlingFleetReader.BuildCard( new DarlingFleetReader.FleetServerRow(2, "sql-1", "sql-1", null, MonitoredEngineKind.SqlServer, false), new DarlingFleetReader.CpuRow(60.0, 30.0), - default, default, default, default, default, default, + default, default, default, default, default, default, default, Now.AddSeconds(-30), default, null, Now, TimeSpan.FromHours(1), DeadlockRateThresholds.Default); diff --git a/Darling/Darling.Tests/PgFactCollectorTests.cs b/Darling/Darling.Tests/PgFactCollectorTests.cs index 5b5b60c09..4950e4ce3 100644 --- a/Darling/Darling.Tests/PgFactCollectorTests.cs +++ b/Darling/Darling.Tests/PgFactCollectorTests.cs @@ -8,8 +8,10 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using System.Threading.Tasks; using Npgsql; @@ -67,6 +69,7 @@ public sealed class PgFactCollectorTests "CollectMemoryFactsAsync", "CollectMemoryGrantFactsAsync", "CollectMemoryPressureEventFactsAsync", + "CollectObservedCoverageAsync", "CollectParameterSensitivityFactsAsync", "CollectPerfmonFactsAsync", "CollectPlanAdvisoryFactsAsync", @@ -106,10 +109,104 @@ public void ImplementsTheSharedSeam_WithLitesCollectMethodSurface() [Fact] public void AllSql_CoversEveryQuery_OnePerCollectMethodPlusTheDmvFallback() { - /* 31 collect methods, one query each, plus the DMV-snapshot fallback the blocking-chain - method appends through PgBlockingPairRowQuery. */ + /* 32 collect methods (31 fact readers plus the #3538 coverage witness), one query each, plus + the DMV-snapshot fallback the blocking-chain method appends through PgBlockingPairRowQuery. */ Assert.Equal(LiteCollectMethodSurface.Length + 1, PgFactCollector.AllSql.Count); Assert.Contains(PgBlockingPairRowQuery.DmvSnapshotSql, PgFactCollector.AllSql); + Assert.Contains(PgFactCollector.CoverageSql, PgFactCollector.AllSql); + } + + /* ---------------- #3538 A2: the coverage witness ---------------- */ + + /// + /// #3538 A2: the coverage witness reads the SAME series the wait fractions are summed from, finds + /// the first in-window row's predecessor by scanning one gap policy back ($4) and clips its + /// interval to the window, and credits an interval past the policy ($5, bound — never a literal, so + /// it cannot drift from the calculator that discarded the delta) as zero. Pinned on the text + /// because each of those is a behaviour a well-meaning simplification would remove: drop the + /// lookback and every window under-credits one cadence; inline 3600 and the next policy change + /// deflates rates again; drop the policy branch and a three-hour outage credits an hour of + /// observation to a delta the calculator threw away. + /// + [Fact] + public void CoverageSql_ReadsTheWaitSeries_LooksBackOnePolicy_ClipsToTheWindow_AndDiscardsPastThePolicy() + { + var sql = PgFactCollector.CoverageSql; + + Assert.Contains("FROM v_wait_stats", sql, StringComparison.Ordinal); + Assert.Contains("SELECT DISTINCT collection_time", sql, StringComparison.Ordinal); + Assert.Contains("LAG(collection_time) OVER (ORDER BY collection_time)", sql, StringComparison.Ordinal); + + /* The lookback bound and the policy are PARAMETERS. */ + Assert.Contains("collection_time >= $4", sql, StringComparison.Ordinal); + Assert.Contains("> $5 THEN 0", sql, StringComparison.Ordinal); + Assert.DoesNotContain("3600", sql, StringComparison.Ordinal); + + /* Clipped to the window start, so observed time can never exceed the nominal window. */ + Assert.Contains("GREATEST(previous_time, $2)", sql, StringComparison.Ordinal); + + /* The edge columns the C# finishes the lead-in and tail gaps from. */ + Assert.Contains("AS orphan_count", sql, StringComparison.Ordinal); + Assert.Contains("MIN(collection_time) AS first_sample", sql, StringComparison.Ordinal); + Assert.Contains("MAX(collection_time) AS last_sample", sql, StringComparison.Ordinal); + + /* Byte-identical to Lite's inline text — the two collectors are a method-for-method port and + the shared dialect is the whole reason this query could be written once. */ + var lite = File.ReadAllText(Path.Combine(RepoRoot(), "Lite", "Analysis", "DuckDbFactCollector.Waits.cs")); + Assert.Contains(sql.Trim(), lite, StringComparison.Ordinal); + } + + /// + /// The three rate-fact sites divide by the OBSERVED duration and bail on an unobserved window, in + /// both SKUs — a wait fact emitted against the nominal window on either side would silently + /// re-open the defect for that SKU only, and the census above cannot see a divisor. + /// + [Theory] + [InlineData("Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs")] + [InlineData("Lite/Analysis/DuckDbFactCollector.Waits.cs")] + public void RateFacts_DivideByObservedDuration_AndBailWhenUnobserved(string relativePath) + { + var source = File.ReadAllText(Path.Combine(RepoRoot(), relativePath)); + + Assert.Contains("waitTimeMs / context.ObservedDurationMs", source, StringComparison.Ordinal); + Assert.DoesNotContain("waitTimeMs / context.PeriodDurationMs", source, StringComparison.Ordinal); + Assert.Equal(2, CountOf(source, "var observedHours = context.ObservedDurationMs / 3_600_000.0;")); + Assert.Equal(3, CountOf(source, "if (context.ObservedDurationMs <= 0) return;")); + + /* Every use of the nominal window left in the file is a metadata statement of what was asked + for, never a divisor. */ + foreach (var line in source.Split('\n').Where(l => l.Contains("context.PeriodDurationMs", StringComparison.Ordinal))) + { + Assert.True( + line.Contains("[\"period_duration_ms\"]", StringComparison.Ordinal) + || line.Contains("var periodHours = ", StringComparison.Ordinal) + || line.Contains("var nominalMs = ", StringComparison.Ordinal), + $"{relativePath} still uses the nominal window somewhere other than metadata: {line.Trim()}"); + } + } + + private static int CountOf(string haystack, string needle) + { + var count = 0; + for (var i = haystack.IndexOf(needle, StringComparison.Ordinal); i >= 0; i = haystack.IndexOf(needle, i + needle.Length, StringComparison.Ordinal)) + count++; + return count; + } + + /* The house idiom for source-anchored pins (AnalysisPassCommandTimeoutTests et al.): walk up from + THIS file, so the pin reads the tree it was compiled from rather than wherever the binary runs. */ + private static string RepoRoot([CallerFilePath] string thisFile = "") + { + var dir = Path.GetDirectoryName(thisFile)!; + while (dir is not null + && !File.Exists(Path.Combine(dir, "PerformanceMonitor.sln")) + && !Directory.Exists(Path.Combine(dir, ".git"))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.NotNull(dir); + return dir!; } [Fact] @@ -247,21 +344,34 @@ public async Task EndToEnd_CollectFacts_AgainstDevPostgres() /* Plant the two representative collectors' inputs: 1. wait_stats — a wait with delta_wait_time_ms > 0 (the WaitStatsSql emission gate) that is neither LCK_M_* nor CX* so the shared grouping helpers no-op: - 900,000 ms over a 3,600,000 ms window = 0.25 fraction, 3,000 tasks = 300 ms avg. */ - using (var plant = new NpgsqlCommand(@" + 900,000 ms over a 3,600,000 ms window = 0.25 fraction, 3,000 tasks = 300 ms avg. + + Two rows, not one (#3538 A2): a baseline reading at the window start with no + knowable delta, and the delta row at the window end whose change accrued over the + hour between them. The fraction now divides by the OBSERVED collection time — the + interval between consecutive readings — and a lone delta row with nothing before it + observes no time at all (the calculator's first sighting), so the single-row fixture + this used to plant would correctly yield no wait fact. The series covers the window + exactly, so the 0.25 the scenario documents is unchanged. */ + foreach (var (time, deltaTasks, deltaWaitMs, deltaSignalMs) in new[] + { + (windowStart, 0L, 0L, 0L), + (windowEnd, 3000L, 900000L, 100000L) + }) + { + using var plant = new NpgsqlCommand(@" INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", connection)) - { +VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", connection); plant.Parameters.AddWithValue(CollectionIdGenerator.Next()); - plant.Parameters.AddWithValue(sampleTime); + plant.Parameters.AddWithValue(time); plant.Parameters.AddWithValue(TestServerId); plant.Parameters.AddWithValue(TestServerName); plant.Parameters.AddWithValue("SOS_SCHEDULER_YIELD"); - plant.Parameters.AddWithValue(3000L); - plant.Parameters.AddWithValue(900000L); - plant.Parameters.AddWithValue(100000L); + plant.Parameters.AddWithValue(deltaTasks); + plant.Parameters.AddWithValue(deltaWaitMs); + plant.Parameters.AddWithValue(deltaSignalMs); await plant.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); } @@ -305,6 +415,16 @@ INSERT INTO cpu_utilization_stats Assert.Equal(800000, wait.Metadata["resource_wait_time_ms"]); Assert.Equal(300.0, wait.Metadata["avg_ms_per_wait"], precision: 10); + /* #3538 A2: the series covered the window exactly, so coverage is full, the wait fact + carries a coverage_fraction of 1, and no COLLECTION_GAP fact is emitted. */ + Assert.NotNull(context.Coverage); + Assert.Equal(1.0, context.Coverage!.Fraction, precision: 6); + Assert.False(context.Coverage.IsPartial); + Assert.Equal(2, context.Coverage.SampleCount); + Assert.Equal(1.0, wait.Metadata["coverage_fraction"], precision: 6); + Assert.Equal(3_600_000, wait.Metadata["period_duration_ms"]); + Assert.DoesNotContain(facts, f => f.Key == WindowCoverage.FactKey); + /* The CPU fact: Value = average SQL CPU %, and no spurious CPU_SPIKE. */ var cpu = Assert.Single(facts, f => f.Source == "cpu"); Assert.Equal("CPU_SQL_PERCENT", cpu.Key); @@ -331,6 +451,8 @@ from the otherwise-empty store. */ ServerUtcOffset = TimeSpan.Zero }; Assert.Empty(await collector.CollectFactsAsync(emptyContext)); + Assert.NotNull(emptyContext.Coverage); + Assert.False(emptyContext.Coverage!.IsObserved); bodySucceeded = true; } @@ -341,6 +463,126 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup } } + /* ---------------- #3527: perfmon facts are per-second rates ---------------- */ + + /// + /// #3527: delta_cntr_value spans one COLLECTION INTERVAL, not one second — read raw, the + /// PERFMON_*_SEC facts overstate by the cadence (60x at 60s, 300x at 5min). The query must + /// select the row's measured sample_interval_seconds (#2234) for the division and filter + /// interval <= 0 rows (no delta was knowable: first sighting, reset, gap) so rn = 1 lands on + /// the newest row a rate can honestly be derived from. + /// + [Fact] + public void PerfmonSql_SelectsTheMeasuredInterval_AndFiltersUnknowableRows() + { + var sql = PgFactCollector.PerfmonSql; + + Assert.Contains("delta_cntr_value, sample_interval_seconds", sql, StringComparison.Ordinal); + Assert.Contains("sample_interval_seconds > 0", sql, StringComparison.Ordinal); + } + + /// + /// #3527 live fixture: a 'Batch Requests/sec' row with delta 6000 over a measured 60s interval + /// must emit PERFMON_BATCH_REQ_SEC = 100 (not 6000), with the raw delta and the divisor in the + /// metadata. A NEWER interval-0 row (unknowable delta) must be skipped — the fact still comes + /// from the older usable row — and a counter with ONLY interval-0 rows emits no fact at all, + /// never a fact of 0. + /// + [Fact] + public async Task EndToEnd_PerfmonFacts_DivideDeltaByMeasuredInterval_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live perfmon fact test."); + + var ct = TestContext.Current.CancellationToken; + const int perfmonServerId = TestServerId - 2; // own id — this test cleans its own rows + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + await using (var cleanup = new NpgsqlCommand( + $"DELETE FROM perfmon_stats WHERE server_id = {perfmonServerId};", connection)) + { + await cleanup.ExecuteNonQueryAsync(ct); + } + + await using var postgres = NpgsqlDataSource.Create(connectionString!); + var collector = new PgFactCollector(postgres); + + var bodySucceeded = false; + try + { + var windowEnd = TruncateToSeconds(DateTime.UtcNow); + var windowStart = windowEnd.AddHours(-1); + + async Task PlantAsync(long id, DateTime time, string counter, long delta, int intervalSeconds) + { + using var plant = new NpgsqlCommand(@" +INSERT INTO perfmon_stats + (collection_id, collection_time, server_id, server_name, + object_name, counter_name, instance_name, cntr_value, delta_cntr_value, sample_interval_seconds) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", connection); + plant.Parameters.AddWithValue(id); + plant.Parameters.AddWithValue(time); + plant.Parameters.AddWithValue(perfmonServerId); + plant.Parameters.AddWithValue("perfmon-per-second-e2e"); + plant.Parameters.AddWithValue("SQLServer:SQL Statistics"); + plant.Parameters.AddWithValue(counter); + plant.Parameters.AddWithValue(""); + plant.Parameters.AddWithValue(delta * 2); + plant.Parameters.AddWithValue(delta); + plant.Parameters.AddWithValue(intervalSeconds); + await plant.ExecuteNonQueryAsync(ct); + } + + /* Batch requests: an older USABLE row (delta 6000 / 60s = 100/sec), then a NEWER + interval-0 row that must not become the fact. */ + await PlantAsync(1, windowStart.AddMinutes(20), "Batch Requests/sec", 6000, 60); + await PlantAsync(2, windowStart.AddMinutes(25), "Batch Requests/sec", 0, 0); + + /* Compilations: one usable row, 300 / 60s = 5/sec. */ + await PlantAsync(3, windowStart.AddMinutes(20), "SQL Compilations/sec", 300, 60); + + /* Re-compilations: ONLY an interval-0 row — no rate is knowable, so no fact. */ + await PlantAsync(4, windowStart.AddMinutes(20), "SQL Re-Compilations/sec", 0, 0); + + var context = new AnalysisContext + { + ServerId = perfmonServerId, + ServerName = "perfmon-per-second-e2e", + TimeRangeStart = windowStart, + TimeRangeEnd = windowEnd, + ServerUtcOffset = TimeSpan.Zero + }; + + var facts = await collector.CollectFactsAsync(context); + + var batch = Assert.Single(facts, f => f.Key == "PERFMON_BATCH_REQ_SEC"); + Assert.Equal(100.0, batch.Value, precision: 10); + Assert.Equal(6000, batch.Metadata["delta_cntr_value"]); + Assert.Equal(60, batch.Metadata["sample_interval_seconds"]); + + var compilations = Assert.Single(facts, f => f.Key == "PERFMON_COMPILATIONS_SEC"); + Assert.Equal(5.0, compilations.Value, precision: 10); + + Assert.DoesNotContain(facts, f => f.Key == "PERFMON_RECOMPILATIONS_SEC"); + Assert.Equal(2, facts.Count); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + { + using var command = new NpgsqlCommand( + $"DELETE FROM perfmon_stats WHERE server_id = {perfmonServerId};", cleanup); + await command.ExecuteNonQueryAsync(cleanupCt); + }); + } + } + private static DateTime TruncateToSeconds(DateTime value) => DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerSecond)), DateTimeKind.Unspecified); diff --git a/Darling/Darling.Tests/PgInt64IdentityWireShapeTests.cs b/Darling/Darling.Tests/PgInt64IdentityWireShapeTests.cs index c9bfa9b02..53882fceb 100644 --- a/Darling/Darling.Tests/PgInt64IdentityWireShapeTests.cs +++ b/Darling/Darling.Tests/PgInt64IdentityWireShapeTests.cs @@ -108,8 +108,12 @@ private static DarlingPgStatementReader.PgStatementRow StatementRow(long queryId private static JsonElement TopQueries(IEnumerable queryIds, out JsonDocument document) { var rows = queryIds.Select(StatementRow).ToList(); + /* #3541 A7: the projection takes a PAGE - rows plus the window's total - rather than bare rows. The + total here is the rows' own sum, which is the "page is the whole window" case; the share arithmetic + is DarlingMcpPgPercentDenominatorTests' subject, not this file's. */ + var page = new DarlingPgStatementReader.PgTopQueriesPage(rows, rows.Sum(r => r.TotalExecTimeMs)); document = JsonDocument.Parse( - DarlingMcpPgStatementTools.BuildTopQueriesJson("pg-target-01", 24, rows, 20)); + DarlingMcpPgStatementTools.BuildTopQueriesJson("pg-target-01", 24, page, 20)); return document.RootElement.GetProperty("queries"); } diff --git a/Darling/Darling.Tests/PgSchemaGeneratorTests.cs b/Darling/Darling.Tests/PgSchemaGeneratorTests.cs index 32c76b380..ea3501c8a 100644 --- a/Darling/Darling.Tests/PgSchemaGeneratorTests.cs +++ b/Darling/Darling.Tests/PgSchemaGeneratorTests.cs @@ -181,7 +181,8 @@ public void CreateTable_LatchStats_MirrorsDeltaColumns() " max_wait_time_ms bigint,\n" + " delta_waiting_requests_count bigint,\n" + " delta_wait_time_ms bigint,\n" + - " delta_max_wait_time_ms bigint\n" + + " delta_max_wait_time_ms bigint,\n" + + " sample_interval_seconds integer\n" + ");", ddl); } @@ -206,7 +207,8 @@ public void CreateTable_SpinlockStats_MapsSpinsPerCollisionToDoublePrecision() " delta_collisions bigint,\n" + " delta_spins bigint,\n" + " delta_sleep_time bigint,\n" + - " delta_backoffs bigint\n" + + " delta_backoffs bigint,\n" + + " sample_interval_seconds integer\n" + ");", ddl); } diff --git a/Darling/Darling.Tests/PgTableTuningTests.cs b/Darling/Darling.Tests/PgTableTuningTests.cs index 4e0a5d15d..ea9642c01 100644 --- a/Darling/Darling.Tests/PgTableTuningTests.cs +++ b/Darling/Darling.Tests/PgTableTuningTests.cs @@ -17,7 +17,9 @@ namespace Darling.Tests; /// Pins the composer performance-tuning statements (covering indexes + per-table autovacuum-insert override, /// Erik's EXPLAIN-backed field fix) so the tested SQL can never silently drift. These are applied as idempotent /// RUNTIME setup (), NOT a versioned migration, so they do not bump StorageVersion or -/// gate the Viewer — the reason there is no schema-version change to pin here. +/// gate the Viewer — the reason there is no schema-version change to pin here. The #3573 alerting-read covering +/// index rides the same list for the same reason: results-invariant, so the Viewer's connect gate must not +/// learn about it. /// public sealed class PgTableTuningTests { @@ -39,8 +41,16 @@ LATERAL probes (server_id, sql_handle, newest-first — bounded by raw retention Assert.Contains("idx_query_stats_server_hash_time ON collect.query_stats (server_id, query_hash, collection_time DESC)", sql, StringComparison.Ordinal); Assert.Contains("idx_query_store_stats_server_db_query_plan_time ON collect.query_store_stats (server_id, database_name, query_id, plan_id, collection_time DESC)", sql, StringComparison.Ordinal); + /* #3573: the alerting pass's forced-plan-failures read gets the fourth COVERING index — the read's + (server_id, collection_time) predicate as the key and every other column it touches as INCLUDE, so + it runs as an Index Only Scan. The plain (server_id, collection_time) composite V1 already generates + was measured on the production store being priced out by the planner in favour of streaming the + fleet's whole two-hour slice; covering is what removes the heap component the cost model mispriced. + The column list is pinned against the read itself in ForcePlanFailuresAccessPathTests. */ + Assert.Contains("idx_query_store_stats_server_time_forcing ON collect.query_store_stats (server_id, collection_time DESC) INCLUDE (database_name, query_id, plan_id, force_failure_count, is_forced_plan, plan_forcing_type, last_force_failure_reason)", sql, StringComparison.Ordinal); + /* Every index is idempotent (no-op where a field box already hand-applied it, or a prior start made it). */ - Assert.Equal(7, CountOccurrences(sql, "CREATE INDEX IF NOT EXISTS")); /* +1: the #1981 handle index */ + Assert.Equal(8, CountOccurrences(sql, "CREATE INDEX IF NOT EXISTS")); /* +1: the #1981 handle index; +1: the #3573 forced-plan covering index */ Assert.DoesNotContain("CREATE INDEX ON", sql, StringComparison.Ordinal); /* Per-table autovacuum-insert override on exactly the FOUR high-rate insert tables (NOT a global GUC @@ -59,7 +69,7 @@ differ by one word and the wrong one would be silently inert. */ Assert.Contains("ALTER TABLE collect.query_plan_dim SET (autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 10000)", sql, StringComparison.Ordinal); Assert.DoesNotContain("collect.query_plan_dim SET (autovacuum_vacuum_insert_scale_factor", sql, StringComparison.Ordinal); - Assert.Equal(12, PgTableTuning.Statements.Count); /* +1 #1981 query_stats handle index, +1 pg_statement_stats, +1 #2402 query_plan_dim */ + Assert.Equal(13, PgTableTuning.Statements.Count); /* +1 #1981 query_stats handle index, +1 pg_statement_stats, +1 #2402 query_plan_dim, +1 #3573 forced-plan covering index */ } /// diff --git a/Darling/Darling.Tests/PgTrendReaderTests.cs b/Darling/Darling.Tests/PgTrendReaderTests.cs index 7852fc5c2..78461cae0 100644 --- a/Darling/Darling.Tests/PgTrendReaderTests.cs +++ b/Darling/Darling.Tests/PgTrendReaderTests.cs @@ -94,6 +94,36 @@ public void AnIntervalWithNoCallsHasANullMeanRatherThanZero() Assert.Contains("WHEN coalesce(calls, 0) > 0", sql, StringComparison.Ordinal); } + /// + /// #3540 (V128): the same reasoning the trend already applied to the DELTAS now applies to the INTERVAL. + /// pg_statement_stats stores sample_interval_seconds beside its deltas — the span the delta accrued + /// over, which for a collector that skips idle rows is NOT the gap between the rows it left behind. Per + /// snapshot the interval is MAX over the queryid's rows (0 only when every row was the unknowable marker), + /// 0 → NULL through NULLIF, NULL (pre-V128) → the LAG this read always used. No ELSE 0 on + /// calls_per_second: a NULL rate is dropped by the reader rather than plotted as 0.00 calls/sec at a + /// restart, and the first pre-V128 snapshot is absent rather than a fabricated 0.0. + /// + [Fact] + public void TheQueryTrendPrefersTheStoredInterval_AndNeverFabricatesZeroCallsPerSecond() + { + var sql = DarlingPgTrendReader.QueryDurationTrendSql; + + Assert.Contains("CASE WHEN MAX(sample_interval_seconds) IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("THEN extract(epoch FROM (collection_time - LAG(collection_time) OVER (ORDER BY collection_time)))", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(MAX(sample_interval_seconds), 0)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", sql, StringComparison.Ordinal); + Assert.Contains("WHEN interval_seconds > 0", sql, StringComparison.Ordinal); + Assert.Contains("/ interval_seconds", sql, StringComparison.Ordinal); + + /* The reader drops the NULL-rate point; it does not read it as 0. */ + var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "DarlingPgTrendReader.cs"); + var reader = source[source.IndexOf("GetQueryDurationTrendAsync(", StringComparison.Ordinal)..]; + reader = reader[..reader.IndexOf("return points;", StringComparison.Ordinal)]; + Assert.Contains("if (reader.IsDBNull(4))", reader, StringComparison.Ordinal); + Assert.Contains("continue;", reader, StringComparison.Ordinal); + Assert.DoesNotContain("reader.IsDBNull(4) ? 0", reader, StringComparison.Ordinal); + } + /// /// The automatic choice must skip the CPU class. pg_wait_sampling's Running means the /// backend was NOT waiting and dominates any healthy server's profile — measured on the rig it grew by diff --git a/Darling/Darling.Tests/PgWaitInstrumentDisclosureTests.cs b/Darling/Darling.Tests/PgWaitInstrumentDisclosureTests.cs new file mode 100644 index 000000000..4d29c0cbb --- /dev/null +++ b/Darling/Darling.Tests/PgWaitInstrumentDisclosureTests.cs @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using ModelContextProtocol.Server; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The read side of #3604: every surface that serves PostgreSQL waits discloses which of the three +/// instruments fed it, the service tier carries its floor caveat, and the service-side pieces the sampler +/// arm depends on (the connector's probe, the worker's detach, the connection names it excludes) agree with +/// the collector's declarations. +/// +public sealed class PgWaitInstrumentDisclosureTests +{ + private static readonly DateTime T0 = new(2026, 9, 18, 12, 0, 0, DateTimeKind.Utc); + + private static DarlingPgWaitSamplingReader.PgWaitSamplingRow Row(string type, string evt, long samples, int periodMs) => new( + EventType: type, Event: evt, QueryId: 7, SampleCount: samples, EstimatedWaitMs: samples * periodMs, + BackendCount: 1, CounterReset: false, CaptureTime: T0); + + private static JsonElement Build(DarlingPgWaitSamplingReader.WaitInstrumentState? instrument) => + JsonDocument.Parse(DarlingMcpPgWaitSamplingTools.BuildWaitSamplingJson( + "srv", 24, + new DarlingPgWaitSamplingReader.PgWaitSamplingPage([Row("Lock", "relation", 30, 1000)], 30), + limit: 20, instrument)).RootElement; + + /* ───────────────────────── the sampled read ───────────────────────── */ + + [Fact] + public void ServiceTier_IsNamedAndCarriesTheFloorCaveat() + { + var root = Build(new DarlingPgWaitSamplingReader.WaitInstrumentState(PgWaitInstrument.ServiceSampled, T0)); + + Assert.Equal("service_sampled", root.GetProperty("instrument").GetString()); + Assert.Equal(PgWaitInstrument.ServiceSampledCaveat, root.GetProperty("instrument_note").GetString()); + Assert.Contains("FLOOR", root.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + Assert.Contains("distinct backends seen in the LAST window", root.GetProperty("note").GetString(), StringComparison.Ordinal); + Assert.Equal(T0, root.GetProperty("instrument_recorded_at").GetDateTime().ToUniversalTime()); + + /* #3645 review: the tally is persisted collector state, reloaded every cycle, so a SERVICE restart does + not reset the series — the note must say what does (cap eviction, arm reversion) and not claim what + does not. Head 3 shipped the false claim; this is what stops it coming back. */ + var note = root.GetProperty("note").GetString()!; + Assert.Contains("SURVIVES a service restart", note, StringComparison.Ordinal); + Assert.DoesNotContain("resets when the service restarts", note, StringComparison.Ordinal); + Assert.Contains("500-key cap", note, StringComparison.Ordinal); + } + + [Fact] + public void ExtensionTier_IsNamedWithoutTheFloorCaveat() + { + var root = Build(new DarlingPgWaitSamplingReader.WaitInstrumentState(PgWaitInstrument.ExtensionSampled, T0)); + + Assert.Equal("extension_sampled", root.GetProperty("instrument").GetString()); + Assert.Contains("10 ms", root.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + Assert.DoesNotContain("FLOOR", root.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + Assert.DoesNotContain("LAST window", root.GetProperty("note").GetString(), StringComparison.Ordinal); + } + + /// A store written before the arm existed, or a token a future arm introduces, reads as unknown + /// — never as a grain the caller might act on. + [Fact] + public void NoRecordedInstrument_OrAnUnknownToken_ReadsAsUnknown() + { + Assert.Equal("unknown", Build(null).GetProperty("instrument").GetString()); + Assert.Equal(JsonValueKind.Null, Build(null).GetProperty("instrument_recorded_at").ValueKind); + + var foreign = Build(new DarlingPgWaitSamplingReader.WaitInstrumentState("kernel_traced", T0)); + Assert.Equal("unknown", foreign.GetProperty("instrument").GetString()); + Assert.Contains("profile_period_ms", foreign.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + } + + [Fact] + public void TheEmptyArm_StatesTheFloorOnlyOnTheServiceTier() + { + Assert.Contains(PgWaitInstrument.ServiceSampledCaveat, + DarlingMcpPgWaitSamplingTools.DescribeInstrumentForEmpty( + new DarlingPgWaitSamplingReader.WaitInstrumentState(PgWaitInstrument.ServiceSampled, T0)), StringComparison.Ordinal); + Assert.Equal(string.Empty, + DarlingMcpPgWaitSamplingTools.DescribeInstrumentForEmpty( + new DarlingPgWaitSamplingReader.WaitInstrumentState(PgWaitInstrument.ExtensionSampled, T0))); + Assert.Contains("No collection cycle has recorded", DarlingMcpPgWaitSamplingTools.DescribeInstrumentForEmpty(null), StringComparison.Ordinal); + } + + /// The instrument is read off the collector's own state row, under the collector's own name and + /// key — so a rename of either fails here rather than silently reading nothing forever. + [Fact] + public void TheInstrumentSqlReadsTheCollectorsOwnStateRow() + { + Assert.Contains("FROM collector_state", DarlingPgWaitSamplingReader.InstrumentSql, StringComparison.Ordinal); + Assert.Contains($"collector_name = '{PgWaitSamplingCollector.Instance.Name}'", DarlingPgWaitSamplingReader.InstrumentSql, StringComparison.Ordinal); + Assert.Contains($"state_key = '{PgWaitSamplingCollector.InstrumentStateKey}'", DarlingPgWaitSamplingReader.InstrumentSql, StringComparison.Ordinal); + } + + /* ───────────────────────── the Aurora read ───────────────────────── */ + + [Fact] + public void TheAuroraRead_DisclosesEngineCumulative() + { + var json = DarlingMcpPgWaitTools.BuildWaitStatsJson( + "srv", 24, + new DarlingPgWaitReader.PgWaitStatsPage( + [new DarlingPgWaitReader.PgWaitRow("IO", "DataFileRead", TotalWaits: 10, TotalWaitTimeMs: 5_000, AvgWaitTimeMs: 500)], + 5_000), + limit: 20); + var root = JsonDocument.Parse(json).RootElement; + + Assert.Equal("engine_cumulative", root.GetProperty("instrument").GetString()); + Assert.Contains("get_pg_wait_sampling", root.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + } + + /* ───────────────────────── descriptions and instructions ───────────────────────── */ + + private static string ToolDescription(Type toolType, string toolName) => + toolType.GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.GetCustomAttribute()?.Name == toolName) + .GetCustomAttribute()!.Description; + + [Fact] + public void BothToolDescriptions_NameTheThreeTiers() + { + var sampling = ToolDescription(typeof(DarlingMcpPgWaitSamplingTools), "get_pg_wait_sampling"); + Assert.Contains("extension_sampled", sampling, StringComparison.Ordinal); + Assert.Contains("service_sampled", sampling, StringComparison.Ordinal); + Assert.Contains("FLOOR", sampling, StringComparison.Ordinal); + Assert.Contains("Aurora native > pg_wait_sampling extension > service sampler", sampling, StringComparison.Ordinal); + /* The old opening claimed the extension as the ONLY source; it is one of two now. */ + Assert.DoesNotContain("from the pg_wait_sampling extension. This is", sampling, StringComparison.Ordinal); + + var stats = ToolDescription(typeof(DarlingMcpPgWaitTools), "get_pg_wait_stats"); + Assert.Contains("engine_cumulative", stats, StringComparison.Ordinal); + Assert.Contains("get_pg_wait_sampling", stats, StringComparison.Ordinal); + } + + [Fact] + public void TheInstructions_NameTheThreeTiersAndTheOrder() + { + var text = DarlingMcpInstructions.Build(DarlingPeerDirectory.Snapshot.Empty); + Assert.Contains("`engine_cumulative`", text, StringComparison.Ordinal); + Assert.Contains("`extension_sampled`", text, StringComparison.Ordinal); + Assert.Contains("`service_sampled`", text, StringComparison.Ordinal); + Assert.Contains("Aurora native > the `pg_wait_sampling` extension > the service-side sampler", text, StringComparison.Ordinal); + Assert.Contains("FLOOR, not parity", text, StringComparison.Ordinal); + } + + /* ───────────────────────── the service-side seams ───────────────────────── */ + + /// The sampler excludes this service's own backends by application_name, and the names it + /// excludes must be the names the connector presents — the Collectors assembly cannot reference the + /// service, so the agreement is pinned rather than referenced. + [Fact] + public void TheExcludedApplicationNames_AreTheOnesTheConnectorPresents() + { + Assert.Equal(MonitoredServerConnection.RemediationApplicationName, PgWaitSamplingCollector.ServiceRemediationApplicationName); + + var connector = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "MonitoredServerConnection.cs"); + Assert.Contains($"ApplicationName = \"{PgWaitSamplingCollector.ServiceApplicationName}\"", connector, StringComparison.Ordinal); + } + + [Fact] + public void TheConnectProbe_AsksPgExtensionForTheModule_AndFlowsToTheTargetInfo() + { + Assert.Contains("pg_extension", DarlingServerConnector.PostgresWaitSamplingProbeQueryText, StringComparison.Ordinal); + Assert.Contains("'pg_wait_sampling'", DarlingServerConnector.PostgresWaitSamplingProbeQueryText, StringComparison.Ordinal); + + var probe = new ConnectionProbeResult( + Success: true, MajorVersion: 0, EngineEdition: 0, EngineEditionDescription: null, + IsAzureSqlDb: false, IsAzureManagedInstance: false, IsAwsRds: false, HasMsdbAccess: true, Error: null, + Engine: CollectorTargetEngine.PostgreSql, PostgresMajorVersion: 18, PostgresVersionNum: 180001, + HasPgWaitSamplingExtension: true); + Assert.True(probe.ToTargetInfo().HasPgWaitSamplingExtension); + Assert.False((probe with { HasPgWaitSamplingExtension = false }).ToTargetInfo().HasPgWaitSamplingExtension); + } + + /// Detached from the sequential body, behind the generic gate, on a tier the detach policy + /// allows — see SweepBodyDetachPolicyTests for the invariant; this pins the predicate itself. + [Fact] + public void TheCollectorIsDetached_ByItsOwnDeclaredName() + { + Assert.True(DarlingWorker.IsPgWaitSamplingCollector(PgWaitSamplingCollector.Instance.Name)); + Assert.True(DarlingWorker.IsPgWaitSamplingCollector("PG_WAIT_SAMPLING")); + Assert.False(DarlingWorker.IsPgWaitSamplingCollector("pg_wait_stats")); + } +} diff --git a/Darling/Darling.Tests/PgWaitSamplerLiveTests.cs b/Darling/Darling.Tests/PgWaitSamplerLiveTests.cs new file mode 100644 index 000000000..055805be6 --- /dev/null +++ b/Darling/Darling.Tests/PgWaitSamplerLiveTests.cs @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The service-sampler arm of pg_wait_sampling (#3604) run for real: the live store doubles as a +/// stock PostgreSQL TARGET without the extension, a lock wait and a CPU burner are held open on it, and the +/// real runs one cycle. What must come out the other end: rows in +/// pg_wait_sampling at the sampler's period, a Lock series and a CPU series among them, +/// the collector's state row saying service_sampled, and the read disclosing it. +/// +/// Gated on DARLING_TEST_PG like every live class. Takes ~30 s by construction — the window IS +/// the test — so it is one cycle, not two; the cumulative-across-cycles property is proven with a fixture +/// reader in Lite.Tests.PgWaitSamplerArmTests, where it costs nothing. +/// +[Collection("live-postgres")] +public sealed class PgWaitSamplerLiveTests +{ + private const int ServerId = 360_4; + + [Fact] + public async Task OneCycleAgainstAStockTarget_LandsServiceSampledRowsTheReadDiscloses() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live sampler test."); + + var ct = TestContext.Current.CancellationToken; + await using var postgres = NpgsqlDataSource.Create(connectionString!); + + /* The TARGET is the store's own instance, reached with a distinct application_name so the sampler's + self-exclusion (by the service's names) does not hide the workload, and so the workload's own + backends are attributable. */ + var targetBuilder = new NpgsqlConnectionStringBuilder(connectionString) { ApplicationName = "pm3604-target", Pooling = false }; + var workloadBuilder = new NpgsqlConnectionStringBuilder(connectionString) { ApplicationName = "pm3604-workload", Pooling = false }; + + var bodySucceeded = false; + try + { + await using (var setup = new NpgsqlConnection(workloadBuilder.ConnectionString)) + { + await setup.OpenAsync(ct); + await using var cmd = new NpgsqlCommand("CREATE TABLE IF NOT EXISTS pm3604_lock_target (id int)", setup); + await cmd.ExecuteNonQueryAsync(ct); + } + + /* Workload 1: a lock wait. Holder takes ACCESS EXCLUSIVE and sits; the waiter blocks on it for the + whole window — wait_event_type Lock, wait_event relation. */ + await using var holder = new NpgsqlConnection(workloadBuilder.ConnectionString); + await holder.OpenAsync(ct); + await using var holderTx = await holder.BeginTransactionAsync(ct); + await using (var lockCmd = new NpgsqlCommand("LOCK TABLE pm3604_lock_target IN ACCESS EXCLUSIVE MODE", holder, holderTx)) + { + await lockCmd.ExecuteNonQueryAsync(ct); + } + + using var workloadStop = new CancellationTokenSource(); + var waiter = Task.Run(async () => + { + await using var c = new NpgsqlConnection(workloadBuilder.ConnectionString); + await c.OpenAsync(workloadStop.Token); + await using var q = new NpgsqlCommand("SELECT count(*) FROM pm3604_lock_target", c) { CommandTimeout = 120 }; + try { await q.ExecuteScalarAsync(workloadStop.Token); } catch (OperationCanceledException) { } + }, CancellationToken.None); + + /* Workload 2: CPU. A backend on processor with no wait event — the arm's CPU/Running row. */ + var burner = Task.Run(async () => + { + await using var c = new NpgsqlConnection(workloadBuilder.ConnectionString); + await c.OpenAsync(workloadStop.Token); + while (!workloadStop.IsCancellationRequested) + { + await using var q = new NpgsqlCommand("SELECT count(*) FROM generate_series(1, 20000000)", c) { CommandTimeout = 120 }; + try { await q.ExecuteScalarAsync(workloadStop.Token); } catch (OperationCanceledException) { } + } + }, CancellationToken.None); + + await Task.Delay(TimeSpan.FromSeconds(2), ct); + + var runtime = new ServerRuntime + { + Config = new MonitoredServer { Name = "pm3604-stock", Host = targetBuilder.Host ?? "localhost", Engine = "postgres" }, + ConnectionString = targetBuilder.ConnectionString, + Target = new CollectorTargetInfo + { + Engine = CollectorTargetEngine.PostgreSql, + PostgresMajorVersion = 18, + PostgresVersionNum = 180000, + /* The rig has no pg_wait_sampling: the connect probe would say false, and so does this. */ + HasPgWaitSamplingExtension = false, + }, + StorageName = "pm3604-stock", + ServerId = ServerId, + }; + + var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator(), NullLogger.Instance); + + var wall = Stopwatch.StartNew(); + var result = await runner.RunAsync(PgWaitSamplingCollector.Instance, runtime, ct); + wall.Stop(); + + workloadStop.Cancel(); + await holderTx.RollbackAsync(CancellationToken.None); + await Task.WhenAll(waiter, burner); + + /* 1. The run itself: rows landed, the window was the window. */ + Assert.True(result.Rows > 0, $"the sampler wrote no rows; note={result.HostNote}"); + var expectedWindowMs = (PgWaitSamplingCollector.SamplerSnapshotsPerCycle - 1) * PgWaitSamplingCollector.SamplerPeriodMs; + Assert.InRange(wall.ElapsedMilliseconds, expectedWindowMs, expectedWindowMs + 30_000); + + /* 2. The rows: at the sampler's period, with the two workloads visible. */ + var rows = new List<(string Type, string Event, long Samples, int PeriodMs, int Backends)>(); + await using (var read = postgres.CreateCommand( + "SELECT event_type, event, sample_count, profile_period_ms, backend_count FROM pg_wait_sampling WHERE server_id = $1")) + { + read.Parameters.AddWithValue(ServerId); + await using var reader = await read.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + rows.Add((reader.GetString(0), reader.GetString(1), reader.GetInt64(2), reader.GetInt32(3), reader.GetInt32(4))); + } + } + + Assert.NotEmpty(rows); + Assert.All(rows, r => Assert.Equal(PgWaitSamplingCollector.SamplerPeriodMs, r.PeriodMs)); + var lockRow = Assert.Single(rows, r => r.Type == "Lock" && r.Event == "relation"); + /* Held for the whole window, so seen in nearly every snapshot; allow for the first snapshot racing the waiter. */ + Assert.InRange(lockRow.Samples, PgWaitSamplingCollector.SamplerSnapshotsPerCycle / 2, PgWaitSamplingCollector.SamplerSnapshotsPerCycle); + Assert.Equal(1, lockRow.Backends); + Assert.Contains(rows, r => r.Type == "CPU" && r.Event == "Running"); + /* The shared exclusions hold on this arm too: the holder is idle in transaction (Client), and no + background sleeper (Activity/Timeout) is counted. */ + Assert.DoesNotContain(rows, r => PgWaitStatsCollector.IgnoredWaitTypes.Contains(r.Type)); + + /* 3. The instrument, recorded by the collector and read back by the reader the tools use. */ + var instrument = await DarlingPgWaitSamplingReader.GetWaitInstrumentAsync(postgres, ServerId, ct); + Assert.NotNull(instrument); + Assert.Equal(PgWaitInstrument.ServiceSampled, instrument.Instrument); + + var tally = await ReadStateAsync(postgres, PgWaitSamplingCollector.TallyStateKey, ct); + Assert.Equal(lockRow.Samples, PgWaitSamplingCollector.ParseTally(tally)[("Lock", "relation", 0)]); + + /* 4. The read discloses it. */ + var page = await DarlingPgWaitSamplingReader.GetPgWaitSamplingPageAsync( + postgres, ServerId, DateTime.UtcNow.AddHours(-1), DateTime.UtcNow.AddMinutes(1), 21, ct); + var json = JsonDocument.Parse(DarlingMcpPgWaitSamplingTools.BuildWaitSamplingJson("pm3604-stock", 1, page, 20, instrument)).RootElement; + Assert.Equal("service_sampled", json.GetProperty("instrument").GetString()); + Assert.Contains("FLOOR", json.GetProperty("instrument_note").GetString(), StringComparison.Ordinal); + + /* Reported, not asserted: the measured cost of one cycle on this rig. */ + Console.WriteLine( + $"pg_wait_sampling service arm: wall {wall.ElapsedMilliseconds} ms, sql {result.SqlMs} ms, store {result.StorageMs} ms, " + + $"{result.Rows} rows, {rows.Count} series; Lock/relation {lockRow.Samples}/{PgWaitSamplingCollector.SamplerSnapshotsPerCycle} snapshots"); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + { + /* One statement per command: Npgsql does not bind positional parameters across a + multi-statement command. */ + foreach (var sql in new[] + { + "DELETE FROM pg_wait_sampling WHERE server_id = $1", + "DELETE FROM collector_state WHERE server_id = $1", + "DELETE FROM collection_log WHERE server_id = $1", + }) + { + await using var command = new NpgsqlCommand(sql, cleanup); + command.Parameters.AddWithValue(ServerId); + await command.ExecuteNonQueryAsync(cleanupCt); + } + + await using var drop = new NpgsqlCommand("DROP TABLE IF EXISTS pm3604_lock_target", cleanup); + await drop.ExecuteNonQueryAsync(cleanupCt); + }); + } + } + + private static async Task ReadStateAsync(NpgsqlDataSource postgres, string key, CancellationToken ct) + { + await using var command = postgres.CreateCommand( + "SELECT state_value FROM collector_state WHERE server_id = $1 AND collector_name = 'pg_wait_sampling' AND state_key = $2"); + command.Parameters.AddWithValue(ServerId); + command.Parameters.AddWithValue(key); + return await command.ExecuteScalarAsync(ct) as string; + } +} diff --git a/Darling/Darling.Tests/PlanCorrectionFrameLiveTests.cs b/Darling/Darling.Tests/PlanCorrectionFrameLiveTests.cs index 5f35375ae..d69ce932d 100644 --- a/Darling/Darling.Tests/PlanCorrectionFrameLiveTests.cs +++ b/Darling/Darling.Tests/PlanCorrectionFrameLiveTests.cs @@ -115,7 +115,7 @@ with either read. */ /* ── arm A: already UTC. The stored value equals collection_time and must come back equal to it. An offset applied here ADDS four hours, because the offset is negative. ── */ var recommendations = await DarlingPlanCorrectionReader.GetPlanCorrectionsAsync( - postgres, ServerId, collectionTime.AddHours(-1), collectionTime.AddHours(1), ct); + postgres, ServerId, collectionTime.AddHours(-1), collectionTime.AddHours(1), cap: 50, ct); var recommendation = Assert.Single(recommendations); Assert.Equal(collectionTime, recommendation.CollectionTime); @@ -147,7 +147,7 @@ with either read. */ /* ── arm B: server-local. The stored value is 240 minutes behind collection_time and must come back ON it, which is the de-skew doing its job. ── */ var blocked = await DarlingBlockingReader.GetRecentBlockedProcessReportsAsync( - postgres, ServerId, collectionTime.AddHours(-1), collectionTime.AddHours(1), ct); + postgres, ServerId, collectionTime.AddHours(-1), collectionTime.AddHours(1), cap: 50, ct); var report = Assert.Single(blocked); Assert.Equal(collectionTime, report.EventTime!.Value); diff --git a/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs b/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs index 02ff9b5f1..37eb79ec1 100644 --- a/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs +++ b/Darling/Darling.Tests/PostgresAlertEvaluatorTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.Linq; using PerformanceMonitor.Alerting; using PerformanceMonitor.Notifications; @@ -224,7 +225,10 @@ public void AMissingFreezeMaxAgeSettingSilencesRatherThanFiringOnEverything(long /// /// The persistence gate is what keeps this from firing on every long-running report. Same age, same - /// holder — only the persistence differs, and only the chronic one alerts. + /// holder — only the persistence differs, and only the chronic one alerts. The chronic case is the + /// classic single-pid incident, and it still fires through the IDENTITY arm with the holder as the + /// subject and the original "in N of M observations" wording — truthful now that the floor guarantees + /// M is a real sample. /// [Fact] public void XminFiresOnlyForAChronicHolderNotALongQuery() @@ -232,15 +236,118 @@ public void XminFiresOnlyForAChronicHolderNotALongQuery() var chronic = new PostgresXminHorizonAlertInfo("session", "12345", 100_000_000, 30, 40, "idle in transaction"); var transient = new PostgresXminHorizonAlertInfo("session", "12345", 100_000_000, 2, 40, "running"); - Assert.NotNull(PostgresAlertEvaluator.EvaluateXmin(chronic)); + var finding = PostgresAlertEvaluator.EvaluateXmin(chronic); + Assert.NotNull(finding); + Assert.Equal("session:12345", finding!.Subject); + Assert.Contains("in 30 of 40 observations", finding.ShortMessage, StringComparison.Ordinal); + Assert.Null(PostgresAlertEvaluator.EvaluateXmin(transient)); } + /// + /// #3537 edge 1: the identity denominator counts only holder-bearing collections, so the first holder + /// after quiet hours arrived as 1 win in 1 observation — 100%, "chronic", off a single sample. The + /// observation floor closes every denominator too small for its majority to mean anything. + /// + [Theory] + [InlineData(1, 1)] + [InlineData(2, 2)] + [InlineData(4, 4)] + [InlineData(3, 4)] + public void XminDoesNotFireBeneathTheObservationFloorHoweverTotalTheFraction(int held, int total) + { + Assert.Null(PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo("session", "1", 900_000_000, held, total, null))); + } + + /// The floor is a floor, not a fudge: at exactly the minimum, a majority still fires. + [Fact] + public void XminIdentityArmFiresAtExactlyTheObservationFloor() + { + Assert.NotNull(PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo( + "session", "1", 900_000_000, + PostgresAlertEvaluator.XminMinimumObservations, + PostgresAlertEvaluator.XminMinimumObservations, + null))); + } + + /// + /// #3537 edge 2: a horizon continuously pinned past the threshold by a PARADE of distinct holders never + /// accumulates any single holder's identity fraction, and the old gate never fired while the alert's own + /// claim was true the whole time. The horizon arm fires on the horizon's persistence across the window's + /// real captures, names the rotating pattern, still carries the latest holder's remedy — and subjects + /// the stable sentinel, not the latest member, so the host's per-subject cooldown holds across + /// rotations. + /// + [Fact] + public void XminRotatingHoldersFireTheHorizonArmUnderTheStableSubject() + { + var finding = PostgresAlertEvaluator.EvaluateXmin(new PostgresXminHorizonAlertInfo( + "session", "9101", 80_000_000, ObservationsHeld: 1, ObservationsTotal: 60, + "state=idle in transaction", ObservationsAboveThreshold: 70, CapturesInWindow: 120)); + + Assert.NotNull(finding); + Assert.Equal(AlertSeverityLevel.Warning, finding!.Severity); + Assert.Equal(PostgresAlertEvaluator.XminRotatingHoldersSubject, finding.Subject); + Assert.Contains("succession of different holders", finding.ShortMessage, StringComparison.Ordinal); + Assert.Contains("session:9101", finding.ShortMessage, StringComparison.Ordinal); + Assert.Contains("70 of the window's 120 collections", finding.ShortMessage, StringComparison.Ordinal); + /* The remedy names the latest holder's cause — the one actionable thing either way. */ + Assert.Contains("idle in transaction", finding.ShortMessage, StringComparison.OrdinalIgnoreCase); + } + + /// + /// The horizon arm's own boundaries: a majority of the window's real captures fires, one capture short + /// of it does not, and a window with fewer captures than the floor cannot fire at any fraction — which + /// is also what keeps the compat default (no capture data supplied) silent. + /// + [Theory] + [InlineData(60, 120, true)] // exactly the majority + [InlineData(59, 120, false)] // one capture short + [InlineData(4, 4, false)] // 100%, but beneath the capture floor + [InlineData(0, 0, false)] // no capture data supplied — the compat default + public void XminHorizonArmNeedsAMajorityOfAtLeastTheFloorsWorthOfCaptures( + int above, int captures, bool fires) + { + var finding = PostgresAlertEvaluator.EvaluateXmin(new PostgresXminHorizonAlertInfo( + "session", "1", 80_000_000, ObservationsHeld: 1, ObservationsTotal: 60, null, + ObservationsAboveThreshold: above, CapturesInWindow: captures)); + + Assert.Equal(fires, finding is not null); + } + + /// + /// The overlap case, told apart by the identity FRACTION alone: a service restarted into an incident + /// already underway sees a stable holder through a window still too young for the identity floor. The + /// horizon arm supplies the persistence evidence, but the wording must not claim a "succession" and + /// the subject stays the holder — there is exactly one, and it is the thing to kill. + /// + [Fact] + public void XminStableHolderInAYoungWindowKeepsTheHolderSubjectOnAHorizonArmFire() + { + var finding = PostgresAlertEvaluator.EvaluateXmin(new PostgresXminHorizonAlertInfo( + "session", "77", 80_000_000, ObservationsHeld: 3, ObservationsTotal: 3, null, + ObservationsAboveThreshold: 3, CapturesInWindow: 6)); + + Assert.NotNull(finding); + Assert.Equal("session:77", finding!.Subject); + Assert.DoesNotContain("succession", finding.ShortMessage, StringComparison.Ordinal); + Assert.Contains("3 of the window's 6 collections", finding.ShortMessage, StringComparison.Ordinal); + } + [Fact] public void XminBelowTheAgeThresholdNeverFiresHoweverPersistent() { Assert.Null(PostgresAlertEvaluator.EvaluateXmin( new PostgresXminHorizonAlertInfo("session", "1", 1_000_000, 40, 40, null))); + + /* Both arms saturated, current age below the bar: the age gate answers first. The latest reading + is what the alert would quote as "current", so a horizon that has already come back under the + threshold must not page off its own history. */ + Assert.Null(PostgresAlertEvaluator.EvaluateXmin( + new PostgresXminHorizonAlertInfo("session", "1", 1_000_000, 40, 40, null, + ObservationsAboveThreshold: 120, CapturesInWindow: 120))); } /// A zero denominator must not divide — it means nothing was observed, so nothing fires. @@ -537,4 +644,172 @@ public void PoisonWaitSubjectIsTheTypeColonEventPairInStoredCasing() PostgresAlertEvaluator.PoisonWaitSubject(row), PostgresAlertEvaluator.EvaluatePoisonWait(row)!.Subject); } + + /* ---------------- poison waits: the SQL Server port (#3539 A4) ---------------- */ + + private static PoisonWaitAccumulation SqlPoison(long accumulatedMs, string waitType = "THREADPOOL", long waits = 1) + => new(waitType, accumulatedMs, waits, 10, new DateTime(2026, 9, 18, 12, 0, 0)); + + /// + /// The constants are LITERALLY shared — one definition on , with the + /// PostgreSQL names as aliases of it in source, not merely three numbers that happen to agree today. The + /// value equality is the cheap half; the source pin is the load-bearing one, because two equal literals + /// are exactly the state that drifts (a future "tune the SQL Server bar" edit would leave the PostgreSQL + /// one behind, and one alert name under one mute key would mean two things again — the #3539 finding). + /// + [Fact] + public void PoisonWaitConstantsAreOneDefinition_SharedByBothEngines() + { + Assert.Equal(PoisonWaitEvaluator.WindowMinutes, PostgresAlertEvaluator.PoisonWaitWindowMinutes); + Assert.Equal(PoisonWaitEvaluator.WarningAvgWaiters, PostgresAlertEvaluator.PoisonWaitWarningAvgWaiters); + Assert.Equal(PoisonWaitEvaluator.CriticalAvgWaiters, PostgresAlertEvaluator.PoisonWaitCriticalAvgWaiters); + + var source = RepoFile.ReadRepoFile("PerformanceMonitor.Alerting", "PostgresAlertEvaluator.cs"); + Assert.Contains("public const int PoisonWaitWindowMinutes = PoisonWaitEvaluator.WindowMinutes;", source, StringComparison.Ordinal); + Assert.Contains("public const double PoisonWaitWarningAvgWaiters = PoisonWaitEvaluator.WarningAvgWaiters;", source, StringComparison.Ordinal); + Assert.Contains("public const double PoisonWaitCriticalAvgWaiters = PoisonWaitEvaluator.CriticalAvgWaiters;", source, StringComparison.Ordinal); + /* The positive control for the pin below: the same Contains form does find a literal initializer + that IS in the file (the metric name), so its silence on a "= 1.0" for the poison bar is a real + absence rather than a matcher that never matches. */ + Assert.Contains("public const string PoisonWaitMetric = \"Poison Wait\";", source, StringComparison.Ordinal); + Assert.DoesNotContain("PoisonWaitWarningAvgWaiters = 1.0", source, StringComparison.Ordinal); + + /* And the figures themselves, as documented on the shared home: 10 minutes, one waiter, ten. */ + Assert.Equal(10, PoisonWaitEvaluator.WindowMinutes); + Assert.Equal(600_000d, PoisonWaitEvaluator.WindowMs); + Assert.Equal(1.0, PoisonWaitEvaluator.WarningAvgWaiters); + Assert.Equal(10.0, PoisonWaitEvaluator.CriticalAvgWaiters); + } + + /// + /// Both engines grade the same accumulated milliseconds to the same tier at every documented boundary — + /// the parity the shared constants promise, checked through both evaluators' own entry points rather + /// than through the constants alone. + /// + [Theory] + [InlineData(0, null)] + [InlineData(599_999, null)] + [InlineData(600_000, "Warning")] + [InlineData(5_999_999, "Warning")] + [InlineData(6_000_000, "Critical")] + public void PoisonWaitGradesIdenticallyOnBothEngines(long accumulatedMs, string? expected) + { + var pg = PostgresAlertEvaluator.EvaluatePoisonWait(Poison(accumulatedMs)); + var sql = PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(accumulatedMs)); + var shared = PoisonWaitEvaluator.Grade(accumulatedMs); + + if (expected is null) + { + Assert.Null(pg); + Assert.Null(sql); + Assert.Null(shared); + return; + } + + var tier = Enum.Parse(expected); + Assert.Equal(tier, pg!.Severity); + Assert.Equal(tier, sql!.Severity); + Assert.Equal(tier, shared); + /* Same numeric pair, too: accumulated ms against the breached bar in ms. */ + Assert.Equal(pg.NumericCurrentValue, (double)sql.AccumulatedWaitMs); + Assert.Equal(pg.NumericThresholdValue, sql.NumericThresholdValue); + } + + /// + /// The SQL Server mirror of : 300,000 + /// THREADPOOL waits of 2 ms each is one task continuously starved for the whole window and fires + /// Warning; the retired avg-ms-per-wait bar (500) read the same window as 2 ms. And the mirror of the + /// false page: one 600 ms wait — the shape the old bar paged CRITICAL on — is silent. + /// + [Fact] + public void SqlServerPoisonWaitFiresOnTheStorm_AndNotOnOneSlowWait() + { + var storm = PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(600_000, waits: 300_000)); + Assert.NotNull(storm); + Assert.Equal(AlertSeverityLevel.Warning, storm!.Severity); + Assert.Equal(1.0, storm.AvgWaiters); + Assert.Equal("THREADPOOL (600s in 10m)", storm.CurrentValueClause); + + Assert.Null(PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(600, waits: 1))); + } + + /// + /// The SQL Server fleet-quiet pin, the twin of : + /// on 43 servers over 4 days the worst ten-minute bucket anywhere held 5,795 ms of THREADPOOL (0.0097 + /// avg waiters — ~100x under the bar), the largest single row was 703 tasks at 8.2 ms (5,779 ms), and + /// one server's daily compile burst sat at 3,154 ms over 8 tasks just under the OLD bar. None may fire. + /// + [Theory] + [InlineData(5_795, "THREADPOOL")] + [InlineData(5_779, "THREADPOOL")] + [InlineData(3_154, "RESOURCE_SEMAPHORE_QUERY_COMPILE")] + public void SqlServerPoisonWaitStaysSilentOnTheWorstFleetBucketObserved(long accumulatedMs, string waitType) + { + Assert.Null(PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(accumulatedMs, waitType))); + } + + /// + /// A (0, 0) row — the delta calculator's "no delta is knowable here" marker, indistinguishable in + /// wait_stats from a genuinely idle interval — is not evidence of anything: it does not fire (nothing + /// accumulated) and it is not a finding of quiet either; the evaluator returns no finding for it and + /// says nothing about the window's health. Which of "observed" and "silent" applies is the engine's + /// call, made on whether rows came back at all (pinned in AlertEngineTests), never on a zero. + /// + [Fact] + public void SqlServerPoisonWaitTreatsAZeroRowAsNoEvidence() + { + Assert.Null(PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(0, waits: 0))); + Assert.Empty(PoisonWaitEvaluator.EvaluateSqlServer(new[] { SqlPoison(0, waits: 0), SqlPoison(0, "RESOURCE_SEMAPHORE", 0) })); + Assert.Empty(PoisonWaitEvaluator.EvaluateSqlServer((IReadOnlyList?)null)); + Assert.Empty(PoisonWaitEvaluator.EvaluateSqlServer(Array.Empty())); + } + + /// Worst-first: severity, then accumulated wait — so the engine's "worst" and mute key are stable. + [Fact] + public void SqlServerPoisonWaitFindingsAreOrderedWorstFirst() + { + var findings = PoisonWaitEvaluator.EvaluateSqlServer(new[] + { + SqlPoison(700_000, "THREADPOOL"), // Warning, more ms + SqlPoison(6_000_000, "RESOURCE_SEMAPHORE"), // Critical + SqlPoison(650_000, "RESOURCE_SEMAPHORE_QUERY_COMPILE"), // Warning, fewer ms + SqlPoison(100, "THREADPOOL"), // under the bar + }); + + Assert.Equal(new[] { "RESOURCE_SEMAPHORE", "THREADPOOL", "RESOURCE_SEMAPHORE_QUERY_COMPILE" }, + findings.Select(f => f.WaitType).ToArray()); + Assert.Equal(10.0, findings[0].BreachedAvgWaiters); + Assert.Equal(1.0, findings[1].BreachedAvgWaiters); + } + + /// + /// The two engines' messages share one clause order — seconds accumulated, the window, the wait count, + /// the average stuck — with only the noun differing (task / backend), so a "Poison Wait" read alike from + /// either engine; and the SQL remedy names the fix for its type like the PostgreSQL one does. + /// + [Fact] + public void SqlServerPoisonWaitMessageMirrorsThePostgresShape() + { + var sql = PoisonWaitEvaluator.EvaluateSqlServer(SqlPoison(600_000, waits: 300_000))!; + var pg = PostgresAlertEvaluator.EvaluatePoisonWait(Poison(600_000, waits: 300_000))!; + + Assert.Equal( + "[THREADPOOL] 600s of wait accumulated in the last 10 minutes across 300,000 waits — on average 1.0 task(s) continuously stuck", + sql.ShortMessage); + Assert.StartsWith("[IPC:BtreePage] 600s of wait accumulated in the last 10 minutes across 300,000 waits — on average 1.0 backend(s) continuously stuck", pg.ShortMessage, StringComparison.Ordinal); + Assert.Equal( + pg.ThresholdValue.Replace("backend(s)", "task(s)", StringComparison.Ordinal), + sql.ThresholdValue); + } + + [Theory] + [InlineData("THREADPOOL", "worker thread")] + [InlineData("threadpool", "worker thread")] + [InlineData("RESOURCE_SEMAPHORE", "memory grants")] + [InlineData("RESOURCE_SEMAPHORE_QUERY_COMPILE", "compile memory")] + [InlineData("SOMETHING_ELSE", "active-query snapshots")] + public void SqlServerPoisonWaitRemedyNamesTheFixForItsType(string waitType, string fragment) + { + Assert.Contains(fragment, PoisonWaitEvaluator.SqlServerRemedyFor(waitType), StringComparison.Ordinal); + } } diff --git a/Darling/Darling.Tests/QueryDopProvenanceTests.cs b/Darling/Darling.Tests/QueryDopProvenanceTests.cs new file mode 100644 index 000000000..53a2c48fd --- /dev/null +++ b/Darling/Darling.Tests/QueryDopProvenanceTests.cs @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Common; +using Xunit; + +namespace Darling.Tests; + +/// +/// Decision-table pins for the shared (#3648) — the one sentence both +/// SKUs' top_cpu_queries / bad_actor_query drill-downs attach when a query-stats group's +/// cross-plan max_dop history disagrees with the newest plan's reading. This SAME table is pinned +/// identically in Darling.Tests (QueryDopProvenanceTests) so the two SKUs cannot drift; the shape is +/// QueryStatExtremesTests', the #2235 precedent for a lifetime-extreme annotation. +/// +public sealed class QueryDopProvenanceTests +{ + private static readonly DateTime Seen = new(2026, 9, 4, 13, 45, 0, DateTimeKind.Unspecified); + + /// + /// The live page (#3648): the newest plan is serial, an older plan ran at 16, three plans in the window. + /// The exact format renderers print verbatim. + /// + [Fact] + public void ParallelHistoryBehindASerialHeadline_IsSpelledOut() + { + Assert.Equal( + "DOP 1 (a parallel plan ran at 16 until 2026-09-04; 3 plans in window)", + QueryDopProvenance.Note(newestPlanMaxDop: 1, maxDopAnyPlan: 16, maxDopAnyPlanLastSeen: Seen, planCount: 3)); + } + + /// Row #3 on the same card: one serial plan, headline equals history, nothing to disclose. + [Fact] + public void HeadlineEqualToTheMaximum_CarriesNoNote() + { + Assert.Null(QueryDopProvenance.Note(1, 1, Seen, 1)); + Assert.Null(QueryDopProvenance.Note(8, 8, Seen, 2)); + } + + /// + /// A cumulative per-plan maximum can never be BELOW the newest plan's own reading unless the group is + /// inconsistent; the guard is <=, so that case is also silent rather than inventing a history. + /// + [Fact] + public void MaximumBelowTheHeadline_CarriesNoNote() + { + Assert.Null(QueryDopProvenance.Note(4, 2, Seen, 1)); + } + + /// No row in the group carried a reading: nothing to compare, nothing to say. + [Fact] + public void NoReadingAnywhere_CarriesNoNote() + { + Assert.Null(QueryDopProvenance.Note(null, null, null, 1)); + } + + /// + /// Newest reading unknown, only serial plans on record: "unknown" is already the whole truth and there + /// is no parallel plan to disclose. + /// + [Fact] + public void UnknownHeadline_WithOnlySerialHistory_CarriesNoNote() + { + Assert.Null(QueryDopProvenance.Note(null, 1, Seen, 2)); + } + + /// + /// Newest reading unknown but a parallel plan is on record — the case where a reader would otherwise + /// reach for the folded maximum, so it is named as unknown and the history is disclosed. + /// + [Fact] + public void UnknownHeadline_WithParallelHistory_SaysUnknownAndDiscloses() + { + Assert.Equal( + "DOP unknown for the newest plan (a parallel plan ran at 16 until 2026-09-04; 2 plans in window)", + QueryDopProvenance.Note(null, 16, Seen, 2)); + } + + /// + /// One plan SHAPE (a recompile to the same query_plan_hash after MAXDOP was lowered resets the counter) + /// still reads as a history; the count is singular and the "until" clause is dropped when the store + /// could not say when the maximum was last seen. + /// + [Fact] + public void SinglePlan_AndNoLastSeen_AreSpelledWithoutInvention() + { + Assert.Equal( + "DOP 1 (a parallel plan ran at 16; 1 plan in window)", + QueryDopProvenance.Note(1, 16, null, 1)); + } +} diff --git a/Darling/Darling.Tests/QueryStoreTrendRoutingLiveTests.cs b/Darling/Darling.Tests/QueryStoreTrendRoutingLiveTests.cs index 7d0c7125d..d1963d331 100644 --- a/Darling/Darling.Tests/QueryStoreTrendRoutingLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreTrendRoutingLiveTests.cs @@ -45,6 +45,12 @@ public sealed class QueryStoreTrendRoutingLiveTests private const string ServerName = "qs-trend-routing-e2e"; + /// The second server (#3541 A2): sampled only in the unmaterialized tail. Distinctive fake id + /// for the same reason as . + private const int TailOnlyServerId = -927361; + + private const string TailOnlyServerName = "qs-trend-routing-tail-only"; + [Fact] public async Task DurationTrend_ServesTheRollupBelowTheBoundary_AndRanksOnlyTheTail() { @@ -91,6 +97,14 @@ await SeedSnapshotsAsync(connection, intervalId: 3102, queryId: 62, intervalStar await SeedSnapshotsAsync(connection, intervalId: 3103, queryId: 63, intervalStart: hour12, avgDurationUs: 300, [(hour12.AddMinutes(5), 3L), (hour12.AddMinutes(20), 9L)], ct); + /* ── a SECOND server with one interval in the tail only (#3541 A2): sampled, so a window over the + rollup region that finds nothing for it is a QUIET stretch — and when that window's head sits + below the rollup's floor, the empty answer has to say which part is quiet and which part is + unserved rather than advising a wider window for both. ── */ + await DarlingMcpTestData.RegisterServerAsync(connection, TailOnlyServerId, TailOnlyServerName, ct); + await SeedSnapshotsAsync(connection, intervalId: 3104, queryId: 64, intervalStart: hour12, + avgDurationUs: 500, [(hour12.AddMinutes(10), 4L)], ct, serverId: TailOnlyServerId, serverName: TailOnlyServerName); + await EnsureAggregatesWithoutRefreshPoliciesAsync(connection, ct); /* Materialize ONLY the 10:00 and 11:00 buckets — the 12:00 hour stays raw, the exact state a live @@ -116,18 +130,20 @@ await SeedSnapshotsAsync(connection, intervalId: 3103, queryId: 63, intervalStar /* 10:00 — rollup bucket: interval P only (21, once). Interval M ran at 10:00 but was FETCHED at 11:00, so the rollup charges it to 11:00 — the collection-hour placement the payload discloses. */ Assert.Equal(hour10, points[0].CollectionTime); + /* The first united point has no predecessor to difference against: null, not 0 (#3541 A12). */ + Assert.False(points[0].HasRate); /* 11:00 — rollup bucket: M's final snapshot (40) + N's (25) = 65 executions over the 3,600 seconds since the previous point. Un-deduped this hour would be 10+40+5+25 = 80 — the rank the rollup already did. */ Assert.Equal(hour11, points[1].CollectionTime); - Assert.Equal(65d / 3600d, points[1].ExecutionsPerSecond, 6); - Assert.Equal(((40d * 100d + 25d * 200d) / 1000d) / 3600d, points[1].Value, 6); + Assert.Equal(65d / 3600d, points[1].ExecutionsPerSecond!.Value, 6); + Assert.Equal(((40d * 100d + 25d * 200d) / 1000d) / 3600d, points[1].Value!.Value, 6); /* 12:00 — the raw tail: interval T deduped to its final snapshot (9, not 3+9), placed at its interval start, rated over the seam to the last rollup bucket. */ Assert.Equal(hour12, points[2].CollectionTime); - Assert.Equal(9d / 3600d, points[2].ExecutionsPerSecond, 6); + Assert.Equal(9d / 3600d, points[2].ExecutionsPerSecond!.Value, 6); /* ── the raw-only route on the SAME fixture: the estimator this replaced. Interval M lands at its interval START (10:00 — so that hour reads 21+40=61) and the 11:00 point carries only N. This @@ -138,9 +154,11 @@ interval START (10:00 — so that hour reads 21+40=61) and the 11:00 point carri Assert.Equal(3, rawPoints.Count); Assert.Equal(hour10, rawPoints[0].CollectionTime); + /* The first united point has no predecessor to difference against: null, not 0 (#3541 A12). */ + Assert.False(rawPoints[0].HasRate); Assert.Equal(hour11, rawPoints[1].CollectionTime); - Assert.Equal(25d / 3600d, rawPoints[1].ExecutionsPerSecond, 6); - Assert.Equal(9d / 3600d, rawPoints[2].ExecutionsPerSecond, 6); + Assert.Equal(25d / 3600d, rawPoints[1].ExecutionsPerSecond!.Value, 6); + Assert.Equal(9d / 3600d, rawPoints[2].ExecutionsPerSecond!.Value, 6); /* ── the MCP payload discloses the routing: which relation served which region, and that the window's head reaches below the rollup's floor ── */ @@ -149,18 +167,29 @@ interval START (10:00 — so that hour reads 21+40=61) and the 11:00 point carri Assert.Equal(3, payload.GetProperty("trend").GetArrayLength()); - var source = payload.GetProperty("source"); - Assert.Equal("rollup+raw", source.GetProperty("tier").GetString()); - Assert.Equal(TimescaleSupport.QueryStoreStatsCorrectedHourlyView, source.GetProperty("rollup").GetString()); - Assert.StartsWith("2026-03-04T12:00:00", source.GetProperty("raw_from").GetString()!, StringComparison.Ordinal); + /* #3541 A2: the disclosure speaks the vocabulary get_query_trend and the two plan-cache siblings + share — `source` is the tier WORD, and the #2736 seam detail lives under `routing`. Same + assertions as before the move, on the moved keys. */ + Assert.Equal("rollup+raw", payload.GetProperty("source").GetString()); + var routing = payload.GetProperty("routing"); + Assert.Equal(TimescaleSupport.QueryStoreStatsCorrectedHourlyView, routing.GetProperty("rollup").GetString()); + Assert.StartsWith("2026-03-04T12:00:00", routing.GetProperty("raw_from").GetString()!, StringComparison.Ordinal); + Assert.Contains("routing.raw_from", payload.GetProperty("aggregate_note").GetString()!, StringComparison.Ordinal); + Assert.Contains("1 hour before routing.raw_from", payload.GetProperty("bucket").GetString()!, StringComparison.Ordinal); /* The window starts at 07:00, below the rollup's 10:00 floor — a degraded answer must label - itself: points before the floor are missing, not zero, and the remedy is named. */ - Assert.StartsWith("2026-03-04T10:00:00", source.GetProperty("unserved_before").GetString()!, StringComparison.Ordinal); - Assert.Contains("--backfill-rollups", source.GetProperty("unserved_note").GetString()!, StringComparison.Ordinal); + itself: points before the floor are missing, not zero, and the remedy is named. The shared + coverage words say the same thing in the shared shape: the series begins at 10:00, three hours + after what was asked for, so it is truncated. */ + Assert.StartsWith("2026-03-04T10:00:00", routing.GetProperty("unserved_before").GetString()!, StringComparison.Ordinal); + Assert.Contains("--backfill-rollups", routing.GetProperty("unserved_note").GetString()!, StringComparison.Ordinal); + Assert.StartsWith("2026-03-04T10:00:00", payload.GetProperty("effective_start").GetString()!, StringComparison.Ordinal); + Assert.Equal(3.0, payload.GetProperty("effective_hours_back").GetDouble()); + Assert.True(payload.GetProperty("truncated").GetBoolean()); /* ── a window ENTIRELY below the floor is a coverage gap, not a quiet server: the empty answer - names the mechanism and the remedy instead of advising a wider window. ── */ + names the mechanism and the remedy instead of advising a wider window — and carries the same + disclosure block the data envelope does, with the served span honestly zero. ── */ var belowFloor = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend( postgres, ServerName, hours_back: 1, as_of: "2026-03-04T09:30:00Z")).RootElement; @@ -169,6 +198,42 @@ names the mechanism and the remedy instead of advising a wider window. ── */ Assert.Contains("--backfill-rollups", message, StringComparison.Ordinal); Assert.Contains("2026-03-04T10:00:00", message, StringComparison.Ordinal); Assert.DoesNotContain("widen", message, StringComparison.OrdinalIgnoreCase); + Assert.Equal("rollup+raw", belowFloor.GetProperty("source").GetString()); + Assert.StartsWith("2026-03-04T09:30:00", belowFloor.GetProperty("effective_start").GetString()!, StringComparison.Ordinal); + Assert.Equal(0.0, belowFloor.GetProperty("effective_hours_back").GetDouble()); + Assert.True(belowFloor.GetProperty("truncated").GetBoolean()); + + /* ── #3541 A2: the tail-only server over a window whose head is below the floor and whose covered + part holds nothing for it. Sampled, so not "unavailable"; nothing in [10:00, 11:00), so + "empty" — but the message must split the window: quiet from the floor on, UNSERVED before it, + and "widen" only for what widening can do. ── */ + var partialHead = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend( + postgres, TailOnlyServerName, hours_back: 2, as_of: "2026-03-04T11:00:00Z")).RootElement; + + Assert.Equal("empty", partialHead.GetProperty("status").GetString()); + var partialMessage = partialHead.GetProperty("message").GetString()!; + Assert.Contains("2026-03-04T10:00:00", partialMessage, StringComparison.Ordinal); + Assert.Contains("unserved rather than quiet", partialMessage, StringComparison.Ordinal); + Assert.Contains("--backfill-rollups", partialMessage, StringComparison.Ordinal); + Assert.Contains("cannot reach the unserved head", partialMessage, StringComparison.Ordinal); + Assert.DoesNotContain("EVER", partialMessage, StringComparison.Ordinal); + + /* And the same server over its tail alone: one raw-arm point at 12:00, exactly where the window + starts, so nothing is truncated and the served span is the whole hour. */ + var tailOnly = JsonDocument.Parse(await DarlingMcpTrendTools.GetQueryStoreDurationTrend( + postgres, TailOnlyServerName, hours_back: 1, as_of: "2026-03-04T13:00:00Z")).RootElement; + + Assert.Equal(1, tailOnly.GetProperty("trend").GetArrayLength()); + Assert.Equal("rollup+raw", tailOnly.GetProperty("source").GetString()); + Assert.StartsWith("2026-03-04T12:00:00", tailOnly.GetProperty("effective_start").GetString()!, StringComparison.Ordinal); + Assert.False(tailOnly.GetProperty("truncated").GetBoolean()); + /* #3541 A12: a lone point has nothing to difference against, so it is UNRATED — `value` and its named + twin are both null (this assertion used to compare two fabricated zeros), the point is still + there (so effective_start above is truthful), and the envelope says why. */ + Assert.Equal(JsonValueKind.Null, tailOnly.GetProperty("trend")[0].GetProperty("value").ValueKind); + Assert.Equal(JsonValueKind.Null, tailOnly.GetProperty("trend")[0].GetProperty("elapsed_ms_per_second").ValueKind); + Assert.Equal(1, tailOnly.GetProperty("unrated_points").GetInt32()); + Assert.Contains("Unknowable is not 0", tailOnly.GetProperty("unrated_note").GetString()!, StringComparison.Ordinal); } /// @@ -178,7 +243,8 @@ names the mechanism and the remedy instead of advising a wider window. ── */ /// private static async Task SeedSnapshotsAsync( NpgsqlConnection connection, long intervalId, long queryId, DateTime intervalStart, - long avgDurationUs, (DateTime When, long Count)[] snapshots, CancellationToken ct) + long avgDurationUs, (DateTime When, long Count)[] snapshots, CancellationToken ct, + int serverId = TestServerId, string serverName = ServerName) { const string sql = @" INSERT INTO collect.query_store_stats @@ -195,8 +261,8 @@ INSERT INTO collect.query_store_stats await using var command = new NpgsqlCommand(sql, connection); command.Parameters.AddWithValue(when); command.Parameters.AddWithValue(intervalId); - command.Parameters.AddWithValue(TestServerId); - command.Parameters.AddWithValue(ServerName); + command.Parameters.AddWithValue(serverId); + command.Parameters.AddWithValue(serverName); command.Parameters.AddWithValue(queryId); command.Parameters.AddWithValue(intervalStart); command.Parameters.AddWithValue(count); diff --git a/Darling/Darling.Tests/RepoFileAdoptionTests.cs b/Darling/Darling.Tests/RepoFileAdoptionTests.cs index 3256cc688..afc6f269c 100644 --- a/Darling/Darling.Tests/RepoFileAdoptionTests.cs +++ b/Darling/Darling.Tests/RepoFileAdoptionTests.cs @@ -116,6 +116,14 @@ public sealed class RepoFileAdoptionTests "FleetCardCollectionStaleNamesItsPopulationTests.cs", "FleetPageAttentionFilterTests.cs", "LockedModeRestoreCoverageTests.cs", + /* #3541 A10: its top-level-key discriminator anchors on the line break BEFORE the key (a per-row + collection_time inside a Select is indented deeper and must not match), and its tool-body slicing + keys on attribute text either side of one. */ + "McpLatestSnapshotStampTests.cs", + /* #3541 A3: its Lite-description anchor spans the line break between `Description(` and the + string on get_plan_corrections, and its tool-body slicing keys on attribute text either side of + one. */ + "McpPageContractTests.cs", "NightlyVersionInjectionTests.cs", "PgCpuCapacityHeadroomTests.cs", "PgIndexBloatGridReachTests.cs", diff --git a/Darling/Darling.Tests/SelfAlertDeliveryStampStoreLivePostgresTests.cs b/Darling/Darling.Tests/SelfAlertDeliveryStampStoreLivePostgresTests.cs new file mode 100644 index 000000000..1b283524b --- /dev/null +++ b/Darling/Darling.Tests/SelfAlertDeliveryStampStoreLivePostgresTests.cs @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: every test here mints its own scratch database through ScratchPostgres and touches + nothing on the shared one, so serializing it against the live-postgres collection would cost suite time + and buy no isolation. */ + +/// +/// The half of #3580's stamp store no unit pin reaches: that the two statements PARSE against a migrated +/// store, that a stamp round-trips to the tick with its Kind, that the upsert replaces rather than +/// duplicates, that the row sits where the class remarks say it sits (server_id = 0, +/// collector_name = 'self_alert', updated_at a naive UTC write time), and that a value the +/// store cannot parse reads as no stamp rather than as a throw — the +/// shape, over a table that has existed since V44 and so needs no rung of its own. +/// +public sealed class SelfAlertDeliveryStampStoreLivePostgresTests +{ + [Fact] + public async Task AStampRoundTrips_Replaces_AndSitsUnderTheFleetSentinel() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live delivery-stamp round-trip (it mints its own scratch database)."); + + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using (var migrate = new NpgsqlConnection(scratch.ConnectionString)) + { + await migrate.OpenAsync(ct); + await PgMigrations.MigrateAsync(migrate, null, ct); + } + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + var log = new CapturingTestLogger(); + var store = new PgSelfAlertDeliveryStampStore(postgres, log); + + /* A fresh store answers null for both keys — no throw, no phantom row. */ + Assert.Null(await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, ct)); + Assert.Null(await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, ct)); + + /* Round-trip to the tick, Kind Utc coming back — and the two keys are independent rows. */ + var digestAt = new DateTime(2026, 9, 17, 23, 30, 12, DateTimeKind.Utc).AddTicks(1234567); + await store.RecordDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, digestAt, ct); + var readBack = await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, ct); + Assert.Equal(digestAt, readBack); + Assert.Equal(DateTimeKind.Utc, readBack!.Value.Kind); + Assert.Null(await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, ct)); + + /* A Kind-Unspecified caller value (the evaluator's clock seam can hand one in) is stamped Utc on the + way in, so it reads back Utc and equal. */ + var rollupAt = new DateTime(2026, 9, 18, 0, 15, 0, DateTimeKind.Unspecified); + await store.RecordDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, rollupAt, ct); + var rollupBack = await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, ct); + Assert.Equal(rollupAt, rollupBack); + Assert.Equal(DateTimeKind.Utc, rollupBack!.Value.Kind); + + /* The upsert REPLACES: a second delivery a day later overwrites the digest's row, and the table holds + exactly two rows under this owner — one per document — not three. */ + var laterDigestAt = digestAt.AddDays(1); + await store.RecordDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, laterDigestAt, ct); + Assert.Equal(laterDigestAt, await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, ct)); + + await using (var connection = await postgres.OpenConnectionAsync(ct)) + { + await using var rows = new NpgsqlCommand(@" +SELECT server_id, collector_name, state_key, state_value, updated_at +FROM collect.collector_state +WHERE collector_name = $1 +ORDER BY state_key", connection); + rows.Parameters.AddWithValue(PgSelfAlertDeliveryStampStore.StateCollectorName); + await using var reader = await rows.ExecuteReaderAsync(ct); + + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal(PgSelfAlertDeliveryStampStore.FleetServerId, reader.GetInt32(0)); + Assert.Equal("self_alert", reader.GetString(1)); + Assert.Equal(PgSelfAlertDeliveryStampStore.CostDigestStateKey, reader.GetString(2)); + /* The value is the round-trip text with its Z — what makes the read side's Kind honest. */ + Assert.EndsWith("Z", reader.GetString(3), StringComparison.Ordinal); + /* updated_at is the WRITE time, naive UTC: within a minute of now, read as Unspecified from a + `timestamp` column, and never the server's local rendering of a timestamptz cast (the trap the + runner's own comment measured at exactly one zone offset). */ + var updatedAt = reader.GetDateTime(4); + Assert.Equal(DateTimeKind.Unspecified, updatedAt.Kind); + Assert.InRange(DateTime.UtcNow - DateTime.SpecifyKind(updatedAt, DateTimeKind.Utc), TimeSpan.FromMinutes(-1), TimeSpan.FromMinutes(1)); + + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal(PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, reader.GetString(2)); + + Assert.False(await reader.ReadAsync(ct), "the upsert duplicated a stamp row instead of replacing it"); + } + + /* A value this build cannot parse — a hand edit, or a writer this build does not know — reads as NO + stamp with a warning, so the gate falls back to memory for the tick rather than the pass dying on a + FormatException; the next delivery overwrites it. */ + await using (var connection = await postgres.OpenConnectionAsync(ct)) + { + await using var poison = new NpgsqlCommand(@" +UPDATE collect.collector_state +SET state_value = 'yesterday-ish' +WHERE server_id = $1 AND collector_name = $2 AND state_key = $3", connection); + poison.Parameters.AddWithValue(PgSelfAlertDeliveryStampStore.FleetServerId); + poison.Parameters.AddWithValue(PgSelfAlertDeliveryStampStore.StateCollectorName); + poison.Parameters.AddWithValue(PgSelfAlertDeliveryStampStore.CostDigestStateKey); + Assert.Equal(1, await poison.ExecuteNonQueryAsync(ct)); + } + + Assert.Null(await store.GetDeliveredAtUtcAsync(PgSelfAlertDeliveryStampStore.CostDigestStateKey, ct)); + Assert.Contains("not a round-trip UTC instant", log.Joined, StringComparison.Ordinal); + Assert.Contains("yesterday-ish", log.Joined, StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/SelfAlertDeliveryStampTests.cs b/Darling/Darling.Tests/SelfAlertDeliveryStampTests.cs new file mode 100644 index 000000000..9fffabf73 --- /dev/null +++ b/Darling/Darling.Tests/SelfAlertDeliveryStampTests.cs @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3580: the two daily documents — the collector-cost digest and the fleet-sweep rollup — gate on +/// DELIVERED-TODAY in the store rather than fired-today in process memory, so a service restart does not +/// re-announce a document the previous process delivered an hour ago, and DOES re-attempt one whose +/// delivery failed. +/// +/// Every pin here runs over BOTH documents (): the issue names both, +/// the fix is one shared gate, and a pin over one document would let the other drift back to memory-only +/// unnoticed. Each case builds the evaluator TWICE over one stamp store — the "simulated restart" is a new +/// evaluator instance with empty dictionaries and the same store, which is exactly what a fresh process is. +/// The deliverer is the product's own derivation fed a fabricated +/// fan-out result, not a hand-written disposition, so "failed" here is the value the shipped deliverer +/// would actually report for a webhook that came back 500. +/// +/// The install-night pair, as pins. Six re-announcements among ~23 posts is +/// ; the failed pair that +/// the restart correctly recovered is . +/// The two are asserted in the same file so neither can be "fixed" at the other's expense: a gate that +/// never re-delivers passes the first and fails the second; a gate that always re-delivers, the reverse. +/// +public class SelfAlertDeliveryStampTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static readonly DateTime Day = new(2026, 9, 17, 23, 30, 0, DateTimeKind.Utc); + + /* ---------------- fakes ---------------- */ + + /// A deliverer that records what it was handed and REPORTS a disposition — the product's own + /// derivation over a fabricated fan-out, so the values are the ones DarlingAlertDeliverer would + /// return, not literals a test author believes it returns. + private sealed class ReportingDeliverer : IAlertDeliverer + { + public List Outcomes { get; } = new(); + + /// What the webhook channel does on the next delivery. Delivered by default — the + /// steady state; a test flips it to Failed to stage the install-night fault. + public AlertChannelOutcome Webhook { get; set; } = AlertChannelOutcome.Delivered; + + public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + Outcomes.Add(outcome); + return Task.CompletedTask; + } + + public Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + Outcomes.Add(outcome); + var attempted = !outcome.Muted; + var fanout = new EmailFanoutResult( + EmailOutcome: AlertChannelOutcome.NotAttempted, + SendError: null, + WebhookOutcome: attempted ? Webhook : AlertChannelOutcome.NotAttempted, + WebhookSendError: attempted && Webhook == AlertChannelOutcome.Failed ? "Slack: 500 Internal Server Error" : null, + AnyChannelConfigured: true); + return Task.FromResult(AlertDelivery.FromFanout(fanout, outcome.Muted, trayChannelPresent: false)); + } + } + + /// The pre-#3580 shape: records, never reports — what every other suite's fake is, and what + /// Lite's deliverer is. The report member is REQUIRED on the seam (CONTRIBUTING, Two-Store Parity), so + /// "never reports" is written down here as an explicit null rather than inherited from a default. + private sealed class SilentDeliverer : IAlertDeliverer + { + public List Outcomes { get; } = new(); + + public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + Outcomes.Add(outcome); + return Task.CompletedTask; + } + + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } + } + + private sealed class RecordingHistoryStore : IAlertHistoryStore + { + public Task RecordAlertAsync(AlertHistoryRecord record) => Task.CompletedTask; + + public Task GetLastEmailSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => + Task.FromResult(null); + + public Task GetLastWebhookSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => + Task.FromResult(null); + + public Task GetLastAlertTimeAsync(string serverId, string metricName, string? dedupKey = null) => + Task.FromResult(null); + } + + /// The store between two "processes": a dictionary keyed the way the table is, with switches + /// to fault either half so the fallback path is the REAL one and not a mirror of it. + private sealed class MemoryStampStore : ISelfAlertDeliveryStampStore + { + public Dictionary Stamps { get; } = new(StringComparer.Ordinal); + public bool ThrowOnRead { get; set; } + public bool ThrowOnWrite { get; set; } + public int Reads { get; private set; } + public int Writes { get; private set; } + + public Task GetDeliveredAtUtcAsync(string stateKey, CancellationToken cancellationToken) + { + Reads++; + if (ThrowOnRead) + { + throw new InvalidOperationException("stamp read: store unreachable"); + } + + return Task.FromResult(Stamps.TryGetValue(stateKey, out var at) ? at : (DateTime?)null); + } + + public Task RecordDeliveredAtUtcAsync(string stateKey, DateTime deliveredAtUtc, CancellationToken cancellationToken) + { + Writes++; + if (ThrowOnWrite) + { + throw new InvalidOperationException("stamp write: store unreachable"); + } + + Stamps[stateKey] = deliveredAtUtc; + return Task.CompletedTask; + } + } + + /// One "process": the product's own settings over a default config, a deliverer, a clock, a + /// logger, a counter — and the SHARED stamp store handed in, so two harnesses over one store are two + /// processes over one database. + private sealed class Harness + { + public DarlingConfig Config { get; } = new(); + public IAlertDeliverer Deliverer { get; } + public MemoryStampStore? Stamps { get; } + public CapturingTestLogger Log { get; } = new(); + public AlertReadFailureCounter ReadFailures { get; } = new(); + public bool Muted { get; set; } + public DateTime Now { get; set; } = Day; + + public Harness(MemoryStampStore? stamps, IAlertDeliverer? deliverer = null) + { + Stamps = stamps; + Deliverer = deliverer ?? new ReportingDeliverer(); + } + + public List Outcomes => Deliverer switch + { + ReportingDeliverer r => r.Outcomes, + SilentDeliverer s => s.Outcomes, + _ => throw new InvalidOperationException("unknown deliverer"), + }; + + public DarlingSelfAlertEvaluator Build() => new( + new DarlingAlertSettings(Config), Deliverer, new RecordingHistoryStore(), _ => Muted, + logger: Log, utcNow: () => Now, readFailures: ReadFailures, deliveryStamps: Stamps); + } + + /* ---------------- the two documents, driven through their apply seams ---------------- */ + + /// One daily document as this suite drives it: its stamp key, its interval, and an apply + /// that always has something to say (a reportable fixture), so the only thing deciding whether a + /// delivery happens is the gate under test. + private sealed record Document(string Name, string StampKey, TimeSpan Interval, Func ApplyAsync); + + /* The digest suite's measured #3440 fixture, named-argument for named-argument, so the row is one the + shipped read could actually return and a member reorder is a compile error here rather than a + silently transposed figure. */ + private static readonly DarlingCollectorCostReader.CostMover[] OneMover = + { + new(ServerId: 1, ServerName: "pm-server-1", CollectorName: "query_store", LatestRuns: 2, + LatestWorstMs: 17_548, LatestMsPerRun: 17_548.0, BaselineMsPerRun: 6_477.0, + BaselineP95MsPerRun: 17_935.0, BaselineWorstDayMsPerRun: 17_935.0, BaselineDays: 13, + EligiblePairs: 177), + }; + + private static readonly DarlingCollectorCostReader.CollectorCostSummaryRow[] OneCensusRow = + { + new(CollectorName: "query_stats", RunCount: 43_891, TotalSqlMs: 61_724_459, MaxSqlMs: 88_561, + TotalStorageMs: 0, TotalRows: 0, ServerCount: 43), + }; + + private static FleetSweepRun[] OneTransitionDay(DateTime now) => new[] + { + new FleetSweepRun( + 1, now.AddHours(-2), now.AddHours(-3), now.AddHours(-2), null, true, 3, 3, true, + "{\"alive\":true}", + JsonSerializer.Serialize(new + { + changes = new { band_transitions = new[] { new { server = "pm-server-1", from = "Healthy", to = "Critical", reason = "deadlocks in span" } } }, + watch = new { opened = Array.Empty(), closed = Array.Empty() }, + fleet = new { bands = new Dictionary { ["Critical"] = 1, ["Healthy"] = 2 } }, + })), + }; + + private static readonly Dictionary Names = new() { [1] = "pm-server-1", [2] = "pm-server-2", [3] = "pm-server-3" }; + + private const string Digest = "collector-cost digest"; + private const string Rollup = "fleet-sweep rollup"; + + /// The theory rows are the documents' NAMES (serializable, so each is its own test case in + /// the runner) and resolve to a driver here. The names double as the {Document} the evaluator's + /// warnings name, which the fault pins assert. + private static Document For(string name) => name switch + { + Digest => new Document( + Digest, + PgSelfAlertDeliveryStampStore.CostDigestStateKey, + DarlingSelfAlertEvaluator.CollectorCostDigestInterval, + (e, _) => e.ApplyCollectorCostDigestAsync(OneMover, OneCensusRow, Ct)), + Rollup => new Document( + Rollup, + PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, + DarlingSelfAlertEvaluator.FleetSweepRollupInterval, + (e, now) => e.ApplyFleetSweepRollupAsync( + OneTransitionDay(now), Array.Empty(), Names, + now - DarlingSelfAlertEvaluator.FleetSweepRollupInterval, now, Ct)), + _ => throw new ArgumentOutOfRangeException(nameof(name), name, "not a daily document this suite knows"), + }; + + /* ---------------- the install-night pair ---------------- */ + + /// + /// The six re-announcements, retired: a document delivered by one process is NOT delivered again by a + /// fresh process an hour later. The stamp the first process wrote is the first process's clock at the + /// fire, Kind Utc; the second process — empty dictionaries, same store — reads it, gates, and never + /// reaches its deliverer. Past the interval the fresh process delivers, because the stamp is old, not + /// because it is fresh. + /// + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task ADeliveredDocument_IsNotReDeliveredByANewProcess_InsideItsInterval(string document) + { + var doc = For(document); + var store = new MemoryStampStore(); + + var first = new Harness(store); + await doc.ApplyAsync(first.Build(), first.Now); + Assert.Single(first.Outcomes); + Assert.True(store.Stamps.TryGetValue(doc.StampKey, out var stamped), "no delivery stamp was written"); + Assert.Equal(first.Now, stamped); + Assert.Equal(DateTimeKind.Utc, stamped.Kind); + + /* The restart: a new evaluator, one hour on, same store. */ + var second = new Harness(store) { Now = first.Now.AddHours(1) }; + var e2 = second.Build(); + await doc.ApplyAsync(e2, second.Now); + Assert.Empty(second.Outcomes); + + /* And the same instance keeps gating for the rest of the day without asking the store again — + the fast path: one read to learn the stamp, then memory. */ + var readsAfterFirstAsk = store.Reads; + second.Now = first.Now.Add(doc.Interval).AddMinutes(-1); + await doc.ApplyAsync(e2, second.Now); + Assert.Empty(second.Outcomes); + Assert.Equal(readsAfterFirstAsk, store.Reads); + + /* Past the interval the fresh process delivers — and re-stamps at ITS clock, without asking the + store again: one writer per store, so once memory is seeded the store cannot know more than it. */ + second.Now = first.Now.Add(doc.Interval).AddMinutes(1); + await doc.ApplyAsync(e2, second.Now); + Assert.Single(second.Outcomes); + Assert.Equal(second.Now, store.Stamps[doc.StampKey]); + Assert.Equal(2, store.Reads); + } + + /// + /// The recovery the install night showed working, kept by construction: a delivery the deliverer + /// reports FAILED writes no stamp, so the next tick retries — in the same process (the memory gate was + /// not set either) and in a fresh one. When the retry lands, the stamp is written at the retry's + /// clock, and the document is quiet from there. + /// + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AFailedDelivery_WritesNoStamp_SoTheNextTick_RestartOrNot_Retries(string document) + { + var doc = For(document); + var store = new MemoryStampStore(); + var deliverer = new ReportingDeliverer { Webhook = AlertChannelOutcome.Failed }; + + var first = new Harness(store, deliverer); + var e1 = first.Build(); + await doc.ApplyAsync(e1, first.Now); + Assert.Single(first.Outcomes); + Assert.Empty(store.Stamps); + Assert.Contains("delivery failed", first.Log.Joined, StringComparison.Ordinal); + Assert.Contains("Slack: 500", first.Log.Joined, StringComparison.Ordinal); + + /* Same process, next hourly tick: retried, still failing, still no stamp — and the store is not + asked again: "no row" was cached on the first tick, and a store this process alone writes cannot + have gained a row since. */ + first.Now = first.Now.AddHours(1); + await doc.ApplyAsync(e1, first.Now); + Assert.Equal(2, first.Outcomes.Count); + Assert.Empty(store.Stamps); + Assert.Equal(1, store.Reads); + + /* The restart, channel repaired: a fresh process asks once, retries and lands, and NOW the stamp exists. */ + deliverer.Webhook = AlertChannelOutcome.Delivered; + var second = new Harness(store, deliverer) { Now = first.Now.AddHours(1) }; + var e2 = second.Build(); + await doc.ApplyAsync(e2, second.Now); + Assert.Equal(3, deliverer.Outcomes.Count); + Assert.Equal(second.Now, store.Stamps[doc.StampKey]); + Assert.Equal(2, store.Reads); + + /* And from there, quiet — in this process and in the next. */ + second.Now = second.Now.AddHours(1); + await doc.ApplyAsync(e2, second.Now); + var third = new Harness(store, deliverer) { Now = second.Now.AddHours(1) }; + await doc.ApplyAsync(third.Build(), third.Now); + Assert.Equal(3, deliverer.Outcomes.Count); + Assert.Equal(3, store.Reads); + } + + /* ---------------- the stamp's age is the whole test ---------------- */ + + /// A stamp OLDER than the interval gates nothing — a fresh process delivers and overwrites it; + /// one a minute inside the interval gates. The gate reads the stamp's age, not its presence. + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AStampsAge_DecidesTheGate_NotItsPresence(string document) + { + var doc = For(document); + var stale = new MemoryStampStore(); + stale.Stamps[doc.StampKey] = Day - doc.Interval - TimeSpan.FromMinutes(1); + var h1 = new Harness(stale); + await doc.ApplyAsync(h1.Build(), h1.Now); + Assert.Single(h1.Outcomes); + Assert.Equal(Day, stale.Stamps[doc.StampKey]); + + var fresh = new MemoryStampStore(); + fresh.Stamps[doc.StampKey] = Day - doc.Interval + TimeSpan.FromMinutes(1); + var h2 = new Harness(fresh); + await doc.ApplyAsync(h2.Build(), h2.Now); + Assert.Empty(h2.Outcomes); + Assert.Equal(0, fresh.Writes); + } + + /* ---------------- store faults fall open to memory, once, loudly ---------------- */ + + /// + /// A stamp READ that throws does not silence the document and does not spam it: the tick falls back + /// to the process-memory gate (empty in a fresh process, so it delivers once), the next tick in the + /// same process is gated by memory, a warning names the document and the fallback, and the read is + /// counted into #3013's swallowed-read census under a name that says which document could not ask. + /// The write that follows the delivery is attempted regardless — a store whose read failed may well + /// take the write, and if it does the NEXT process is spared the re-announcement. + /// + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AStampReadFault_FallsBackToProcessMemory_DeliversOnce_AndWarns(string document) + { + var doc = For(document); + var store = new MemoryStampStore { ThrowOnRead = true }; + var h = new Harness(store); + var e = h.Build(); + + await doc.ApplyAsync(e, h.Now); + Assert.Single(h.Outcomes); + Assert.Matches(new Regex(@"Warning: .*delivery stamp could not be read"), h.Log.Joined); + Assert.Contains(doc.Name, h.Log.Joined, StringComparison.Ordinal); + Assert.Equal(1, h.ReadFailures.ReadInstance().ReadFailures); + /* One read name for both documents — the #3013 census keys a counted site on a literal, and the + gate is one site; the log line above is where the document is named. */ + Assert.Equal("daily-document delivery-stamp self-alert", h.ReadFailures.ReadInstance().LastFailureRead); + + /* The write still happened, so a store that only failed to READ has the stamp for the next process. */ + Assert.Equal(1, store.Writes); + Assert.Equal(h.Now, store.Stamps[doc.StampKey]); + + /* Same process, next tick: memory gates it; the store is not asked again (memory answered). */ + h.Now = h.Now.AddHours(1); + await doc.ApplyAsync(e, h.Now); + Assert.Single(h.Outcomes); + Assert.Equal(1, h.ReadFailures.ReadInstance().ReadFailures); + Assert.Equal(1, store.Reads); + } + + /// + /// The fault is met ONCE per process, not once per gate consult. The evaluate half's pre-check and the + /// apply half's own check both run on one tick, seconds apart; a gate that re-asked the store after a + /// fault would log the same failure twice and count it twice in #3013's census on every tick the fault + /// persisted. Staged with a delivery that ALSO fails, so nothing but the cached "asked, nothing known" + /// sentinel can be what stops the second consult from reaching the store — a successful delivery would + /// have populated memory on its own and hidden the difference. + /// + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AStampReadFault_IsMetOnce_PerProcess_EvenWhenNothingLands(string document) + { + var doc = For(document); + var store = new MemoryStampStore { ThrowOnRead = true, ThrowOnWrite = true }; + var deliverer = new ReportingDeliverer { Webhook = AlertChannelOutcome.Failed }; + var h = new Harness(store, deliverer); + var e = h.Build(); + + /* Two consults on "one tick" (the evaluate half, then the apply half): one store read, one + warning, one count — and the document is attempted both times, because nothing is known to have + been delivered and the memory gate is open. */ + await doc.ApplyAsync(e, h.Now); + await doc.ApplyAsync(e, h.Now); + Assert.Equal(2, deliverer.Outcomes.Count); + Assert.Equal(1, store.Reads); + Assert.Equal(1, h.ReadFailures.ReadInstance().ReadFailures); + Assert.Single(Regex.Matches(h.Log.Joined, "delivery stamp could not be read")); + + /* The next hour: still nothing landed, still one read on record — the retry runs from memory. */ + h.Now = h.Now.AddHours(1); + await doc.ApplyAsync(e, h.Now); + Assert.Equal(3, deliverer.Outcomes.Count); + Assert.Equal(1, store.Reads); + Assert.Equal(1, h.ReadFailures.ReadInstance().ReadFailures); + + /* A fresh process meets the fault once more — per process is the unit. */ + var next = new Harness(store, deliverer) { Now = h.Now.AddHours(1) }; + await doc.ApplyAsync(next.Build(), next.Now); + Assert.Equal(2, store.Reads); + Assert.Equal(1, next.ReadFailures.ReadInstance().ReadFailures); + } + + /// A stamp WRITE that throws leaves the document delivered and the process gated — memory is + /// stamped before the store is asked — with a warning that says the next restart will re-announce. + /// Not counted: a write is not a condition read (the resolution-row precedent). + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AStampWriteFault_StillGatesThisProcess_AndWarns_Uncounted(string document) + { + var doc = For(document); + var store = new MemoryStampStore { ThrowOnWrite = true }; + var h = new Harness(store); + var e = h.Build(); + + await doc.ApplyAsync(e, h.Now); + Assert.Single(h.Outcomes); + Assert.Matches(new Regex(@"Warning: .*delivery stamp could not be written"), h.Log.Joined); + Assert.Equal(0, h.ReadFailures.ReadInstance().ReadFailures); + + h.Now = h.Now.AddHours(1); + await doc.ApplyAsync(e, h.Now); + Assert.Single(h.Outcomes); + + /* And, stated: a fresh process over this store WILL re-announce — the bounded cost of a store + that would not take the write, and the pre-#3580 posture exactly. */ + var next = new Harness(store) { Now = h.Now }; + await doc.ApplyAsync(next.Build(), next.Now); + Assert.Single(next.Outcomes); + } + + /* ---------------- what counts as delivered ---------------- */ + + /// A deliverer that does not REPORT — every pre-#3580 fake, Lite's deliverer — stamps: null is + /// "unreported", treated as every fire before #3580 was, and never read as "failed". A deliverer that + /// knows a send failed says so. + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AnUnreportedDelivery_Stamps(string document) + { + var doc = For(document); + var store = new MemoryStampStore(); + var h = new Harness(store, new SilentDeliverer()); + await doc.ApplyAsync(h.Build(), h.Now); + + Assert.Single(h.Outcomes); + Assert.Equal(h.Now, store.Stamps[doc.StampKey]); + } + + /// A MUTED delivery stamps: the mute rule chose the silence, the history row was written + /// flagged muted, and nothing is owed — retrying hourly would write 24 muted rows a day for a document + /// the operator asked not to see. + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task AMutedDelivery_Stamps_BecauseNothingIsOwed(string document) + { + var doc = For(document); + var store = new MemoryStampStore(); + var h = new Harness(store) { Muted = true }; + await doc.ApplyAsync(h.Build(), h.Now); + + var fired = Assert.Single(h.Outcomes); + Assert.True(fired.Muted); + Assert.Equal(h.Now, store.Stamps[doc.StampKey]); + } + + /// No stamp store at all is the pre-#3580 gate exactly — process memory only, so a fresh + /// evaluator re-delivers. Stated so the seam's default is a documented posture and not an accident; + /// production always passes the store. + [Theory] + [InlineData(Digest)] + [InlineData(Rollup)] + public async Task WithoutAStampStore_TheGateIsProcessMemoryOnly(string document) + { + var doc = For(document); + var first = new Harness(stamps: null); + var e1 = first.Build(); + await doc.ApplyAsync(e1, first.Now); + first.Now = first.Now.AddHours(1); + await doc.ApplyAsync(e1, first.Now); + Assert.Single(first.Outcomes); + + var second = new Harness(stamps: null) { Now = first.Now }; + await doc.ApplyAsync(second.Build(), second.Now); + Assert.Single(second.Outcomes); + } + + /* ---------------- the store's row identity ---------------- */ + + /// The stamp rows sit under the fleet sentinel BOTH existing fleet-scope writers already use + /// (the retention purge's run-record and the sweep's fleet watch items), and under an owner name that + /// is not a collector definition's — so no declared-key read and no per-database prune can reach them. + /// The two keys are distinct, and the SQL names the table with its schema. + [Fact] + public void TheStampRows_UseTheFleetSentinel_AndAnOwnerNameNoCollectorClaims() + { + Assert.Equal(FleetSweepStore.FleetScopeServerId, PgSelfAlertDeliveryStampStore.FleetServerId); + Assert.Equal(0, PgSelfAlertDeliveryStampStore.FleetServerId); + Assert.Equal("self_alert", PgSelfAlertDeliveryStampStore.StateCollectorName); + Assert.NotEqual(PgSelfAlertDeliveryStampStore.CostDigestStateKey, PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey); + + /* Not a collector definition's name, and not either of the two other worker-owned owner names + already in the table (the backfill worker's and the open-interval state's) — three owners, three + names, no row can be read as another's. */ + Assert.DoesNotContain( + PgSelfAlertDeliveryStampStore.StateCollectorName, + CollectorCatalog.All.Select(c => c.Name), + StringComparer.Ordinal); + Assert.NotEqual(QueryStoreBackfillState.StateCollectorName, PgSelfAlertDeliveryStampStore.StateCollectorName); + Assert.NotEqual(QueryStoreOpenIntervalState.StateCollectorName, PgSelfAlertDeliveryStampStore.StateCollectorName); + + Assert.Contains("collect.collector_state", PgSelfAlertDeliveryStampStore.GetSql, StringComparison.Ordinal); + Assert.Contains("collect.collector_state", PgSelfAlertDeliveryStampStore.UpsertSql, StringComparison.Ordinal); + Assert.Contains("ON CONFLICT (server_id, collector_name, state_key)", PgSelfAlertDeliveryStampStore.UpsertSql, StringComparison.Ordinal); + } + + /// The worker hands the evaluator the store-backed stamps — the seam's null default is for + /// harnesses, and a production evaluator built without it would be the six re-announcements back. + [Fact] + public void TheWorker_HandsTheEvaluatorTheStoreBackedStamps() + { + var worker = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs"); + var evaluatorBuild = worker.IndexOf("_selfAlerts = new DarlingSelfAlertEvaluator(", StringComparison.Ordinal); + Assert.True(evaluatorBuild >= 0, "#3580 pin: the worker no longer constructs DarlingSelfAlertEvaluator where this pin looks"); + var end = worker.IndexOf(");", evaluatorBuild, StringComparison.Ordinal); + Assert.Contains("deliveryStamps: new PgSelfAlertDeliveryStampStore(", worker[evaluatorBuild..end], StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/SelfDiskWarnGbFloorRungTests.cs b/Darling/Darling.Tests/SelfDiskWarnGbFloorRungTests.cs new file mode 100644 index 000000000..faa4cfd52 --- /dev/null +++ b/Darling/Darling.Tests/SelfDiskWarnGbFloorRungTests.cs @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Linq; +using System.Reflection; +using Npgsql; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// V126 / #3528: the Store Disk Pressure warning's GB floor moves onto config.config_alert_settings. +/// +/// The self-store warn condition was percent-only, its own comment calling a GB floor "a trivial +/// follow-up if an operator ever wants one" — and #3528's example is the want: 400 GB free on a 4 TB store +/// volume fired a CRITICAL "act now". The floor is an AND qualifier (the pvs_floor_gb composition, +/// deliberately not the target-volume pair's OR, whose GB dimension ADDS fires), so a large volume at a +/// low percent stays quiet until absolute free space is genuinely short; 0 removes the floor. +/// +/// The "I am the top rung" claims have moved ON to +/// (V127), the same handoff this file received from (V125) +/// and that file received from (V124). What stays here is +/// everything true of this rung wherever it sits in the ladder; what left is every claim that was really +/// about being NEWEST — keeping a copy of those would assert this rung is still the top, which is how the +/// NEXT rung's build goes red. +/// +public sealed class SelfDiskWarnGbFloorRungTests +{ + private const int RungVersion = 126; + private const int PreviousVersion = 125; + + /// This rung's sentinel ordinal in the viewer probe. No longer the last argument — V127 + /// appended its own — so this is a position within the signature rather than its end. + private const int ProbeOrdinal = 101; + + private const string FloorColumn = "self_disk_free_warn_gb"; + + /* ---- the rung ------------------------------------------------------------------------------------ */ + + [Fact] + public void TheRungIsRegisteredInADenseLadder() + { + var versions = PgMigrations.Scripts.Select(s => s.Version).ToList(); + + Assert.Equal( + "self-disk-warn-gb-floor", + PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name); + + Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version); + Assert.Equal(StorageVersion.SchemaVersion, versions.Max()); + + /* Not `RungVersion == SchemaVersion` any more: that asserted this rung is the newest, which + stopped being true when V127 landed. The invariant that outlives the handoff is that the + LADDER's top and the declared version agree, which the two lines above already say. */ + Assert.True(RungVersion < StorageVersion.SchemaVersion); + + Assert.Equal(versions.Distinct().OrderBy(v => v), versions); + } + + /// + /// The rung adds ONE column to the singleton settings row, schema-qualified, with the shipped constant + /// as its default. + /// + /// The DEFAULT is compared against DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb rather + /// than a literal (restated in the rung only because a rung is a SQL string), so a moved shipped + /// default cannot leave upgraded stores qualifying at a floor no surface reports. The default is + /// NON-ZERO on upgrade deliberately, unlike V122's knobs: their acceptance was "an untouched store + /// fires exactly where it did", while #3528's is that the untouched firing IS the defect — the issue's + /// own example is a default-configured store paging with 400 GB of runway. Any store volume at or + /// under 500 GB (floor ÷ warn percent) keeps the exact pre-#3528 percent behaviour. + /// + [Fact] + public void TheRungAddsTheColumn_SchemaQualified_WithTheShippedConstantAsDefault() + { + var rung = PgMigrations.Scripts.Single(s => s.Version == RungVersion).Sql; + + /* Schema-qualified for the reason every config rung is: the migrate session's search_path puts + collect first, so a bare name would resolve to the wrong schema (and the wrong ACL). */ + Assert.Equal(1, CountOf(rung, "ALTER TABLE config.config_alert_settings")); + Assert.DoesNotContain("ALTER TABLE config_alert_settings", rung, StringComparison.Ordinal); + + /* IF NOT EXISTS so re-running the ladder over a store that already has it is a no-op rather than + a 42701 that aborts the whole migration. integer, matching self_disk_free_warn_percent and the + low-disk GB columns on this same table — a whole-GB knob has no meaningful fractional part. */ + Assert.Equal(1, CountOf(rung, "ADD COLUMN IF NOT EXISTS")); + Assert.DoesNotContain("double precision", rung, StringComparison.Ordinal); + Assert.Contains( + $"ADD COLUMN IF NOT EXISTS {FloorColumn} integer NOT NULL DEFAULT " + + ((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb).ToString(CultureInfo.InvariantCulture) + ";", + rung, StringComparison.Ordinal); + + /* And the C# seed names the same figure, so a fresh file-plane config and an upgraded store row + agree without either citing the other. The constant is whole-valued by construction — the cast + in the assertion above must not be hiding a fractional shipped default. */ + Assert.Equal(DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb, (int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb); + Assert.Equal((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb, new AlertsConfig().SelfDiskFreeWarnGb); + + /* No reload beacon of its own: config_alert_settings already carries V17's statement-level + trg_bump_alert_settings, so a second trigger here would be a duplicate bump per write. */ + Assert.DoesNotContain("config_bump_version", rung, StringComparison.Ordinal); + + /* And no per-table GRANT: this table carries table-level grants with no column carve, which is + what every earlier knob rung on it says. */ + Assert.DoesNotContain("GRANT", rung, StringComparison.Ordinal); + } + + /* ---- the probe (three sites, top arm) ------------------------------------------------------------- */ + + /// + /// The viewer probe's three sites carry this rung's sentinel, and a store that stopped here maps to it. + /// + /// The probe asks the question, the caller reads the answer, the map has the parameter — three + /// sites, and a sentinel present at only some of them shifts every LATER ordinal onto the wrong column. + /// The top-arm claims (last argument, textual newest-first ordering) moved to + /// with V127. + /// + [Fact] + public void TheProbeCarriesThisRungsSentinel_AndAFullyMigratedStoreMapsToTheLaddersTop() + { + Assert.Contains($"column_name = '{FloorColumn}'", ViewerDataService.StoreSchemaProbeSql, StringComparison.Ordinal); + + var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); + Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal); + Assert.Contains("hasSelfDiskWarnGbFloor", viewer, StringComparison.Ordinal); + + Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion); + + var method = typeof(ViewerDataService) + .GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!; + var arity = method.GetParameters().Length; + + /* A position within the signature, not its end: `ProbeOrdinal == arity - 1` asserted this rung is + the NEWEST sentinel, which stopped being true the moment V127 appended its own. Strictly-less is + the form every other non-top rung's test here uses. */ + Assert.True(ProbeOrdinal < arity - 1); + + /* Every sentinel true = a fully-migrated store, which must map to exactly this version. Built by + reflection so the arity tracks the signature. */ + var all = Enumerable.Repeat((object)true, arity).ToArray(); + Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!); + + /* This rung's own arm answers for a store that stopped here. Expressed as "false above" rather than + as one named ordinal, so a rung landing on top of this one does not quietly turn this case into a + test of that rung. */ + var atThisRung = Enumerable.Range(0, arity).Select(i => (object)(i <= ProbeOrdinal)).ToArray(); + Assert.Equal(RungVersion, (int)method.Invoke(null, atThisRung)!); + + /* One rung behind: the same store WITHOUT this rung's sentinel reports the previous rung. */ + var behind = (object[])atThisRung.Clone(); + behind[ProbeOrdinal] = false; + Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!); + + /* And in the source, the arm sits ABOVE V125's — newest-first is the whole contract of that method. + It returns this rung's own literal now, not the build's version: the "returns + StorageVersion.SchemaVersion" half of the top-arm claim moved to V127's test with the top. */ + var v126 = viewer.IndexOf("if (hasSelfDiskWarnGbFloor)", StringComparison.Ordinal); + var v125 = viewer.IndexOf("if (hasCollectorScheduleDatabases)", StringComparison.Ordinal); + Assert.True(v126 >= 0, "the viewer has no V126 sentinel arm — a store that stopped here would map to 125"); + Assert.True(v125 >= 0, "the V125 arm is gone, so this pin is comparing against nothing"); + Assert.True(v126 < v125, "the V126 arm sits below V125's, so a V126 store maps one rung low"); + Assert.Contains( + "return " + RungVersion.ToString(CultureInfo.InvariantCulture) + ";", + viewer[v126..], StringComparison.Ordinal); + } + + /* ---- every settings-row surface handles the column ------------------------------------------------ */ + + /// + /// EVERY surface that reads or writes the settings row names the column — the viewer INCLUDED. The + /// wired lists drive ordinals or parameter positions, so a column added to one and not the others + /// re-maps reads and writes at once. + /// + /// The viewer's select/upsert was this rung's pinned abstinence, unlike every earlier knob + /// rung: the knob landed backend-first (store plane + the two MCP tools), and this assertion pinned + /// DoesNotContain until the viewer pass (#3563) wired the Settings window's box — the flip that + /// paragraph promised. What the abstinence protected still holds now that it is wired: the viewer's + /// explicit column lists mean a Save writes the floor rather than nulling it out. + /// + [Fact] + public void EverySettingsRowSurfaceNamesTheColumn_TheViewerIncluded() + { + var service = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "StoreConfigProvider.cs"); + var tools = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpAlertTools.cs"); + + /* The service must READ it, not merely select it — ApplyToConfig replaces config.Alerts wholesale, + so a selected-but-unread column resets the floor to the shipped default on every worker start. + Both halves, because one of the two being present is what an off-by-one produces. */ + Assert.Contains(FloorColumn, service, StringComparison.Ordinal); + Assert.Contains("SelfDiskFreeWarnGb = reader.GetInt32(", service, StringComparison.Ordinal); + + /* The MCP read names the column and the report/accept pair carries the wire key — readable AND + writable, because a read-only knob leaves the UPDATE in someone's runbook. */ + Assert.Contains(FloorColumn, DarlingAlertReader.AlertSettingsSelectSql, StringComparison.Ordinal); + Assert.Contains("disk_free_warn_gb = s.SelfDiskFreeWarnGb", tools, StringComparison.Ordinal); + Assert.Contains( + "case \"disk_free_warn_gb\": AddInt(\"self_disk_free_warn_gb\", n, \"self_alerts.disk_free_warn_gb\", 0, int.MaxValue); break;", + tools, StringComparison.Ordinal); + + /* The viewer pass (#3563): the viewer's select reads the column, its upsert WRITES it (or Save + silently drops whatever the box held), and its reader maps the appended ordinal. */ + var viewerSettings = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.AlertSettings.cs"); + Assert.Contains(FloorColumn, ViewerDataService.AlertSettingsSelectSql, StringComparison.Ordinal); + Assert.Contains( + $"{FloorColumn} = EXCLUDED.{FloorColumn}", ViewerDataService.AlertSettingsUpsertSql, StringComparison.Ordinal); + Assert.Contains("SelfDiskFreeWarnGb = reader.GetInt32(66)", viewerSettings, StringComparison.Ordinal); + } + + /// + /// The Settings window's box (#3563): prefilled from the row, saved through the same bound the MCP + /// writer accepts and the read-side clamp keeps (>= 0 — 0 removes the floor), following the + /// alerts master switch like every #2107 sibling, and Restore Defaults writes the shipped constant. + /// + /// The viewer restates the shipped default as a literal — in the row initializer and the + /// Restore Defaults button — because DarlingSelfAlertEvaluator lives on the Service assembly the + /// viewer does not reference (its percent sibling's "10" has the same shape). Both literals are pinned + /// equal to the constant here, the same equality the rung SQL's own default carries, so a moved shipped + /// default cannot leave the window handing out a floor no other surface reports. + /// + [Fact] + public void TheSettingsWindowBox_PrefillsSavesGatesAndRestores_AtTheSharedBoundAndDefault() + { + var window = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Viewer", "SettingsWindow.xaml.cs"); + var xaml = RepoFile.ReadRepoFile( + "Darling", "PerformanceMonitor.Darling.Viewer", "SettingsWindow.xaml"); + + /* The box exists, one knob over from its percent sibling. */ + Assert.Contains("x:Name=\"AlertSelfDiskWarnGbBox\"", xaml, StringComparison.Ordinal); + + /* Prefill, save gate (the [0, int.MaxValue) bound's floor), and the master-switch follow. */ + Assert.Contains("AlertSelfDiskWarnGbBox.Text = r.SelfDiskFreeWarnGb.ToString(", window, StringComparison.Ordinal); + Assert.Contains( + "if (int.TryParse(AlertSelfDiskWarnGbBox.Text, out var selfDiskGb) && selfDiskGb >= 0)", + window, StringComparison.Ordinal); + Assert.Contains("row.SelfDiskFreeWarnGb = selfDiskGb;", window, StringComparison.Ordinal); + Assert.Contains("AlertSelfDiskWarnGbBox.IsEnabled = enabled;", window, StringComparison.Ordinal); + + /* Restore Defaults writes the shipped figure, and the viewer row seeds it — both as literals + pinned equal to the constant. */ + var shipped = ((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb).ToString(CultureInfo.InvariantCulture); + Assert.Contains($"AlertSelfDiskWarnGbBox.Text = \"{shipped}\";", window, StringComparison.Ordinal); + Assert.Equal((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb, AlertSettingsRow.Defaults().SelfDiskFreeWarnGb); + Assert.Equal((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb, new AlertsConfig().SelfDiskFreeWarnGb); + } + + /// + /// The viewer row round-trips through the bind at the APPENDED ordinal — the four-parallel-sequences + /// trap (the column list, the upsert's $N placeholders, the bind order, and the reader ordinals), + /// held for the one sequence no SQL text pin can see: the bind. The V124 rung's shape, one knob on. + /// + [Fact] + public void TheViewerRow_RoundTripsThroughTheBind_AtTheAppendedOrdinal() + { + var bind = typeof(ViewerDataService) + .GetMethod("BindAlertSettings", BindingFlags.NonPublic | BindingFlags.Static)!; + + var row = AlertSettingsRow.Defaults(); + /* Deliberately NOT the shipped 50 — a bind that dropped the column and fell back to the default + would otherwise still present the right value at the position. 75 is the MCP round-trip test's + sample, for the same reason. */ + row.SelfDiskFreeWarnGb = 75; + + using var command = new NpgsqlCommand(); + bind.Invoke(null, new object[] { command, row }); + + /* The bind supplies exactly as many parameters as the upsert's highest placeholder. */ + var highestPlaceholder = System.Text.RegularExpressions.Regex + .Matches(ViewerDataService.AlertSettingsUpsertSql, @"\$(\d+)") + .Select(m => int.Parse(m.Groups[1].Value, CultureInfo.InvariantCulture)) + .Max(); + Assert.Equal(command.Parameters.Count, highestPlaceholder); + + /* The new column rides at the END — appended, the rule every knob rung on this table follows, + so every earlier ordinal keeps its column. */ + Assert.Equal(75, Assert.IsType>(command.Parameters[^1]).TypedValue); + } + + /* ---- the seam reaches the gate -------------------------------------------------------------------- */ + + /// + /// The settings adapter defaults to the shipped constant and clamps a hand-edited store value at the + /// 0 floor — the raw-in/clamped-out split every knob on this table uses, with 0 IN range because it + /// removes the floor (the pvs_floor_gb reading) rather than being nonsense. The write bound in + /// is the same + /// [0, int.MaxValue], so no accepted value is one this clamp rewrites. + /// + [Fact] + public void TheSettingsSeamDefaultsToTheConstant_AndClampsAtZero() + { + var config = new DarlingConfig(); + var settings = new DarlingAlertSettings(config); + + Assert.Equal((int)DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb, settings.SelfDiskFreeWarnGb); + + config.Alerts.SelfDiskFreeWarnGb = -5; + Assert.Equal(0, settings.SelfDiskFreeWarnGb); + + config.Alerts.SelfDiskFreeWarnGb = 0; + Assert.Equal(0, settings.SelfDiskFreeWarnGb); + + config.Alerts.SelfDiskFreeWarnGb = 400; + Assert.Equal(400, settings.SelfDiskFreeWarnGb); + } + + private static int CountOf(string haystack, string needle) + { + var count = 0; + for (var at = haystack.IndexOf(needle, StringComparison.Ordinal); + at >= 0; + at = haystack.IndexOf(needle, at + needle.Length, StringComparison.Ordinal)) + { + count++; + } + + return count; + } +} diff --git a/Darling/Darling.Tests/SerialLoopStoreSizeSourceTests.cs b/Darling/Darling.Tests/SerialLoopStoreSizeSourceTests.cs index ae1a7e61f..d5fd096bc 100644 --- a/Darling/Darling.Tests/SerialLoopStoreSizeSourceTests.cs +++ b/Darling/Darling.Tests/SerialLoopStoreSizeSourceTests.cs @@ -376,17 +376,18 @@ const on its right-hand side. Over stripped source a comparison against a litera Assert.True( literal.Count == 0, $"{literal.Count} ObjectKind comparison(s) in DarlingMcpStoreMetricsTools.cs compare against " - + "something other than StoreSelfMetrics.StoreObjectKind. The kind is one const with six " + + "something other than a StoreSelfMetrics.*ObjectKind const. Every kind is one const with several " + "consumers because a reader filtering on a kind the writer stopped writing returns zero " + $"rows rather than erroring: {string.Join(", ", literal)}"); } /// /// Every .ObjectKind == / != comparison in : how many there are, - /// and the ones whose right-hand side is not - /// . Read over STRIPPED source, where a literal - /// right-hand side survives as blanks — so the next code token after the operator is not the const, - /// and the site is reported. + /// and the ones whose right-hand side is not one of the StoreSelfMetrics.*ObjectKind constants + /// ( and, since #3582, its siblings for every other kind + /// the sweep writes — the invariant is "no retyped literal", not "only the store kind"). Read over + /// STRIPPED source, where a literal right-hand side survives as blanks — so the next code token after + /// the operator is not a const of that shape, and the site is reported. /// private static (int Compared, List NotTheConst) ObjectKindComparisons(string source) { @@ -400,7 +401,7 @@ private static (int Compared, List NotTheConst) ObjectKindComparisons(st compared++; var rest = code[(m.Index + m.Length)..]; - if (!rest.TrimStart().StartsWith("StoreSelfMetrics.StoreObjectKind", StringComparison.Ordinal)) + if (!Regex.IsMatch(rest.TrimStart(), @"^StoreSelfMetrics\.[A-Za-z]+ObjectKind\b", RegexOptions.CultureInvariant)) { var line = code.Take(m.Index).Count(c => c == '\n') + 1; offenders.Add($"line {line}"); @@ -484,6 +485,10 @@ public void TheScanner_ReadsQueriesAndNotProseAboutThem(string source, bool expe [Theory] [InlineData("if (r.ObjectKind == StoreSelfMetrics.StoreObjectKind) { }\n", 1, 0)] [InlineData("if (p.ObjectKind != StoreSelfMetrics.StoreObjectKind) { }\n", 1, 0)] + /* #3582: the sibling kind consts pass; a StoreSelfMetrics member that is NOT a kind const does not. */ + [InlineData("if (r.ObjectKind == StoreSelfMetrics.ContinuousAggregateObjectKind) { }\n", 1, 0)] + [InlineData("if (r.ObjectKind != StoreSelfMetrics.JobHistoryObjectKind) { }\n", 1, 0)] + [InlineData("if (r.ObjectKind == StoreSelfMetrics.OtherObjectName) { }\n", 1, 1)] [InlineData("if (r.ObjectKind == \"store\") { }\n", 1, 1)] [InlineData("if (r.ObjectKind != \"store\") { }\n", 1, 1)] [InlineData("/* r.ObjectKind == \"store\" is what this used to do. */\n", 0, 0)] diff --git a/Darling/Darling.Tests/ServerHealthClassifierTests.cs b/Darling/Darling.Tests/ServerHealthClassifierTests.cs index 237c786b8..e1a6028fc 100644 --- a/Darling/Darling.Tests/ServerHealthClassifierTests.cs +++ b/Darling/Darling.Tests/ServerHealthClassifierTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Linq; using PerformanceMonitor.Common; using Xunit; @@ -71,35 +72,155 @@ public void CpuSeverity_NoData_IsUnknown() => public void MemorySeverity_CriticalOnPressure(bool pressure, HealthSeverity expected) => Assert.Equal(expected, ServerHealthClassifier.MemorySeverity(pressure)); + private static readonly TimeSpan Hour = TimeSpan.FromHours(1); + private static readonly TimeSpan Day = TimeSpan.FromHours(24); + private static readonly TimeSpan Week = TimeSpan.FromHours(168); + + /* ── the blocking RATE band (#3539 A3) ── */ + + /// + /// The three arms over a one-hour window, where a count and a per-hour rate coincide: the 60 s wait arm + /// is Critical; the count arm is Critical at 20/hr and Warning at 5/hr; the 10 s wait arm is Warning; + /// the quiet mode (1–4 reports) is Healthy by count. + /// [Theory] [InlineData(0, 0.0, HealthSeverity.Healthy)] - [InlineData(1, 0.0, HealthSeverity.Warning)] // any blocking is Warning - [InlineData(2, 0.0, HealthSeverity.Warning)] - [InlineData(5, 0.0, HealthSeverity.Critical)] // >= 5 events Critical - [InlineData(0, 10.0, HealthSeverity.Warning)] // >= 10s max wait Warning - [InlineData(0, 60.0, HealthSeverity.Critical)] // >= 60s max wait Critical - public void BlockingSeverity_BandsOnCountAndWait(int count, double maxSeconds, HealthSeverity expected) => - Assert.Equal(expected, ServerHealthClassifier.BlockingSeverity(count, maxSeconds)); + [InlineData(1, 0.0, HealthSeverity.Healthy)] // the quiet mode: 51 of 88 measured active hours hold 1–4 + [InlineData(4, 0.0, HealthSeverity.Healthy)] + [InlineData(5, 0.0, HealthSeverity.Warning)] // 5/hr, the top of the quiet mode + [InlineData(19, 0.0, HealthSeverity.Warning)] // inside the measured trough + [InlineData(20, 0.0, HealthSeverity.Critical)] // 20/hr, the lower edge of the storm mode + [InlineData(232, 0.0, HealthSeverity.Critical)] // the worst measured hour + [InlineData(1, 10.0, HealthSeverity.Warning)] // the 10 s wait arm, whatever the rate + [InlineData(1, 60.0, HealthSeverity.Critical)] // the 60 s wait arm, whatever the rate + public void BlockingSeverity_BandsOnRateAndWait_OverAnHour(int count, double maxSeconds, HealthSeverity expected) => + Assert.Equal(expected, ServerHealthClassifier.BlockingSeverity(count, maxSeconds, Hour)); + + /// + /// #3539 A3's headline: the SAME per-hour rate bands identically over an hour, a day and a week, so the + /// count no longer means something different on every surface that windows it. Counts are + /// integer-rate-times-whole-hours so the asserted rate is exactly the one the band sees. + /// + [Theory] + [InlineData(4, HealthSeverity.Healthy)] + [InlineData(5, HealthSeverity.Warning)] + [InlineData(19, HealthSeverity.Warning)] + [InlineData(20, HealthSeverity.Critical)] + public void BlockingSeverity_TheSameRate_BandsTheSame_OverAnHourADayAndAWeek(long ratePerHour, HealthSeverity expected) + { + Assert.Equal(expected, ServerHealthClassifier.BlockingSeverity(ratePerHour, 0.0, Hour)); + Assert.Equal(expected, ServerHealthClassifier.BlockingSeverity(ratePerHour * 24, 0.0, Day)); + Assert.Equal(expected, ServerHealthClassifier.BlockingSeverity(ratePerHour * 168, 0.0, Week)); + } + + /// + /// The defect as filed: five reports were Critical at a 168-hour read and Healthy at a one-hour read + /// of the same server under count >= 5 → Critical. Now five reports in a WEEK is 0.03/hr and + /// Healthy by count, five in an HOUR is the Warning tier, and the same count over the two windows bands + /// differently — which a count trigger cannot do at all, so this is the pin that goes red on a revert + /// to counting. + /// + [Fact] + public void BlockingSeverity_FiveReportsAWeek_IsNoLongerCritical() + { + Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(5, 0.0, Week)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(5, 0.0, Hour)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(20, 0.0, Hour)); + Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(20, 0.0, Day)); // 0.8/hr + } + + /// The wait arms are per-event magnitude claims and do NOT normalise: a 60-second block is + /// Critical over a week exactly as over an hour, and a 10-second one is Warning — the rate has no say. + [Fact] + public void BlockingSeverity_TheWaitArms_AreRateIndependent() + { + foreach (var window in new[] { Hour, Day, Week }) + { + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 60.0, window)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(1, 10.0, window)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(1, 59.9, window)); + } + } /// - /// #3368: the count ladder has ONE Warning arm, so every count that is not Critical and not zero lands - /// on the same severity. A second arm at >= 2 existed above it returning the same Warning and - /// decided nothing. - /// - /// Stated as a PROPERTY over the whole non-Critical range rather than as the old pair of - /// InlineData rows: the rows above happened to cover 1 and 2 and would have kept passing if a - /// third indistinguishable arm were added, where this cannot. + /// The unrateable arm — a window under an hour or undeclared — fails away from Healthy and never into + /// Critical by count (#3368's rule, one metric over): the wait arms still decide (they need no + /// denominator), past them a non-zero count reads Warning even at 10,000 reports in 15 minutes, and a + /// zero count reads Unknown, not Healthy. /// [Fact] - public void BlockingSeverity_HasOneWarningArmOnTheCount() + public void BlockingSeverity_ASubHourOrUndeclaredWindow_FallsToWarning_NeverCriticalByCount() { - for (var count = 1; count < 5; count++) + foreach (var window in new[] { default, TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(59) }) { - Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(count, 0.0)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(1, 0.0, window)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(10_000, 0.0, window)); + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(0, 0.0, window)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(1, 60.0, window)); + Assert.Null(ServerHealthClassifier.BlockingRatePerHour(1, window)); } - Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(0, 0.0)); - Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(5, 0.0)); + Assert.Equal(1.0, ServerHealthClassifier.BlockingRatePerHour(24, Day)); + Assert.Equal(ServerHealthThresholds.DeadlockRateMinimumWindow, ServerHealthThresholds.BlockingRateMinimumWindow); + } + + /// + /// The tiers sit where the 14-day measurement puts them (MEASUREMENTS for #3539, 43 SQL Server + /// primaries, 14,448 server-hours): 88 active hours, of which 51 hold 1–4 reports, 5 hold 5–10, 9 hold + /// 11–19 and 23 hold 20 or more. Restated here as the count-per-hour histogram so a moved constant has + /// to argue with the distribution rather than with a literal. + /// + [Fact] + public void BlockingTiers_SitAtTheTopOfTheQuietMode_AndTheFootOfTheStormMode() + { + Assert.Equal(5.0, ServerHealthThresholds.BlockingWarnPerHour); + Assert.Equal(20.0, ServerHealthThresholds.BlockingCriticalPerHour); + Assert.Equal(60.0, ServerHealthThresholds.BlockingCriticalWaitSeconds); + Assert.Equal(10.0, ServerHealthThresholds.BlockingWarnWaitSeconds); + + /* (reports in the hour, server-hours measured at that count) — the histogram's bands, at their + upper edges. The quiet mode ends at 4 and bands Healthy by count; the storm mode begins at 20. */ + var quietMode = new (int Count, int Hours)[] { (1, 35), (2, 11), (4, 5) }; + foreach (var (count, _) in quietMode) + { + Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(count, 0.0, Hour)); + } + + Assert.Equal(51, quietMode.Sum(b => b.Hours)); // of 88 active hours + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(10, 0.0, Hour)); // 5–10: 5 hours + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(19, 0.0, Hour)); // 11–19: 9 hours + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(20, 0.0, Hour)); // 20+: 23 hours + } + + /* ── the collector SHARE band (#3539 A8d) ── */ + + /// + /// One failing of forty and forty of forty no longer band alike: any FAILING collector is Warning, and a + /// FAILING share past the collector-health classifier's own 20% bar is Critical. Nothing failing is + /// Healthy when collectors were banded, and Unknown when none were (#3539 A6 — no collection to call + /// clean); no denominator with something failing is Warning and never Critical — a share nobody + /// computed cannot escalate. + /// + [Theory] + [InlineData(0, 40, HealthSeverity.Healthy)] + [InlineData(0, 1, HealthSeverity.Healthy)] + [InlineData(0, 0, HealthSeverity.Unknown)] // nothing banded: not a clean collection, an unmeasured one + [InlineData(0, -1, HealthSeverity.Unknown)] + [InlineData(1, 40, HealthSeverity.Warning)] // 2.5% + [InlineData(8, 40, HealthSeverity.Warning)] // exactly 20% — the bar is strict, as the classifier's is + [InlineData(9, 40, HealthSeverity.Critical)] // 22.5% + [InlineData(40, 40, HealthSeverity.Critical)] + [InlineData(1, 0, HealthSeverity.Warning)] // no denominator declared + [InlineData(40, 0, HealthSeverity.Warning)] + public void CollectorSeverity_GradesOnTheFailingShare(int failing, int total, HealthSeverity expected) => + Assert.Equal(expected, ServerHealthClassifier.CollectorSeverity(failing, total)); + + [Fact] + public void CollectorSeverity_TheBar_IsTheCollectorHealthClassifiersOwn() + { + Assert.Equal(20.0, CollectorHealthClassifier.WarningFailureRatePercent); + Assert.Equal(22.5, ServerHealthClassifier.FailingCollectorSharePercent(9, 40)); + Assert.Null(ServerHealthClassifier.FailingCollectorSharePercent(9, 0)); } /// @@ -160,12 +281,6 @@ public void ThreadsSeverity_LowAvailable_IsWarning() => public void ThreadsSeverity_Ample_IsHealthy() => Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.ThreadsSeverity(512, 400, 0, 0)); - [Theory] - [InlineData(0, HealthSeverity.Healthy)] - [InlineData(1, HealthSeverity.Warning)] - public void CollectorSeverity_AnyFailingWarning(int failing, HealthSeverity expected) => - Assert.Equal(expected, ServerHealthClassifier.CollectorSeverity(failing)); - /* ── overall reduce ── */ [Fact] @@ -193,11 +308,116 @@ public void OverallMetricSeverity_WarningWhenNoCritical() [Fact] public void OverallMetricSeverity_AllCalm_IsHealthy_UnknownNeverEscalates() { - // No CPU snapshot (Unknown) and no threads snapshot (Unknown) must not escalate the card. - var m = new ServerHealthMetrics { CpuPercentForAlert = null, TotalThreads = null }; + // No CPU snapshot (Unknown) and no threads snapshot (Unknown) must not escalate the card. Memory + // is measured and calm, so the fold has one real reading to answer Healthy from (#3539 A6: with + // NO reading it answers Unknown, pinned separately below). + var m = new ServerHealthMetrics { CpuPercentForAlert = null, TotalThreads = null, HasMemoryPressure = false }; Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.OverallMetricSeverity(m)); } + /// + /// #3539 A6: a bundle on which NOT ONE metric was measured folds to Unknown, not Healthy, and the fleet + /// band reads it as Warning — the never-collected server's band — so it leaves the healthy mass. The + /// pre-fix fold answered Healthy here ("0 of 6 measured" was the only tell), which put an online server + /// nothing had banded yet in healthy_count. One measured reading is enough to lift the fold off + /// Unknown, which is what keeps #3528's partially-measured Healthy exactly where it was. + /// + [Fact] + public void OverallMetricSeverity_NothingMeasured_IsUnknown_AndBandsWarning() + { + var nothing = new ServerHealthMetrics(); + Assert.Equal((0, 6), ServerHealthClassifier.MeasuredMetricCounts(nothing)); + + var overall = ServerHealthClassifier.OverallMetricSeverity(nothing); + Assert.Equal(HealthSeverity.Unknown, overall); + Assert.Equal( + FleetHealthBand.Warning, + ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: false, overall)); + + /* One measured, calm reading and the fold is Healthy again — the #3528 partial-coverage card. */ + var one = nothing with { CollectorCount = 40 }; + Assert.Equal((1, 6), ServerHealthClassifier.MeasuredMetricCounts(one)); + Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.OverallMetricSeverity(one)); + + /* A Warning among Unknowns is still Warning — the nothing-measured arm never de-escalates. */ + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.OverallMetricSeverity(nothing with { CpuPercentForAlert = 85 })); + /* And the order the readings arrive in cannot matter: Warning first then Healthy, Healthy first + then Warning, both Warning. */ + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.OverallMetricSeverity(nothing with { CpuPercentForAlert = 85, CollectorCount = 40 })); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.OverallMetricSeverity(nothing with { HasMemoryPressure = false, FailedCollectorCount = 1, CollectorCount = 40 })); + } + + /* ── measured-metric coverage (#3528) ── */ + + [Fact] + public void MeasuredMetricCounts_FullyMeasuredBundle_CountsAllSix() + { + var m = new ServerHealthMetrics + { + CpuPercentForAlert = 50, + TotalThreads = 512, + AvailableThreads = 400, + HasMemoryPressure = false, + BlockingCount = 0, + BlockingWindow = TimeSpan.FromHours(1), // #3539 A3: a zero count is measured only over a window + DeadlockCount = 0, + DeadlockWindow = TimeSpan.FromHours(1), + CollectorCount = 40, // #3539 A6: zero failing is measured only with a denominator + }; + + Assert.Equal((6, 6), ServerHealthClassifier.MeasuredMetricCounts(m)); + } + + [Fact] + public void MeasuredMetricCounts_UnknownHeavyBundle_SaysSo_WhileTheFoldStillReadsHealthy() + { + /* The PostgreSQL-card shape #3528 was filed about: five of the six metrics structurally Unknown + (no CPU/threads snapshot, DMV-sourced memory/blocking/deadlocks nulled), only the collector row + measured. The fold deliberately skips Unknown, so the band label is still Healthy — and the + counts are what let a consumer render that label as "Healthy — 1 of 6 measured" instead of an + unqualified green. The collector row is measured only because a denominator was declared + (#3539 A6): forty banded, none failing. */ + var m = new ServerHealthMetrics { CollectorCount = 40 }; + + Assert.Equal((1, 6), ServerHealthClassifier.MeasuredMetricCounts(m)); + + var overall = ServerHealthClassifier.OverallMetricSeverity(m); + Assert.Equal(HealthSeverity.Healthy, overall); + Assert.Equal(FleetHealthBand.Healthy, + ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: false, overall)); + } + + [Fact] + public void MeasuredMetricCounts_AreRankNeutral() + { + /* The counts describe, they never rank: two bundles differing only in how many metrics are + measured score identically, which is the Unknown rank-neutrality + UnmeasuredMetricsAreNotHealthyTests pins, restated against the new fields' own inputs. */ + var measured = new ServerHealthMetrics + { + CpuPercentForAlert = 50, + TotalThreads = 512, + AvailableThreads = 400, + HasMemoryPressure = false, + BlockingCount = 0, + BlockingWindow = TimeSpan.FromHours(1), + DeadlockCount = 0, + DeadlockWindow = TimeSpan.FromHours(1), + CollectorCount = 40, + }; + /* One of six measured (the collectors row, #3539 A6's denominator declared) against six of six: + the partial-coverage neutrality #3528 promised. A bundle measuring NOTHING is the one case that + does move, and OverallMetricSeverity_NothingMeasured_IsUnknown_AndBandsWarning owns it. */ + var unmeasured = new ServerHealthMetrics { CollectorCount = 40 }; + + Assert.NotEqual( + ServerHealthClassifier.MeasuredMetricCounts(measured), + ServerHealthClassifier.MeasuredMetricCounts(unmeasured)); + Assert.Equal( + ServerHealthClassifier.FleetHealthScore(FleetHealthBand.Healthy, measured), + ServerHealthClassifier.FleetHealthScore(FleetHealthBand.Healthy, unmeasured)); + } + /* ── fleet band (collapse) ── */ [Fact] @@ -225,6 +445,23 @@ public void ClassifyBand_OnlineCalm_IsHealthy() => Assert.Equal(FleetHealthBand.Healthy, ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: false, HealthSeverity.Healthy)); + /// #3539 A6: an online card whose fold is Unknown (nothing measured) is Warning — the same band + /// the awaiting-first-collection server gets, for the same reason — and never Healthy, stale or not. + /// Offline still wins over it. + [Fact] + public void ClassifyBand_OnlineNothingMeasured_IsWarning_LikeAwaitingFirstCollection() + { + Assert.Equal(FleetHealthBand.Warning, + ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: false, HealthSeverity.Unknown)); + Assert.Equal(FleetHealthBand.Warning, + ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: true, HealthSeverity.Unknown)); + Assert.Equal(FleetHealthBand.Offline, + ServerHealthClassifier.ClassifyBand(isOnline: false, awaitingFirstCollection: false, collectionStale: false, HealthSeverity.Unknown)); + Assert.Equal( + ServerHealthClassifier.ClassifyBand(isOnline: null, awaitingFirstCollection: true, collectionStale: false, HealthSeverity.Unknown), + ServerHealthClassifier.ClassifyBand(isOnline: true, awaitingFirstCollection: false, collectionStale: false, HealthSeverity.Unknown)); + } + /* ── worst-first score ── */ [Fact] diff --git a/Darling/Darling.Tests/ServerPageTabsTests.cs b/Darling/Darling.Tests/ServerPageTabsTests.cs index eb7438233..1ff7a3cb4 100644 --- a/Darling/Darling.Tests/ServerPageTabsTests.cs +++ b/Darling/Darling.Tests/ServerPageTabsTests.cs @@ -95,6 +95,11 @@ public sealed class ServerPageTabsTests ["get_pg_top_queries"] = "pg_statement_stats", ["get_pg_plans"] = "pg_plan_capture", ["get_pg_plan_capture_readiness"] = "pg_plan_capture_readiness", + /* #3607: a READ over the config collector's table, not a collector of its own - the audit is + computed from the newest pg_server_config snapshot, so its not_collected gate names that + collector, which is what this map records. Two reads over one collector is the same shape + as get_pg_server_config / get_pg_server_config_changes above. */ + ["get_pg_logging_audit"] = "pg_server_config", ["get_pg_blocking"] = "pg_blocking", ["get_pg_io_stats"] = "pg_io_stats", ["get_pg_autovacuum_health"] = "pg_autovacuum_stats", diff --git a/Darling/Darling.Tests/StartupCommandTimeoutTests.cs b/Darling/Darling.Tests/StartupCommandTimeoutTests.cs index 8a6d47d6e..74661e816 100644 --- a/Darling/Darling.Tests/StartupCommandTimeoutTests.cs +++ b/Darling/Darling.Tests/StartupCommandTimeoutTests.cs @@ -125,6 +125,14 @@ private static readonly (string File, string Member, int Bootstrap, int ConnectP ("DarlingDeltaCalculator.cs", "SeedFileIoStatsAsync", 1, 0, 0, 0), ("DarlingDeltaCalculator.cs", "SeedPerfmonStatsAsync", 1, 0, 0, 0), ("DarlingDeltaCalculator.cs", "SeedMemoryGrantStatsAsync", 1, 0, 0, 0), + /* #3540 A4: the six families the restart seed gained, one bounded read each, same regime — run once, + awaited, ahead of the collection loop. */ + ("DarlingDeltaCalculator.cs", "SeedLatchStatsAsync", 1, 0, 0, 0), + ("DarlingDeltaCalculator.cs", "SeedSpinlockStatsAsync", 1, 0, 0, 0), + ("DarlingDeltaCalculator.cs", "SeedProcedureStatsAsync", 1, 0, 0, 0), + ("DarlingDeltaCalculator.cs", "SeedQueryStatsAsync", 1, 0, 0, 0), + ("DarlingDeltaCalculator.cs", "SeedPgWaitStatsAsync", 1, 0, 0, 0), + ("DarlingDeltaCalculator.cs", "SeedPgStatementStatsAsync", 1, 0, 0, 0), ("StoreConfigProvider.cs", "WarnAboutFileOnlyServersAsync", 1, 0, 0, 0), ("StoreConfigProvider.cs", "ReadRegisteredServersForComparisonAsync", 1, 0, 0, 0), ("StoreConfigProvider.cs", "CountAsync", 1, 0, 0, 0), @@ -158,7 +166,7 @@ hold. It is a single-row INSERT into the same collect.collection_log #2928 bound }; /// The group's own totals, so a member that stops creating commands fails loudly. - private const int ExpectedBootstrapSites = 26; + private const int ExpectedBootstrapSites = 32; private const int ExpectedConnectProbeSites = 2; diff --git a/Darling/Darling.Tests/StoreConfigProviderTests.cs b/Darling/Darling.Tests/StoreConfigProviderTests.cs index c90fab686..a1b3f5ff5 100644 --- a/Darling/Darling.Tests/StoreConfigProviderTests.cs +++ b/Darling/Darling.Tests/StoreConfigProviderTests.cs @@ -197,6 +197,44 @@ public void Resolve_RejectsDestructiveRetention_AndNegativeFrequency_FallingBack Assert.Equal(1, eff.RetentionDays); } + /// + /// #3532: a delta-family cadence past + /// would exceed the shared delta gap policy every cycle — the collector re-baselines each run and + /// stores (0, 0) forever, fabricating permanent quiet. The viewer's editor refuses to write such a + /// row, but a hand-written or pre-fix row can still exist, so the resolver treats it as "no override" + /// and falls through to the next level, exactly like a negative frequency. + /// + [Fact] + public void Resolve_RejectsADeltaFamilyCadencePastTheGapPolicyCap_FallingThrough() + { + var def = CollectorScheduleDefaults.All["wait_stats"]; + var cap = CollectorDeltaCalculator.MaxDeltaFrequencyMinutes; + + /* A poisoned fleet row falls all the way through to the code default. */ + var fleetBad = new[] { new ScheduleOverride(null, "wait_stats", cap + 60, null, true) }; + Assert.Equal(def.FrequencyMinutes, StoreConfigProvider.ResolveSchedule("wait_stats", 1, fleetBad).FrequencyMinutes); + + /* A poisoned per-server row falls through to a VALID fleet row, per-column. */ + var layered = new[] + { + new ScheduleOverride(null, "wait_stats", 15, null, true), + new ScheduleOverride(1, "wait_stats", cap + 1, null, true), + }; + Assert.Equal(15, StoreConfigProvider.ResolveSchedule("wait_stats", 1, layered).FrequencyMinutes); + + /* The cap itself is honored, the PostgreSQL delta family is covered, and a snapshot collector + keeps its long cadence — the bound is per-collector-kind, not blanket. */ + var atCap = new[] { new ScheduleOverride(null, "wait_stats", cap, null, true) }; + Assert.Equal(cap, StoreConfigProvider.ResolveSchedule("wait_stats", 1, atCap).FrequencyMinutes); + + var pgBad = new[] { new ScheduleOverride(null, "pg_wait_stats", cap + 60, null, true) }; + Assert.Equal(CollectorScheduleDefaults.All["pg_wait_stats"].FrequencyMinutes, + StoreConfigProvider.ResolveSchedule("pg_wait_stats", 1, pgBad).FrequencyMinutes); + + var snapshot = new[] { new ScheduleOverride(null, "database_size_stats", cap + 60, null, true) }; + Assert.Equal(cap + 60, StoreConfigProvider.ResolveSchedule("database_size_stats", 1, snapshot).FrequencyMinutes); + } + /* ---------------- live (DARLING_TEST_PG): the V17 bump trigger, rolled back ---------------- */ [Fact] diff --git a/Darling/Darling.Tests/StoreSelfMetricsTests.cs b/Darling/Darling.Tests/StoreSelfMetricsTests.cs index 90683d6b5..ef59e70b2 100644 --- a/Darling/Darling.Tests/StoreSelfMetricsTests.cs +++ b/Darling/Darling.Tests/StoreSelfMetricsTests.cs @@ -222,9 +222,210 @@ public void Retention_IsTheSweepsOwnBoundedDelete_At400Days() Assert.Contains("WHERE metric_time < $1", StoreSelfMetrics.RetentionDeleteSql, StringComparison.Ordinal); } + /* ---------------- #3582: the rows the inventory was blind to, and the reconciliation ---------------- */ + + /// + /// #3582: the aggregate rows are sized through the MATERIALIZATION and named by the VIEW. Pinned + /// because the hypertable arm cannot be made to see them — timescaledb_information.hypertables + /// ends AND ca.mat_hypertable_id IS NULL on 2.28.1 — so a second enumeration over the + /// aggregates view is the only route, and it has to size the internal hypertable while reporting the + /// name an operator knows. The chunk count comes from the chunks view and NOT from a count over + /// chunk_compression_stats, which returns zero rows for a hypertable whose compression is off + /// (measured: a two-chunk materialization yielded no rows until compression was enabled). + /// + [Fact] + public void ContinuousAggregateInsertSql_SizesTheMaterialization_UnderTheViewName() + { + var sql = StoreSelfMetrics.ContinuousAggregateInsertSql; + + Assert.Contains("FROM timescaledb_information.continuous_aggregates ca", sql, StringComparison.Ordinal); + Assert.Contains("ca.view_name,", sql, StringComparison.Ordinal); + Assert.Contains($"'{StoreSelfMetrics.ContinuousAggregateObjectKind}'", sql, StringComparison.Ordinal); + Assert.Equal("continuous_aggregate", StoreSelfMetrics.ContinuousAggregateObjectKind); + + /* Both size functions take the MATERIALIZATION regclass, built from the view's own columns. */ + const string mat = "format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass"; + Assert.Contains($"hypertable_detailed_size({mat})", sql, StringComparison.Ordinal); + Assert.Contains($"chunk_compression_stats({mat})", sql, StringComparison.Ordinal); + + /* The chunk count: from the chunks view, matched on the materialization's name, never inferred + from the compression-stats row count. */ + Assert.Contains("FROM timescaledb_information.chunks ch", sql, StringComparison.Ordinal); + Assert.Contains("ch.hypertable_name = ca.materialization_hypertable_name", sql, StringComparison.Ordinal); + Assert.DoesNotMatch(@"count\(\*\)::integer AS chunk_count\s+FROM chunk_compression_stats", sql); + + /* And the hypertable arm is UNFILTERED — the omission is the view's, not a predicate of ours. */ + Assert.DoesNotContain("WHERE", StoreSelfMetrics.HypertableInsertSql, StringComparison.Ordinal); + } + + /// + /// #3582: the three product-owned plain tables the walk used to lump are named, schema-qualified, sized + /// with pg_total_relation_size and row-counted from the planner's estimate — and the census predicate that keeps them OUT + /// of the catch-all names the same three by (schema, relation). The two halves are pinned + /// against each other: a table named here and not there would be counted twice, one named there and + /// not here would vanish from both. + /// + [Fact] + public void TableInsertSql_NamesTheThreeProductTables_AndTheCensusExcludesExactlyThose() + { + var sql = StoreSelfMetrics.TableInsertSql; + Assert.Equal("table", StoreSelfMetrics.TableObjectKind); + Assert.Contains($"'{StoreSelfMetrics.TableObjectKind}'", sql, StringComparison.Ordinal); + + var qualified = new[] { QueryStoreTextStore.TableName, QueryStorePlanMap.TableName, StoreSelfMetrics.AlertLogTable }; + Assert.Equal(new[] { "collect.query_store_text", "collect.query_store_plan_map", "config.config_alert_log" }, qualified); + + foreach (var table in qualified) + { + Assert.Contains($"'{table}',", sql, StringComparison.Ordinal); + Assert.Contains($"pg_total_relation_size('{table}')", sql, StringComparison.Ordinal); + /* The planner's estimate, NULL where it is -1 (never analysed) — never a 15 GiB count(*) an hour. */ + Assert.Contains($"(SELECT CASE WHEN c.reltuples >= 0 THEN c.reltuples::bigint END FROM pg_class c WHERE c.oid = '{table}'::regclass)", sql, StringComparison.Ordinal); + + /* The census names the same table by the same compound constant, compared against the + concatenated schema.relation — no hand-typed (schema, relation) tuple to drift (review catch). */ + Assert.Contains($"'{table}'", StoreSelfMetrics.NamedRelationPredicateSql, StringComparison.Ordinal); + } + + Assert.Contains("(n.nspname || '.' || c.relname) IN (", StoreSelfMetrics.NamedRelationPredicateSql, StringComparison.Ordinal); + /* The two payload dimensions are in the same predicate, so they leave the catch-all too. */ + Assert.Contains($"'collect.{PayloadDimensions.QueryTextDimTable}'", StoreSelfMetrics.NamedRelationPredicateSql, StringComparison.Ordinal); + Assert.Contains($"'collect.{PayloadDimensions.QueryPlanDimTable}'", StoreSelfMetrics.NamedRelationPredicateSql, StringComparison.Ordinal); + /* Exactly five names, and none of them typed by hand: every quoted name in the predicate is one of + the five constants. */ + var quoted = System.Text.RegularExpressions.Regex.Matches(StoreSelfMetrics.NamedRelationPredicateSql, @"'([^']+)'").Select(m => m.Groups[1].Value).ToArray(); + Assert.Equal(6, quoted.Length); /* five names plus the '.' separator literal */ + Assert.Equal( + new[] { $"collect.{PayloadDimensions.QueryTextDimTable}", $"collect.{PayloadDimensions.QueryPlanDimTable}" }.Concat(qualified).OrderBy(x => x, StringComparer.Ordinal), + quoted.Where(q => q != ".").OrderBy(x => x, StringComparer.Ordinal)); + Assert.Equal(3, sql.Split("UNION ALL").Length); + Assert.DoesNotContain("count(*)", sql, StringComparison.Ordinal); + } + + /// + /// #3582: the two catch-all rows, in both store shapes. The census is the same predicate three ways — + /// which relations count, which are system, which are already named — and the TimescaleDB variant adds + /// the three anti-joins that remove what the hypertable and aggregate rows already sized (their roots, + /// and every chunk relation), while the plain variant names no TimescaleDB catalog at all, because on + /// that store none exists and the statement would fail. NOT c.relisshared is pinned by itself: + /// the shared catalogs are outside pg_database_size, and summing them made a rig census EXCEED + /// the database — an over-100% reconciliation is wrong in the direction nobody checks. + /// + [Theory] + [InlineData(nameof(StoreSelfMetrics.UnenumeratedInsertSql), true)] + [InlineData(nameof(StoreSelfMetrics.UnenumeratedPlainInsertSql), false)] + public void UnenumeratedInsertSql_WritesOtherAndSystem_OverTheSharedCensus(string sqlName, bool timescale) + { + var sql = (string)typeof(StoreSelfMetrics).GetField(sqlName, System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static)!.GetValue(null)!; + + Assert.Equal("other", StoreSelfMetrics.OtherObjectKind); + Assert.Equal("system", StoreSelfMetrics.SystemObjectKind); + Assert.Contains($"'{StoreSelfMetrics.OtherObjectName}', '{StoreSelfMetrics.OtherObjectKind}'", sql, StringComparison.Ordinal); + Assert.Contains($"'{StoreSelfMetrics.SystemObjectName}', '{StoreSelfMetrics.SystemObjectKind}'", sql, StringComparison.Ordinal); + Assert.Contains("FROM census WHERE NOT is_system", sql, StringComparison.Ordinal); + Assert.Contains("FROM census WHERE is_system", sql, StringComparison.Ordinal); + + /* The shared fragments, verbatim — the MCP reader's live top-N composes the same three. */ + Assert.Contains(StoreSelfMetrics.CensusRelationPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains(StoreSelfMetrics.SystemSchemaPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains("NOT " + StoreSelfMetrics.NamedRelationPredicateSql, sql, StringComparison.Ordinal); + + Assert.Contains("c.relkind IN ('r', 'm', 'p', 'S')", StoreSelfMetrics.CensusRelationPredicateSql, StringComparison.Ordinal); + Assert.Contains("NOT c.relisshared", StoreSelfMetrics.CensusRelationPredicateSql, StringComparison.Ordinal); + Assert.Contains("starts_with(n.nspname, '_timescaledb_')", StoreSelfMetrics.SystemSchemaPredicateSql, StringComparison.Ordinal); + Assert.Contains("'pg_catalog', 'information_schema'", StoreSelfMetrics.SystemSchemaPredicateSql, StringComparison.Ordinal); + + /* An empty bucket is a ZERO row, never a missing one — the reconciliation reads absence as "the + statement did not run". */ + Assert.Contains("coalesce(sum(pg_total_relation_size(oid)), 0)::bigint, count(*)::integer", sql, StringComparison.Ordinal); + + if (timescale) + { + Assert.Contains(StoreSelfMetrics.TimescaleInventoriedPredicateSql, sql, StringComparison.Ordinal); + Assert.Contains("FROM timescaledb_information.hypertables h", sql, StringComparison.Ordinal); + Assert.Contains("FROM timescaledb_information.continuous_aggregates ca", sql, StringComparison.Ordinal); + Assert.Contains("FROM _timescaledb_catalog.chunk ch", sql, StringComparison.Ordinal); + /* Name joins, never a regclass cast a vanished relation would make RAISE. */ + Assert.DoesNotContain("::regclass", StoreSelfMetrics.TimescaleInventoriedPredicateSql, StringComparison.Ordinal); + } + else + { + /* The plain variant READS no TimescaleDB catalog. It still NAMES timescaledb_information as a + string literal inside the system-schema predicate, which is a different thing. */ + Assert.DoesNotContain("FROM timescaledb_information", sql, StringComparison.Ordinal); + Assert.DoesNotContain("FROM _timescaledb_catalog", sql, StringComparison.Ordinal); + } + } + + /// + /// #3574 (managed-mode self-proof): the owner's evidence row embeds the SAME evidence SELECT the MCP + /// reader runs — one string, two consumers — and maps its columns the way the class summary states. + /// The count is written only where the view admits the sweep's role to every job's history (the two + /// tests the reader's Visibility derives All from, in SQL), the newest row travels as its + /// AGE at the sweep, and the window width is computed from the two binds rather than restated. + /// + [Fact] + public void JobHistoryInsertSql_EmbedsTheSharedEvidenceRead_AndMapsTheOverloadedColumns() + { + var sql = StoreSelfMetrics.JobHistoryInsertSql; + + Assert.Equal("job_history", StoreSelfMetrics.JobHistoryObjectKind); + Assert.Contains($"'{StoreSelfMetrics.JobHistoryObjectKind}'", sql, StringComparison.Ordinal); + Assert.Contains(StoreSelfMetrics.JobHistoryEvidenceSql, sql, StringComparison.Ordinal); + Assert.Contains("(metric_time, object_name, object_kind, row_count, total_runs, schedule_interval_ms, last_run_duration_ms)", sql, StringComparison.Ordinal); + + /* $2 is metric_time, $1 the window start the embedded SELECT already binds. */ + Assert.Matches(@"SELECT\s+\$2,\s+e\.reader_role,\s+'job_history',", sql); + Assert.Contains("h.start_time >= $1", sql, StringComparison.Ordinal); + + /* The census-or-NULL rule for the count. */ + Assert.Matches(@"CASE\s+WHEN e\.reader_is_database_owner_member\s+OR \(e\.job_count > 0 AND e\.owner_member_job_count >= e\.job_count\)\s+THEN e\.rows_observed\s+END", sql); + + /* Population, window width from the binds, age of the newest row — in that column order. */ + Assert.Contains("e.jobs_run_in_window,", sql, StringComparison.Ordinal); + Assert.Contains("(EXTRACT(EPOCH FROM (($2::timestamp AT TIME ZONE 'UTC') - $1)) * 1000)::bigint", sql, StringComparison.Ordinal); + Assert.Contains("(EXTRACT(EPOCH FROM (($2::timestamp AT TIME ZONE 'UTC') - e.newest_row_at)) * 1000)::bigint", sql, StringComparison.Ordinal); + Assert.Equal(24, StoreSelfMetrics.JobHistoryEvidenceWindowHours); + } + + /// + /// The kind vocabulary is the on-disk contract: every kind the sweep writes has a named constant, the + /// constant appears quoted in the statement that writes it, and no two kinds share a spelling. A + /// drifted kind returns zero rows to every reader rather than an error. + /// + [Fact] + public void ObjectKinds_AreNamedOnce_DistinctAndWrittenByTheirArms() + { + var kinds = new (string Kind, string Sql)[] + { + (StoreSelfMetrics.HypertableObjectKind, StoreSelfMetrics.HypertableInsertSql), + (StoreSelfMetrics.ContinuousAggregateObjectKind, StoreSelfMetrics.ContinuousAggregateInsertSql), + (StoreSelfMetrics.BackgroundJobObjectKind, StoreSelfMetrics.BackgroundJobInsertSql), + (StoreSelfMetrics.DimensionObjectKind, StoreSelfMetrics.DimensionInsertSql), + (StoreSelfMetrics.TableObjectKind, StoreSelfMetrics.TableInsertSql), + (StoreSelfMetrics.OtherObjectKind, StoreSelfMetrics.UnenumeratedInsertSql), + (StoreSelfMetrics.SystemObjectKind, StoreSelfMetrics.UnenumeratedInsertSql), + (StoreSelfMetrics.JobHistoryObjectKind, StoreSelfMetrics.JobHistoryInsertSql), + (StoreSelfMetrics.StoreObjectKind, StoreSelfMetrics.StoreInsertSql), + }; + + Assert.Equal(kinds.Length, kinds.Select(k => k.Kind).Distinct(StringComparer.Ordinal).Count()); + Assert.All(kinds, k => Assert.Contains($"'{k.Kind}'", k.Sql, StringComparison.Ordinal)); + + /* The three pre-#3582 spellings the shipped stores already hold 400 days of. */ + Assert.Equal("hypertable", StoreSelfMetrics.HypertableObjectKind); + Assert.Equal("dimension", StoreSelfMetrics.DimensionObjectKind); + Assert.Equal("background_job", StoreSelfMetrics.BackgroundJobObjectKind); + } + [Theory] [InlineData(nameof(StoreSelfMetrics.HypertableInsertSql))] + [InlineData(nameof(StoreSelfMetrics.ContinuousAggregateInsertSql))] [InlineData(nameof(StoreSelfMetrics.DimensionInsertSql))] + [InlineData(nameof(StoreSelfMetrics.TableInsertSql))] + [InlineData(nameof(StoreSelfMetrics.UnenumeratedInsertSql))] + [InlineData(nameof(StoreSelfMetrics.UnenumeratedPlainInsertSql))] + [InlineData(nameof(StoreSelfMetrics.JobHistoryInsertSql))] [InlineData(nameof(StoreSelfMetrics.StoreInsertSql))] [InlineData(nameof(StoreSelfMetrics.RetentionDeleteSql))] public void SweepSql_IsPostgresDialect_PositionalParams_NoBareNow(string sqlName) @@ -251,6 +452,19 @@ makes a run's rows join and keeps the timestamps naive UTC by the cross-store co /// converts + applies compression policies so real background jobs exist, then asserts one run writes /// hypertable, dimension, store, AND background_job rows — the job rows carrying a schedule interval, /// because "duration vs cadence" is the series' whole point. + /// + /// #3582 extends it to the kinds the inventory was blind to, and to the reconciliation. + /// The aggregates are created first (), so + /// one run must also write one continuous_aggregate row per rollup view, the three named + /// table rows, exactly one other and one system row, and the owner's + /// job_history row — and the rows of that one sweep, read back through the real MCP reader and + /// its real reconciliation, must RECONCILE against the sweep's own pg_database_size: every byte + /// attributed to some row, residual inside the bar, no object left over from an older sweep. On a rig + /// the residual was exactly the database directory's non-relation files (161,471 bytes on 17 MiB); + /// here only the bar is asserted, because refresh policies fire immediately on creation (#1788) and + /// move the catalogs between the sweep's statements. The connection is the database owner, so the + /// owner row must carry a COUNT (not the NULL a filtered role writes) and decode as Observed + /// for the role that swept. /// [Fact] public async Task Sweep_EndToEnd_WritesEveryObjectKind_IncludingBackgroundJobs_AgainstDevPostgres() @@ -269,19 +483,28 @@ public async Task Sweep_EndToEnd_WritesEveryObjectKind_IncludingBackgroundJobs_A Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed"); await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + /* #3582: the aggregates, so the materializations exist to be inventoried. */ + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); await TimescaleSupport.ApplyCompressionPolicyAsync(connection, null, ct); var written = await StoreSelfMetrics.SweepAsync( connection, timescaleAvailable: true, DateTime.UtcNow, null, ct); Assert.True(written > 0, "the sweep wrote nothing"); - await using var kinds = new NpgsqlCommand(@" + await using var kinds = new NpgsqlCommand($@" SELECT - count(*) FILTER (WHERE object_kind = 'hypertable'), - count(*) FILTER (WHERE object_kind = 'dimension'), - count(*) FILTER (WHERE object_kind = 'store'), - count(*) FILTER (WHERE object_kind = 'background_job'), - count(*) FILTER (WHERE object_kind = 'background_job' AND schedule_interval_ms > 0) + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.HypertableObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.DimensionObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.StoreObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.BackgroundJobObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.BackgroundJobObjectKind}' AND schedule_interval_ms > 0), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.ContinuousAggregateObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.ContinuousAggregateObjectKind}' AND total_bytes > 0 AND chunk_count IS NOT NULL), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.TableObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.OtherObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.SystemObjectKind}'), + count(*) FILTER (WHERE object_kind = '{StoreSelfMetrics.JobHistoryObjectKind}'), + (SELECT count(*) FROM timescaledb_information.continuous_aggregates) FROM collect.store_metrics", connection); await using var reader = await kinds.ExecuteReaderAsync(ct); Assert.True(await reader.ReadAsync(ct)); @@ -291,15 +514,62 @@ public async Task Sweep_EndToEnd_WritesEveryObjectKind_IncludingBackgroundJobs_A Assert.Equal(1, reader.GetInt64(2)); Assert.True(reader.GetInt64(3) > 0, "no background_job rows — the compression policies just applied guarantee jobs exist"); Assert.True(reader.GetInt64(4) > 0, "background_job rows carry no schedule interval — duration-vs-cadence needs it"); + + /* #3582: one aggregate row per aggregate the catalog knows — the population the hypertable arm + cannot see — each sized (a materialization's root alone is non-zero) and chunk-counted. */ + var aggregatesInCatalog = reader.GetInt64(11); + Assert.True(aggregatesInCatalog > 0, "EnsureContinuousAggregatesAsync left no aggregates to inventory"); + Assert.Equal(aggregatesInCatalog, reader.GetInt64(5)); + Assert.Equal(aggregatesInCatalog, reader.GetInt64(6)); + Assert.Equal(3, reader.GetInt64(7)); + Assert.Equal(1, reader.GetInt64(8)); + Assert.Equal(1, reader.GetInt64(9)); + Assert.Equal(1, reader.GetInt64(10)); await reader.CloseAsync(); /* And the READ path carries the new fields end to end (the review catch: written but never read back would leave get_store_metrics returning job rows with null metrics). */ await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); var latest = await PerformanceMonitor.Darling.Service.Mcp.DarlingStoreMetricsReader.GetLatestAsync(dataSource, ct); - var job = latest.FirstOrDefault(r => r.ObjectKind == "background_job"); + var job = latest.FirstOrDefault(r => r.ObjectKind == StoreSelfMetrics.BackgroundJobObjectKind); Assert.NotNull(job); Assert.True(job!.ScheduleIntervalMs is > 0, "the reader dropped the job's schedule interval"); + + /* #3582: the reconciliation, through the real reader over the real rows. One sweep, so no row is + from an older one; both catch-all rows present; every byte attributed inside the bar; and the + named rows are a non-trivial share even of an empty store (roots and indexes are real bytes). */ + var inventory = PerformanceMonitor.Darling.Service.Mcp.DarlingStoreMetricsReader.ComputeInventory(latest); + Assert.NotNull(inventory); + Assert.Equal(0, inventory!.StaleRowCount); + Assert.True(inventory.CatchAllPresent, "a catch-all row is missing from the sweep"); + Assert.True(inventory.EnumeratedBytes > 0); + Assert.True(inventory.DatabaseBytes > inventory.EnumeratedBytes); + Assert.True(inventory.Reconciled, + $"the inventory did not reconcile: database {inventory.DatabaseBytes}, attributed {inventory.AttributedBytes}, " + + $"residual {inventory.ResidualBytes}, bar {inventory.ToleranceBytes}"); + Assert.Contains(StoreSelfMetrics.ContinuousAggregateObjectKind, inventory.BytesByKind.Keys); + Assert.Contains(StoreSelfMetrics.TableObjectKind, inventory.BytesByKind.Keys); + + /* #3574: the owner's row. This connection is the database owner, so the sweep's role was admitted + to every job's history and wrote a COUNT; decoded, that is an Observed reading for that role, + over the 24-hour window, taken moments ago. */ + var history = Assert.Single(latest, r => r.ObjectKind == StoreSelfMetrics.JobHistoryObjectKind); + Assert.NotNull(history.RowCount); + Assert.NotNull(history.TotalRuns); + Assert.Equal(StoreSelfMetrics.JobHistoryEvidenceWindowHours * 3_600_000L, history.ScheduleIntervalMs); + + var owner = PerformanceMonitor.Darling.Service.Mcp.DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(latest, DateTime.UtcNow); + Assert.Equal(PerformanceMonitor.Darling.Service.Mcp.DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Observed, owner.Status); + Assert.Equal(history.ObjectName, owner.ReaderRole); + Assert.Equal(history.RowCount, owner.RowsObserved); + Assert.Equal(24.0, owner.WindowHours); + Assert.True(owner.AgeHours is >= 0 and < 1); + /* A row seen implies a newest-row instant that decodes to no later than the sweep itself. */ + if (owner.RowsObserved > 0) + { + Assert.NotNull(owner.NewestRowAt); + Assert.True(owner.NewestRowAt <= owner.ObservedAt); + } } /* ---------------- #2136 synthetic scale test ---------------- */ @@ -705,6 +975,15 @@ public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationTok Outcomes.Add(outcome); return Task.CompletedTask; } + + /* #3580: DeliverAndReportAsync is REQUIRED on the seam rather than defaulted (CONTRIBUTING, Two-Store + Parity), so every fake answers it by hand. This one reports nothing: null is "unreported", which the + two daily documents treat as delivered, exactly as every fire before #3580 was. */ + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + { + await DeliverAsync(outcome, cancellationToken); + return null; + } } private sealed class CadenceFakeHistoryStore : IAlertHistoryStore @@ -751,6 +1030,7 @@ private sealed class CadenceFakeSettings : IAlertEngineSettings public int DiskCriticalFreePercent { get; set; } = 3; public int DiskCriticalFreeGb { get; set; } = 2; public int SelfDiskFreeWarnPercent { get; set; } = 10; + public int SelfDiskFreeWarnGb { get; set; } = 50; public int CollectionStaleMinutes { get; set; } = 30; public int CollectionFailureThreshold { get; set; } = 10; public int PvsThresholdPercent { get; set; } = 40; diff --git a/Darling/Darling.Tests/SweepBodyDetachPolicyTests.cs b/Darling/Darling.Tests/SweepBodyDetachPolicyTests.cs index 37c25297d..f96b59696 100644 --- a/Darling/Darling.Tests/SweepBodyDetachPolicyTests.cs +++ b/Darling/Darling.Tests/SweepBodyDetachPolicyTests.cs @@ -39,6 +39,13 @@ namespace Darling.Tests; /// has zero fanout at p90 11,964ms. A fanout-derived split would detach the cheap collector and leave /// the expensive one starving the tier. See #2840. /// +/// The third member is detached for a different reason (#3604). pg_wait_sampling's +/// service-sampler arm holds its connection for thirty one-second pg_stat_activity snapshots per cycle +/// BY DESIGN — a deterministic 30 s run, not a bimodal tail — and awaited inline that would delay every +/// other collector on a stock PostgreSQL target by half a minute every five. The criterion generalises: a +/// single-run cost that would starve the fast tier if it sat in the body, whether measured or designed. It +/// sits on the five-minute tier, so the one-minute-tier invariant below holds for it as for the others. +/// /// The 4.5x evidence. use2 runs the same Balanced preset with Query Store dead since /// 2026-08-17 17:36. Its query_stats delivered cadence stepped from 4.69-9.02 min (Query Store /// live) to 1.44-1.57 min (dead) the following day, and held there for two weeks. @@ -46,10 +53,11 @@ namespace Darling.Tests; public sealed class SweepBodyDetachPolicyTests { /// The collectors #2700/#2717 detach, by the criterion documented on this class. - private static readonly string[] ExpectedDetached = { "query_store", "plan_correction" }; + private static readonly string[] ExpectedDetached = { "query_store", "plan_correction", "pg_wait_sampling" }; private static bool IsDetached(string name) => - DarlingWorker.IsQueryStoreCollector(name) || DarlingWorker.IsPlanCorrectionCollector(name); + DarlingWorker.IsQueryStoreCollector(name) || DarlingWorker.IsPlanCorrectionCollector(name) + || DarlingWorker.IsPgWaitSamplingCollector(name); /// /// The invariant that makes detaching safe, and the one that GENERALISES: a detached collector runs diff --git a/Darling/Darling.Tests/TimescaleAggregateCompressionTests.cs b/Darling/Darling.Tests/TimescaleAggregateCompressionTests.cs new file mode 100644 index 000000000..947d69ddf --- /dev/null +++ b/Darling/Darling.Tests/TimescaleAggregateCompressionTests.cs @@ -0,0 +1,777 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins the continuous-aggregate compression ladder (#3581): the per-tier compress_after as a +/// DERIVATION from each tier's refresh window rather than a chosen number, the daily band's instant as a minute +/// the hourly phase grid does not use and an hour per aggregate, the largest-first one-per-night staging of the +/// backlog, the registry that every one of those reads from, and — gated on DARLING_TEST_PG — the +/// ensure itself against a real TimescaleDB store: every aggregate compression-enabled, one once-a-day policy +/// per aggregate, a settled second pass that adds nothing, and the raw compression converge leaving the family +/// alone — and (#3620) every materialization held at one raw chunk of width, with a re-run that changes none. +/// +/// Ungated pins read the shipped registry and the shipped CREATE text, never a copy of either, so a new +/// aggregate is covered the moment it is registered and a pin cannot agree with a derivation the product does +/// not have. +/// +[Collection("live-postgres")] +public sealed class TimescaleAggregateCompressionTests +{ + /// + /// THE DERIVATION. compress_after is each tier's refresh start_offset plus one raw chunk, and + /// that lands on exactly the two values #3581 ruled — 2 days hourly, 4 days daily — as an EXPRESSION. + /// + /// The disjointness condition the margin exists for is compress_after ≥ start_offset + bucket + /// (a refresh window aligns its start down to a bucket boundary; a chunk compresses only when its whole + /// range is past the window). Asserted per tier with the tier's own bucket, so a tier whose bucket widened + /// past the margin goes red here rather than quietly overlapping. The strict inequality against the bare + /// offset is asserted too — a margin of zero would satisfy nothing this file promises. + /// + [Fact] + public void CompressAfter_IsEachTiersRefreshOffsetPlusOneRawChunk_AndClearsTheAlignedWindow() + { + Assert.Equal(TimeSpan.FromDays(TimescaleSupport.ChunkIntervalDays), TimescaleSupport.AggregateCompressMarginSpan); + Assert.Equal(TimeSpan.FromDays(1), TimescaleSupport.AggregateCompressMarginSpan); + + Assert.Equal( + TimescaleSupport.HourlyRefreshStartSpan + TimescaleSupport.AggregateCompressMarginSpan, + TimescaleSupport.HourlyAggregateCompressAfterSpan); + Assert.Equal( + TimescaleSupport.DailyRefreshStartSpan + TimescaleSupport.AggregateCompressMarginSpan, + TimescaleSupport.DailyAggregateCompressAfterSpan); + + /* The ruling's numbers, as the values the expressions evaluate to today. */ + Assert.Equal(TimeSpan.FromDays(2), TimescaleSupport.HourlyAggregateCompressAfterSpan); + Assert.Equal(TimeSpan.FromDays(4), TimescaleSupport.DailyAggregateCompressAfterSpan); + Assert.Equal("2 days", TimescaleSupport.HourlyAggregateCompressAfter); + Assert.Equal("4 days", TimescaleSupport.DailyAggregateCompressAfter); + + /* The condition itself, per tier, with the tier's own bucket width. */ + Assert.True( + TimescaleSupport.HourlyAggregateCompressAfterSpan >= TimescaleSupport.HourlyRefreshStartSpan + TimescaleSupport.HourlyBucket, + "the hourly tier's compress_after no longer clears its refresh window aligned down to a bucket"); + Assert.True( + TimescaleSupport.DailyAggregateCompressAfterSpan >= TimescaleSupport.DailyRefreshStartSpan + TimescaleSupport.DailyBucket, + "the daily tier's compress_after no longer clears its refresh window aligned down to a bucket"); + + Assert.True(TimescaleSupport.HourlyAggregateCompressAfterSpan > TimescaleSupport.HourlyRefreshStartSpan); + Assert.True(TimescaleSupport.DailyAggregateCompressAfterSpan > TimescaleSupport.DailyRefreshStartSpan); + Assert.True(TimescaleSupport.DailyAggregateCompressAfterSpan > TimescaleSupport.HourlyAggregateCompressAfterSpan); + + /* The literal renderer refuses a non-day span, because a compress_after that stopped being whole days + means one of its inputs did — a design change, not a formatting one. */ + Assert.Throws(() => TimescaleSupport.WholeDaysInterval(TimeSpan.FromHours(36))); + Assert.Throws(() => TimescaleSupport.WholeDaysInterval(TimeSpan.Zero)); + Assert.Equal("7 days", TimescaleSupport.WholeDaysInterval(TimeSpan.FromDays(7))); + } + + /// + /// The registry is the three creation lists and nothing else: twenty aggregates, each registered once, each + /// carrying the tier of the list it came from, each aliasing its bucket bucket, and each grouping by + /// server_id — the last two recovered from the shipped CREATE text, which is what lets the + /// segmentby/orderby the ensure emits be a property of the registry rather than an assumption. + /// + [Fact] + public void EveryAggregate_IsRegisteredOnce_WithItsTier_ABucketColumn_AndServerIdInItsGroupKey() + { + var targets = TimescaleSupport.AggregateCompressionTargets; + + Assert.Equal(20, targets.Count); + Assert.Equal(6, TimescaleSupport.HourlyAggregates.Length); + Assert.Equal(7, TimescaleSupport.DailyAggregates.Length); + Assert.Equal(7, TimescaleSupport.BaselineAggregates.Length); + Assert.Equal( + TimescaleSupport.HourlyAggregates.Length + TimescaleSupport.DailyAggregates.Length + TimescaleSupport.BaselineAggregates.Length, + targets.Count); + + Assert.Equal(targets.Count, targets.Select(t => t.View).Distinct(StringComparer.Ordinal).Count()); + + /* Order and tier are the source lists', in order — the same order the ensure sweep creates in, so the + hour each aggregate takes on the band follows creation order and nothing else. */ + var expected = TimescaleSupport.HourlyAggregates.Select(a => (a.View, Hourly: true)) + .Concat(TimescaleSupport.DailyAggregates.Select(a => (a.View, Hourly: false))) + .Concat(TimescaleSupport.BaselineAggregates.Select(a => (a.View, Hourly: true))) + .ToArray(); + Assert.Equal(expected, targets.Select(t => (t.View, t.Hourly)).ToArray()); + + foreach (var (createSql, view, hourly) in targets) + { + Assert.Equal("bucket", TimescaleSupport.AggregateBucketColumnFor(createSql)); + Assert.Contains( + TimescaleSupport.AggregateCompressionSegmentBy, + TimescaleSupport.RefreshGroupingTermsFor(createSql), + StringComparer.Ordinal); + + Assert.Equal( + hourly ? TimescaleSupport.HourlyAggregateCompressAfterSpan : TimescaleSupport.DailyAggregateCompressAfterSpan, + TimescaleSupport.AggregateCompressAfterSpanFor(view)); + Assert.Equal( + hourly ? TimescaleSupport.HourlyAggregateCompressAfter : TimescaleSupport.DailyAggregateCompressAfter, + TimescaleSupport.AggregateCompressAfterFor(view)); + } + + /* The daily tier is exactly the seven hierarchical dailies; every daily view's CREATE reads FROM an + hourly aggregate, never from raw — which is why its refresh window, and therefore its compress_after, + is the daily one. */ + foreach (var (createSql, view) in TimescaleSupport.DailyAggregates) + { + Assert.Contains("time_bucket('1 day', bucket)", createSql, StringComparison.Ordinal); + Assert.EndsWith("_daily", view, StringComparison.Ordinal); + } + + /* THE PARSE IS CONTROLLED: the alias comes out of the text, so a definition that aliased differently + would compress in ITS column, and a definition with no bucket or no alias is refused rather than + defaulted to a column that does not exist. */ + Assert.Equal("slot", TimescaleSupport.AggregateBucketColumnFor( + "CREATE MATERIALIZED VIEW x WITH (timescaledb.continuous) AS SELECT server_id, time_bucket('1 hour', t) AS slot, sum(v) FROM r GROUP BY 1, 2 WITH NO DATA")); + Assert.Equal("bucket", TimescaleSupport.AggregateBucketColumnFor( + "SELECT server_id, time_bucket(INTERVAL '1 day', time_bucket('1 hour', t)) AS bucket FROM r")); + Assert.Throws(() => TimescaleSupport.AggregateBucketColumnFor("SELECT server_id, t AS bucket FROM r")); + Assert.Throws(() => TimescaleSupport.AggregateBucketColumnFor("SELECT time_bucket('1 hour', t), server_id FROM r")); + } + + /// + /// Every aggregate that carries a retention policy compresses well inside its own horizon — walked over + /// itself (#1905's shape), so a tier added tomorrow is + /// checked the day it is added. + /// + /// Two inequalities. compress_after < drop_after is the one that keeps compression from being + /// a no-op on a tier (a chunk dropped before it could compress was never compressed). The second is the + /// issue's own claim about the short interval-identity tiers — that they "still get most of their life + /// compressed" — stated as 2 × compress_after < drop_after, which is what "most" means at 1-day + /// materialization chunks: the 7-day tier spends five of seven days compressed and the 10-day tier six of + /// ten. At 10-day chunks that share is smaller, which the ensure's chunk-width paragraph states; this pin is + /// about the horizons, not the width. + /// + [Fact] + public void EveryRetainedAggregate_CompressesWellInsideItsOwnHorizon() + { + var checkedTiers = 0; + + foreach (var (relation, dropAfter, _, _) in TimescaleSupport.RetentionPolicies) + { + if (!TimescaleSupport.IsAggregateCompressionTarget(relation)) + { + continue; + } + + checkedTiers++; + var horizon = ParseDays(dropAfter); + var compressAfter = TimescaleSupport.AggregateCompressAfterSpanFor(relation); + + Assert.True( + compressAfter < horizon, + $"{relation} compresses after {compressAfter} but is dropped after {dropAfter} — compression would never happen on this tier"); + Assert.True( + compressAfter + compressAfter < horizon, + $"{relation} compresses after {compressAfter} against a {dropAfter} horizon, so less than half its life is compressed"); + } + + /* The control: the walk covered the whole aggregate ladder — every hourly history tier, both + interval-identity tiers and the seven baselines. Zero here is a filter that matched nothing. */ + Assert.Equal(14, checkedTiers); + } + + /// + /// THE DAILY BAND (#3581): its minute is one no hourly-grid member starts on, past the recorded ceiling of + /// every refresh that could still be running, derived from the grid's geometry rather than from the ceiling; + /// its hours are one per aggregate, distinct, off the midnight hour, and inside the day. + /// + /// The non-collision claim is asserted against the shipped grid's OUTPUT — the refresh minutes through + /// and the compression minutes through + /// — not against a copy of either rule, so a + /// re-derived grid that moved onto this minute is red here even though every other grid pin still + /// passes. + /// + [Fact] + public void TheDailyBand_SitsOnAMinuteNoHourlyGridMemberUses_PastEveryRecordedCeiling_OneAggregatePerHour() + { + var minute = TimescaleSupport.AggregateCompressionBandMinute; + + /* Geometry, not the ceiling: the last minute of the heaviest refresh's window. */ + Assert.Equal( + TimescaleSupport.HeaviestRefreshStartMinute + TimescaleSupport.HeaviestRefreshWindowMinutes - 1, + minute); + Assert.Equal(35, minute); + + /* No hourly refresh starts on it, no raw compression policy starts on it, and it is the minute + immediately before the raw compression band opens — the tiling identity from the daily band's side. */ + var refreshMinutes = TimescaleSupport.HourlyRefreshPhaseOrder.Select(TimescaleSupport.RefreshPhaseMinutesFor).ToArray(); + Assert.DoesNotContain(minute, refreshMinutes); + Assert.DoesNotContain(minute, TimescaleSupport.CompressionPhaseMinutes); + Assert.Equal(TimescaleSupport.CompressionPhaseMinutes[0], minute + 1); + + /* Past the recorded ceiling of the heaviest refresh, with the margin stated: 1,200 s after its start + against 896 s. The grid asserts the ceiling fits the window; this asserts the band sits past the + ceiling inside that window, and a ceiling that grew to meet it fails here. */ + var secondsPastHeaviest = (minute - TimescaleSupport.HeaviestRefreshStartMinute) * 60; + Assert.Equal(1200, secondsPastHeaviest); + Assert.True( + secondsPastHeaviest > TimescaleSupport.HeaviestHourlyRefreshObservedCeilingSeconds, + $"the daily band's minute is {secondsPastHeaviest}s past the heaviest refresh's start against a " + + $"{TimescaleSupport.HeaviestHourlyRefreshObservedCeilingSeconds}s recorded ceiling — a once-a-day rewrite would " + + "start inside the heaviest refresh's tail"); + + /* And past every light refresh's ceiling, measured from each one's own start. */ + foreach (var view in TimescaleSupport.HourlyRefreshPhaseOrder) + { + if (string.Equals(view, TimescaleSupport.HeaviestHourlyRefreshView, StringComparison.Ordinal)) + { + continue; + } + + var secondsPast = (minute - TimescaleSupport.RefreshPhaseMinutesFor(view)) * 60; + Assert.True( + secondsPast > TimescaleSupport.OtherHourlyRefreshObservedCeilingSeconds, + $"{view}'s refresh could still be running at :{minute:00} against its {TimescaleSupport.OtherHourlyRefreshObservedCeilingSeconds}s ceiling"); + } + + /* The run has the rest of the hour to the next refresh START — the relation-agnostic clearance the + raw band's watch is measured in, so the two bands are stated in one unit. */ + Assert.Equal(TimescaleSupport.MinutesInHourlyCadence - minute, TimescaleSupport.CompressionMinuteClearanceMinutes(minute)); + Assert.Equal(25, TimescaleSupport.CompressionMinuteClearanceMinutes(minute)); + + /* THE HOURS: one per aggregate in registry order from hour 1, distinct, never the midnight hour, all + inside the day. Asserted as identities against the registry so a twenty-first aggregate is placed + without editing this, and as a fit so a twenty-fourth is red rather than wrapped onto midnight. */ + Assert.Equal(1, TimescaleSupport.AggregateCompressionBandFirstHour); + Assert.Equal(24, TimescaleSupport.HoursInDailyCadence); + Assert.Equal(TimeSpan.FromDays(1), TimescaleSupport.AggregateCompressionScheduleSpan); + Assert.Equal("1 day", TimescaleSupport.AggregateCompressionScheduleInterval); + + var hours = TimescaleSupport.AggregateCompressionTargets + .Select(t => TimescaleSupport.AggregateCompressionBandHourFor(t.View)) + .ToArray(); + + Assert.Equal(TimescaleSupport.AggregateCompressionTargets.Count, hours.Distinct().Count()); + Assert.DoesNotContain(0, hours); + Assert.All(hours, hour => Assert.InRange(hour, TimescaleSupport.AggregateCompressionBandFirstHour, TimescaleSupport.HoursInDailyCadence - 1)); + Assert.Equal( + Enumerable.Range(TimescaleSupport.AggregateCompressionBandFirstHour, TimescaleSupport.AggregateCompressionTargets.Count).ToArray(), + hours); + Assert.True( + TimescaleSupport.AggregateCompressionBandFirstHour + TimescaleSupport.AggregateCompressionTargets.Count <= TimescaleSupport.HoursInDailyCadence, + "the registry has outgrown the day at one aggregate per hour — re-derive the band rather than wrapping onto midnight"); + + Assert.Throws(() => TimescaleSupport.AggregateCompressionBandHourFor("query_stats")); + Assert.Throws(() => TimescaleSupport.AggregateCompressAfterSpanFor("collection_log")); + } + + /// + /// The statements carry what the derivations say, and only that: the tier's window, the daily cadence, the + /// server_id segment, the bucket order, if_not_exists, and a FIXED anchor computed in UTC at + /// the aggregate's hour on the band, the requested number of nights after the next UTC midnight. + /// + [Fact] + public void TheStatements_CarryTheTiersWindow_TheDailyCadence_AndAFixedUtcAnchorOnTheBand() + { + Assert.Equal( + "ALTER MATERIALIZED VIEW collect.query_stats_hourly SET (timescaledb.compress, timescaledb.compress_segmentby = 'server_id', timescaledb.compress_orderby = 'bucket DESC')", + TimescaleSupport.EnableAggregateCompressionSql(TimescaleSupport.QueryStatsHourlyView)); + + foreach (var (_, view, hourly) in TimescaleSupport.AggregateCompressionTargets) + { + var hour = TimescaleSupport.AggregateCompressionBandHourFor(view); + var night = hourly ? 3 : 0; + var sql = TimescaleSupport.AddAggregateCompressionPolicySql(view, night); + + Assert.StartsWith($"SELECT add_compression_policy('collect.{view}', ", sql, StringComparison.Ordinal); + Assert.Contains($"compress_after => INTERVAL '{(hourly ? "2 days" : "4 days")}'", sql, StringComparison.Ordinal); + Assert.Contains("schedule_interval => INTERVAL '1 day'", sql, StringComparison.Ordinal); + Assert.Contains("if_not_exists => true", sql, StringComparison.Ordinal); + Assert.Contains( + $"initial_start => (date_trunc('day', now() AT TIME ZONE 'UTC') + INTERVAL '1 day' + INTERVAL '{night} days' + INTERVAL '{hour} hours {TimescaleSupport.AggregateCompressionBandMinute} minutes') AT TIME ZONE 'UTC'", + sql, + StringComparison.Ordinal); + + /* UTC, never the session zone — the same trap AddContinuousAggregatePolicySql documents. */ + Assert.DoesNotContain("date_trunc('day', now())", sql, StringComparison.Ordinal); + + /* Never the raw tier's values. */ + Assert.DoesNotContain($"INTERVAL '{TimescaleSupport.CompressAfterDays} days'", sql, StringComparison.Ordinal); + Assert.DoesNotContain($"schedule_interval => INTERVAL '{TimescaleSupport.CompressScheduleInterval}'", sql, StringComparison.Ordinal); + } + + Assert.Throws(() => TimescaleSupport.AggregateCompressionInitialStartSql(24, 35, 0)); + Assert.Throws(() => TimescaleSupport.AggregateCompressionInitialStartSql(1, 60, 0)); + Assert.Throws(() => TimescaleSupport.AggregateCompressionInitialStartSql(1, 35, -1)); + + /* The converge: job id bound and cast (#1586), compress_after through jsonb_set against the job's own + config, the daily cadence, fixed schedule, an anchor at the bound hour and minute — and never + `scheduled`, so it can neither arm nor pause. */ + var converge = TimescaleSupport.SetAggregateCompressionPolicySql; + Assert.Contains("WHERE j.job_id = $1::integer", converge, StringComparison.Ordinal); + Assert.Contains("jsonb_set(j.config, '{compress_after}', to_jsonb($2::text))", converge, StringComparison.Ordinal); + Assert.Contains("schedule_interval => INTERVAL '1 day'", converge, StringComparison.Ordinal); + Assert.Contains("fixed_schedule => true", converge, StringComparison.Ordinal); + Assert.Contains("($3::int * INTERVAL '1 hour') + ($4::int * INTERVAL '1 minute')", converge, StringComparison.Ordinal); + Assert.DoesNotContain("scheduled =>", converge, StringComparison.Ordinal); + + /* The state read joins the job on EITHER identity (view or materialization), counts chunks on the + materialization at BOTH tiers' delays, and is scoped to collect. */ + var state = TimescaleSupport.AggregateCompressionStateSql; + Assert.Contains("j.hypertable_schema = ca.view_schema AND j.hypertable_name = ca.view_name", state, StringComparison.Ordinal); + Assert.Contains("j.hypertable_schema = ca.materialization_hypertable_schema AND j.hypertable_name = ca.materialization_hypertable_name", state, StringComparison.Ordinal); + Assert.Contains("c.hypertable_name = ca.materialization_hypertable_name", state, StringComparison.Ordinal); + Assert.Contains("now() - INTERVAL '2 days'", state, StringComparison.Ordinal); + Assert.Contains("now() - INTERVAL '4 days'", state, StringComparison.Ordinal); + Assert.Contains("WHERE ca.view_schema = 'collect'", state, StringComparison.Ordinal); + Assert.Contains("proc_name LIKE '%compression%'", state, StringComparison.Ordinal); + Assert.Contains("proc_name LIKE '%columnstore%'", state, StringComparison.Ordinal); + + /* #1778's activity read resolves the materialization for an aggregate's job and counts at the job's own + delay — the count that read zero forever for this family when keyed on the job's name. */ + var activity = TimescaleSupport.CompressionActivitySql; + Assert.Contains("COALESCE(ca.materialization_hypertable_name, j.hypertable_name)", activity, StringComparison.Ordinal); + Assert.Contains("COALESCE((j.config->>'compress_after')::interval", activity, StringComparison.Ordinal); + Assert.Contains("LEFT JOIN timescaledb_information.continuous_aggregates AS ca", activity, StringComparison.Ordinal); + Assert.Contains("AS compress_after_seconds", activity, StringComparison.Ordinal); + } + + /// + /// THE MATERIALIZATION CHUNK WIDTH (#3620) is one raw chunk, as a DERIVATION from + /// and not a written day: the span, the literal the + /// statement interpolates, and the statement itself all read from the constant, so the raw tables and their + /// rollups cannot be at different widths. The write resolves the materialization from the catalog by view + /// name (the internal _materialized_hypertable_N name is TimescaleDB's and differs per store), refuses + /// a view the registry does not carry, and the read joins dimensions on the materialization identity, + /// time dimension only, in seconds — the shape the ensure compares with integer equality. + /// + [Fact] + public void MaterializationChunkInterval_IsOneRawChunk_DerivedFromChunkIntervalDays_AndTheStatementsResolveTheMaterialization() + { + Assert.Equal(TimeSpan.FromDays(TimescaleSupport.ChunkIntervalDays), TimescaleSupport.MaterializationChunkIntervalSpan); + Assert.Equal(TimescaleSupport.AggregateCompressMarginSpan, TimescaleSupport.MaterializationChunkIntervalSpan); + Assert.Equal(TimescaleSupport.WholeDaysInterval(TimeSpan.FromDays(TimescaleSupport.ChunkIntervalDays)), TimescaleSupport.MaterializationChunkInterval); + Assert.Equal($"{TimescaleSupport.ChunkIntervalDays} days", TimescaleSupport.MaterializationChunkInterval); + + /* Today's value, as what the expression evaluates to. */ + Assert.Equal(TimeSpan.FromDays(1), TimescaleSupport.MaterializationChunkIntervalSpan); + Assert.Equal("1 days", TimescaleSupport.MaterializationChunkInterval); + + /* The raw tables' own CREATE and the materializations' SET carry the SAME literal. */ + var raw = TimescaleSupport.CreateHypertableSql("collect.query_stats", "collected_at"); + Assert.Contains($"INTERVAL '{TimescaleSupport.MaterializationChunkInterval}'", raw, StringComparison.Ordinal); + + foreach (var (_, view, _) in TimescaleSupport.AggregateCompressionTargets) + { + var set = TimescaleSupport.SetMaterializationChunkIntervalSql(view); + Assert.Contains("SELECT set_chunk_time_interval(", set, StringComparison.Ordinal); + Assert.Contains("format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass", set, StringComparison.Ordinal); + Assert.Contains($"INTERVAL '{TimescaleSupport.MaterializationChunkInterval}'", set, StringComparison.Ordinal); + Assert.Contains($"WHERE ca.view_schema = 'collect' AND ca.view_name = '{view}'", set, StringComparison.Ordinal); + Assert.DoesNotContain("_materialized_hypertable", set, StringComparison.Ordinal); + } + + Assert.Throws(() => TimescaleSupport.SetMaterializationChunkIntervalSql("query_stats")); + Assert.Throws(() => TimescaleSupport.SetMaterializationChunkIntervalSql("not_an_aggregate")); + + var state = TimescaleSupport.MaterializationChunkIntervalStateSql; + Assert.Contains("EXTRACT(EPOCH FROM d.time_interval)::bigint", state, StringComparison.Ordinal); + Assert.Contains("JOIN timescaledb_information.dimensions AS d", state, StringComparison.Ordinal); + Assert.Contains("d.hypertable_schema = ca.materialization_hypertable_schema", state, StringComparison.Ordinal); + Assert.Contains("d.hypertable_name = ca.materialization_hypertable_name", state, StringComparison.Ordinal); + Assert.Contains("d.dimension_type = 'Time'", state, StringComparison.Ordinal); + Assert.Contains("WHERE ca.view_schema = 'collect'", state, StringComparison.Ordinal); + } + + /// + /// THE STAGING (#3581), pinned on a synthetic store: aggregates with a backlog take consecutive nights + /// largest first; aggregates with nothing eligible take night zero whatever their size; a fresh store — + /// nothing eligible anywhere — is therefore all night zero with no special case; size ties break on + /// registry order so the result is deterministic. + /// + [Fact] + public void Staging_OrdersTheBacklogLargestFirstOnePerNight_AndPutsEverythingWithoutOneOnNightZero() + { + static TimescaleSupport.AggregateCompressionState State(string view, long bytes, long eligible) => + new(view, CompressionEnabled: true, JobId: null, CompressAfterSeconds: null, ScheduleIntervalSeconds: null, + FixedSchedule: false, PhaseHour: null, PhaseMinute: null, MaterializationBytes: bytes, EligibleChunksNow: eligible); + + var gib = 1L << 30; + var staged = TimescaleSupport.StageAggregateCompressionNights(new[] + { + State(TimescaleSupport.QueryStatsHourlyView, 3 * gib, 30), + State(TimescaleSupport.QueryStoreStatsHourlyView, 55 * gib, 30), + State(TimescaleSupport.QueryStoreStatsIntervalHourlyView, 71 * gib, 5), + State(TimescaleSupport.QueryStoreStatsCorrectedHourlyView, 54 * gib, 30), + /* Large but nothing eligible yet: a store that materialized this one recently. Night zero. */ + State(TimescaleSupport.QueryStoreStatsIntervalDailyView, 33 * gib, 0), + /* Empty on this store (no writable Query Store primary): night zero. */ + State(TimescaleSupport.QueryStoreStatsDailyView, 0, 0), + State(TimescaleSupport.PerfmonBaselineView, 0, 0), + }); + + Assert.Equal( + new[] + { + (TimescaleSupport.QueryStoreStatsIntervalHourlyView, 0), + (TimescaleSupport.QueryStoreStatsHourlyView, 1), + (TimescaleSupport.QueryStoreStatsCorrectedHourlyView, 2), + (TimescaleSupport.QueryStatsHourlyView, 3), + /* Night zero, in REGISTRY order rather than input order — the daily (registry position 8) + ahead of the interval daily (11) ahead of the baseline (13). */ + (TimescaleSupport.QueryStoreStatsDailyView, 0), + (TimescaleSupport.QueryStoreStatsIntervalDailyView, 0), + (TimescaleSupport.PerfmonBaselineView, 0), + }, + staged.ToArray()); + + /* Eligible-chunk COUNT does not order the backlog — size does. The 71 GiB aggregate with five eligible + chunks went first, ahead of three with thirty. The count is what the first run has to do; the bytes + are what it costs and saves. */ + Assert.Equal(TimescaleSupport.QueryStoreStatsIntervalHourlyView, staged[0].View); + + /* A fresh store: every aggregate on night zero, in registry order, and every policy created at once. */ + var fresh = TimescaleSupport.StageAggregateCompressionNights( + TimescaleSupport.AggregateCompressionTargets.Select(t => State(t.View, 8192, 0)).Reverse().ToArray()); + Assert.All(fresh, s => Assert.Equal(0, s.NightOffset)); + Assert.Equal(TimescaleSupport.AggregateCompressionTargets.Select(t => t.View).ToArray(), fresh.Select(s => s.View).ToArray()); + + /* Ties on size break on registry order, not on input order. */ + var tied = TimescaleSupport.StageAggregateCompressionNights(new[] + { + State(TimescaleSupport.ProcedureStatsHourlyView, gib, 3), + State(TimescaleSupport.QueryStatsHourlyView, gib, 3), + }); + Assert.Equal(TimescaleSupport.QueryStatsHourlyView, tied[0].View); + Assert.Equal(0, tied[0].NightOffset); + Assert.Equal(1, tied[1].NightOffset); + + Assert.Empty(TimescaleSupport.StageAggregateCompressionNights(Array.Empty())); + } + + /// + /// The predicate the raw compression converge excludes on covers exactly the twenty aggregates — bare or + /// collect.-qualified — and none of the raw hypertables, so the raw converge keeps #1778's reach + /// over every hypertable it had and gains no reach over this family. + /// + [Fact] + public void IsAggregateCompressionTarget_CoversEveryAggregate_AndNoRawHypertable() + { + foreach (var (_, view, _) in TimescaleSupport.AggregateCompressionTargets) + { + Assert.True(TimescaleSupport.IsAggregateCompressionTarget(view), view); + Assert.True(TimescaleSupport.IsAggregateCompressionTarget("collect." + view), view); + } + + foreach (var table in TimescaleSupport.CompressionPhaseOrder) + { + Assert.False(TimescaleSupport.IsAggregateCompressionTarget(table), table); + } + + Assert.False(TimescaleSupport.IsAggregateCompressionTarget(null)); + Assert.False(TimescaleSupport.IsAggregateCompressionTarget(string.Empty)); + Assert.False(TimescaleSupport.IsAggregateCompressionTarget(TimescaleSupport.CollectionLogTable)); + + /* The two registries are disjoint by name — a raw hypertable and an aggregate can never share one, so + the exclusion can never hide a raw table from its own converge. */ + Assert.Empty(TimescaleSupport.CompressionPhaseOrder.Intersect( + TimescaleSupport.AggregateCompressionTargets.Select(t => t.View), StringComparer.Ordinal)); + } + + /// + /// The ensure against a real TimescaleDB store (gated on DARLING_TEST_PG): the aggregates are + /// created, the ensure runs, and afterwards every registered aggregate reads + /// compression_enabled = true with exactly one compression job on the daily cadence at its tier's + /// compress_after, pinned to its hour on the band at :35. A second pass adds nothing and says + /// so. The raw compression converge, run afterwards, moves nothing — the family is excluded rather than + /// retuned to the hourly tick — and #1778's activity read sees the jobs at their own delay. + /// + /// Restores the fixture's shape (#1873): the aggregates this test creates are dropped afterwards + /// through , and their compression policies go + /// with them — TimescaleDB removes every job on a dropped aggregate (measured on 2.28.1: zero jobs remain + /// for a dropped, compression-enabled aggregate). Nothing compresses during the run: the policies' first + /// runs are anchored to the next UTC midnight at the earliest, and the fixture holds no chunk two days + /// old. Snapshot what already exists and drop only what this test created, the TimescaleSupportTests + /// idiom. + /// + [Fact] + public async Task EndToEnd_AggregateCompression_EnablesEveryAggregate_AttachesOneDailyPolicyEach_AndTheRawConvergeLeavesThemAlone_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live aggregate-compression test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + + var preexistingCaggs = await ExistingCaggsAsync(connection, ct); + + var bodySucceeded = false; + try + { + /* The aggregates have to exist, over hypertables, before anything can be compressed on them — the same + ordering the worker's TimescaleDB block runs. */ + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await TimescaleSupport.ConvergeContinuousAggregateRefreshAsync(connection, null, ct); + var created = await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + Assert.Equal(TimescaleSupport.AggregateCompressionTargets.Count, created); + + /* The widths the store gave the fresh materializations, before the ensure narrows them: on 2.28.1 + every one reads ten raw chunks (hierarchical ones take their parent's, which is already ten). Read + so the change count below is asserted against what was actually wide, not against a version fact. */ + var wideBefore = (await MaterializationChunkIntervalSecondsAsync(connection, ct)) + .Count(kv => kv.Value != (long)TimescaleSupport.MaterializationChunkIntervalSpan.TotalSeconds); + + var firstLog = new CapturingTestLogger(); + var first = await TimescaleSupport.EnsureAggregateCompressionAsync(connection, firstLog, ct); + Assert.Equal(TimescaleSupport.AggregateCompressionTargets.Count, first); + + /* THE WIDTH (#3620): every registered materialization's time dimension reads one raw chunk after the + ensure, from the catalog; the summary line says how many the start changed, and it is the number + that were wide. */ + var widths = await MaterializationChunkIntervalSecondsAsync(connection, ct); + foreach (var (_, view, _) in TimescaleSupport.AggregateCompressionTargets) + { + Assert.True(widths.TryGetValue(view, out var seconds), $"{view} has no time dimension on its materialization"); + Assert.Equal((long)TimescaleSupport.MaterializationChunkIntervalSpan.TotalSeconds, seconds); + } + + Assert.Contains($"20/20 materializations chunked at {TimescaleSupport.MaterializationChunkInterval}", firstLog.Joined, StringComparison.Ordinal); + Assert.Contains($"{wideBefore} changed this start", firstLog.Joined, StringComparison.Ordinal); + + /* A settled store issues no set_chunk_time_interval at all: the direct call returns zero changes and + says so, and the widths are what they were. */ + var widthLog = new CapturingTestLogger(); + Assert.Equal(0, await TimescaleSupport.EnsureMaterializationChunkIntervalAsync(connection, widthLog, ct)); + Assert.Contains("0 changed this start", widthLog.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("materialization now chunks at", widthLog.Joined, StringComparison.Ordinal); + Assert.Equal(widths, await MaterializationChunkIntervalSecondsAsync(connection, ct)); + + /* Every aggregate: compression enabled, exactly one compression job, at its tier's window, on the daily + cadence, on a fixed schedule at its hour and the band's minute. Read back from the catalog, not from + the ensure's return value. */ + foreach (var (_, view, hourly) in TimescaleSupport.AggregateCompressionTargets) + { + using var read = new NpgsqlCommand(@" + SELECT + ca.compression_enabled, + count(j.job_id), + min(EXTRACT(EPOCH FROM (j.config->>'compress_after')::interval)::bigint), + min(EXTRACT(EPOCH FROM j.schedule_interval)::bigint), + bool_and(j.fixed_schedule), + min(EXTRACT(HOUR FROM j.initial_start AT TIME ZONE 'UTC')::int), + min(EXTRACT(MINUTE FROM j.initial_start AT TIME ZONE 'UTC')::int) + FROM timescaledb_information.continuous_aggregates AS ca + LEFT JOIN timescaledb_information.jobs AS j + ON j.proc_name LIKE '%compression%' + AND j.hypertable_schema = ca.view_schema + AND j.hypertable_name = ca.view_name + WHERE ca.view_schema = 'collect' AND ca.view_name = $1 + GROUP BY ca.compression_enabled", connection); + read.Parameters.AddWithValue(view); + using var reader = await read.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), $"{view} is not a continuous aggregate on the fixture"); + + Assert.True(reader.GetBoolean(0), $"{view} is not compression-enabled"); + Assert.Equal(1L, reader.GetInt64(1)); + Assert.Equal((long)(hourly ? TimescaleSupport.HourlyAggregateCompressAfterSpan : TimescaleSupport.DailyAggregateCompressAfterSpan).TotalSeconds, reader.GetInt64(2)); + Assert.Equal((long)TimescaleSupport.AggregateCompressionScheduleSpan.TotalSeconds, reader.GetInt64(3)); + Assert.True(reader.GetBoolean(4), $"{view}'s compression job is not on a fixed schedule"); + Assert.Equal(TimescaleSupport.AggregateCompressionBandHourFor(view), reader.GetInt32(5)); + Assert.Equal(TimescaleSupport.AggregateCompressionBandMinute, reader.GetInt32(6)); + } + + /* The summary line names both windows and the band, and is rendered — a placeholder/argument + mismatch renders wrong with no error anywhere, which no return-value assertion can catch. */ + Assert.Contains("continuous-aggregate compression on 20/20 aggregates", firstLog.Joined, StringComparison.Ordinal); + Assert.Contains($"compress_after {TimescaleSupport.HourlyAggregateCompressAfter} for the hourly-refreshed tier", firstLog.Joined, StringComparison.Ordinal); + Assert.Contains($"{TimescaleSupport.DailyAggregateCompressAfter} for the daily tier", firstLog.Joined, StringComparison.Ordinal); + Assert.Contains($"minute :{TimescaleSupport.AggregateCompressionBandMinute:00}Z, from hour {TimescaleSupport.AggregateCompressionBandFirstHour:00}Z", firstLog.Joined, StringComparison.Ordinal); + + /* Idempotent: the settled store adds nothing, converges nothing, and still reports the full ladder. */ + var secondLog = new CapturingTestLogger(); + var second = await TimescaleSupport.EnsureAggregateCompressionAsync(connection, secondLog, ct); + Assert.Equal(first, second); + Assert.Contains("(0 added this start, 0 converged", secondLog.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("gets a once-a-day compression policy", secondLog.Joined, StringComparison.Ordinal); + Assert.Contains("0 changed this start", secondLog.Joined, StringComparison.Ordinal); + + /* THE EXCLUSION: the raw compression converge sees these once-a-day jobs in its unscoped read and must + leave every one of them on the daily cadence. Asserted on the catalog after the converge, not only on + its count — a count of zero is also what a converge that read nothing returns. */ + var rawConvergeLog = new CapturingTestLogger(); + await TimescaleSupport.ConvergeCompressionScheduleAsync(connection, rawConvergeLog, ct); + Assert.DoesNotContain("retuned query_stats_hourly's compression policy", rawConvergeLog.Joined, StringComparison.Ordinal); + + using (var cadences = new NpgsqlCommand(@" + SELECT count(*) + FROM timescaledb_information.jobs AS j + JOIN timescaledb_information.continuous_aggregates AS ca + ON ca.view_schema = j.hypertable_schema AND ca.view_name = j.hypertable_name + WHERE j.proc_name LIKE '%compression%' + AND ca.view_schema = 'collect' + AND j.schedule_interval = INTERVAL '1 day'", connection)) + { + Assert.Equal((long)TimescaleSupport.AggregateCompressionTargets.Count, (long)(await cadences.ExecuteScalarAsync(ct))!); + } + + /* #1778's activity read covers the family at its own delay. */ + var activity = await TimescaleSupport.ReadCompressionActivityAsync(connection, null, ct); + var aggregateActivity = activity.Where(a => TimescaleSupport.IsAggregateCompressionTarget(a.HypertableName)).ToArray(); + Assert.Equal(TimescaleSupport.AggregateCompressionTargets.Count, aggregateActivity.Length); + foreach (var item in aggregateActivity) + { + Assert.Equal(TimescaleSupport.AggregateCompressAfterSpanFor(item.HypertableName!), item.CompressAfter); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await new LiveCleanupBatch(cleanup).DropContinuousAggregatesAsync( + (await ExistingCaggsAsync(cleanup, cleanupCt)).Except(preexistingCaggs, StringComparer.Ordinal), cleanupCt)); + } + } + + /// + /// A drifted aggregate policy — one an earlier build could have left on different values — is converged + /// onto the shipped window, cadence and band instant on the next ensure, and the ensure after that finds + /// nothing. Live, because the whole point is what alter_job does to a real job row. + /// + [Fact] + public async Task EndToEnd_AggregateCompression_ConvergesADriftedPolicy_ThenSettles_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live aggregate-compression converge test."); + + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), + "the dev fixture is expected to have TimescaleDB installed"); + + var preexistingCaggs = await ExistingCaggsAsync(connection, ct); + + var bodySucceeded = false; + try + { + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await TimescaleSupport.ConvergeContinuousAggregateRefreshAsync(connection, null, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + await TimescaleSupport.EnsureAggregateCompressionAsync(connection, null, ct); + + var view = TimescaleSupport.ProcedureStatsHourlyView; + + /* Drift one policy the way an older build would have left it: the raw tier's window and tick, an + anchor off the band. */ + using (var drift = new NpgsqlCommand($@" + SELECT alter_job( + j.job_id, + schedule_interval => INTERVAL '{TimescaleSupport.CompressScheduleInterval}', + config => jsonb_set(j.config, '{{compress_after}}', to_jsonb('{TimescaleSupport.CompressAfterDays} days'::text)), + fixed_schedule => false) + FROM timescaledb_information.jobs AS j + WHERE j.proc_name LIKE '%compression%' AND j.hypertable_schema = 'collect' AND j.hypertable_name = '{view}'", connection)) + { + Assert.NotNull(await drift.ExecuteScalarAsync(ct)); + } + + var convergeLog = new CapturingTestLogger(); + await TimescaleSupport.EnsureAggregateCompressionAsync(connection, convergeLog, ct); + Assert.Contains($"moved {view}'s compression policy", convergeLog.Joined, StringComparison.Ordinal); + Assert.Contains("1 converged", convergeLog.Joined, StringComparison.Ordinal); + + using (var read = new NpgsqlCommand($@" + SELECT + EXTRACT(EPOCH FROM (j.config->>'compress_after')::interval)::bigint, + EXTRACT(EPOCH FROM j.schedule_interval)::bigint, + j.fixed_schedule, + EXTRACT(HOUR FROM j.initial_start AT TIME ZONE 'UTC')::int, + EXTRACT(MINUTE FROM j.initial_start AT TIME ZONE 'UTC')::int + FROM timescaledb_information.jobs AS j + WHERE j.proc_name LIKE '%compression%' AND j.hypertable_schema = 'collect' AND j.hypertable_name = '{view}'", connection)) + { + using var reader = await read.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + Assert.Equal((long)TimescaleSupport.HourlyAggregateCompressAfterSpan.TotalSeconds, reader.GetInt64(0)); + Assert.Equal((long)TimescaleSupport.AggregateCompressionScheduleSpan.TotalSeconds, reader.GetInt64(1)); + Assert.True(reader.GetBoolean(2)); + Assert.Equal(TimescaleSupport.AggregateCompressionBandHourFor(view), reader.GetInt32(3)); + Assert.Equal(TimescaleSupport.AggregateCompressionBandMinute, reader.GetInt32(4)); + } + + var settledLog = new CapturingTestLogger(); + await TimescaleSupport.EnsureAggregateCompressionAsync(connection, settledLog, ct); + Assert.Contains("0 converged", settledLog.Joined, StringComparison.Ordinal); + Assert.DoesNotContain("moved ", settledLog.Joined, StringComparison.Ordinal); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + await new LiveCleanupBatch(cleanup).DropContinuousAggregatesAsync( + (await ExistingCaggsAsync(cleanup, cleanupCt)).Except(preexistingCaggs, StringComparer.Ordinal), cleanupCt)); + } + } + + /// Every collect aggregate's materialization chunk interval in seconds, read from + /// timescaledb_information.dimensions through the SAME join the ensure uses — the catalog, not the + /// ensure's return value, is what the width assertions read. + private static async Task> MaterializationChunkIntervalSecondsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) + { + using var command = new NpgsqlCommand(TimescaleSupport.MaterializationChunkIntervalStateSql, connection); + using var reader = await command.ExecuteReaderAsync(ct); + var widths = new Dictionary(StringComparer.Ordinal); + while (await reader.ReadAsync(ct)) + { + if (!reader.IsDBNull(1)) + { + widths[reader.GetString(0)] = reader.GetInt64(1); + } + } + + return widths; + } + + /// The continuous aggregates standing in collect right now — the snapshot the restore + /// diffs against, so a test drops only what it created (the TimescaleSupportTests idiom). + private static async Task ExistingCaggsAsync(NpgsqlConnection connection, System.Threading.CancellationToken ct) + { + using var command = new NpgsqlCommand( + "SELECT view_name FROM timescaledb_information.continuous_aggregates WHERE view_schema = 'collect'", connection); + using var reader = await command.ExecuteReaderAsync(ct); + var names = new List(); + while (await reader.ReadAsync(ct)) + { + names.Add(reader.GetString(0)); + } + + return names.ToArray(); + } + + private static TimeSpan ParseDays(string interval) + { + var parts = interval.Split(' ', StringSplitOptions.RemoveEmptyEntries); + Assert.Equal(2, parts.Length); + Assert.Equal("days", parts[1]); + return TimeSpan.FromDays(int.Parse(parts[0], CultureInfo.InvariantCulture)); + } +} diff --git a/Darling/Darling.Tests/TimescaleSupportTests.cs b/Darling/Darling.Tests/TimescaleSupportTests.cs index dd8ab8586..4488ca8e2 100644 --- a/Darling/Darling.Tests/TimescaleSupportTests.cs +++ b/Darling/Darling.Tests/TimescaleSupportTests.cs @@ -164,7 +164,11 @@ managed store compact (#1458). */ public void IsCompressionJobStuck_NextStartNegativeInfinity_IsStuck() { /* The dominant failure mode: next_start = -infinity on a job that is NOT running — the scheduler - abandoned it and never re-fires it. */ + abandoned it and never re-fires it. ONE read says so here, and one read is what the predicate + judges; since #3575 the reader (ReadStuckCompressionJobsAsync) asks twice five seconds apart before + it believes this arm, because the view assembles "not running" and "-infinity" from independent + sources and reads this exact shape for a few milliseconds at either edge of every healthy run. + The predicate itself stays single-shot — CompressionStuckConfirmReadTests pins the second read. */ Assert.True(TimescaleSupport.IsCompressionJobStuck( nextStartIsNegativeInfinity: true, jobStatus: "Scheduled", lastRunStartedAtUtc: null, scheduleInterval: TimeSpan.FromHours(12), nowUtc: s_now, out var reason)); @@ -178,7 +182,13 @@ public void IsCompressionJobStuck_NegativeInfinityWhileRunning_IsTheMidRunMarker -infinity WITH job_status = 'Running' — the engine only computes the real next start when the run finishes. An unconditioned -infinity arm flagged every healthy job caught mid-run (the field's transient stuck→self-healed alert noise, and the CI flake where the live test caught - its own re-arm-triggered run). Mid-run belongs to the elapsed-bound arm: */ + its own re-arm-triggered run). Mid-run belongs to the elapsed-bound arm. + + This guard was necessary and was not sufficient (#3575): 'Running' is pg_stat_activity, read + live, and -infinity is the bgw_job_stat row, read under the statement's snapshot, so the guard + is blind for the milliseconds between the scheduler committing -infinity and the worker + reporting itself active, and again between the worker leaving and its mark_end becoming + visible. That is the reader's problem to close (it re-reads), not this predicate's: */ Assert.False(TimescaleSupport.IsCompressionJobStuck( nextStartIsNegativeInfinity: true, jobStatus: "Running", lastRunStartedAtUtc: s_now.AddMinutes(-3), scheduleInterval: TimeSpan.FromHours(12), nowUtc: s_now, out _)); @@ -257,6 +267,131 @@ the point is that last_run_started_at is never read raw. */ TimescaleSupport.StuckCompressionJobsSql, StringComparison.Ordinal); } + /* ---------------- the -infinity arm's sentence, by TimescaleDB version (#3591) ---------------- */ + + [Theory] + [InlineData(null)] + [InlineData("2.14.2")] + [InlineData("2.26.0")] + [InlineData("2.26.3")] + public void NegInfinityArm_BelowTheFix_OrUnknown_SaysTheSchedulerWillNeverRunIt(string? extversion) + { + /* Below 2.26.4 a persisted -infinity is returned to the scheduler as the due time and the job is never + due again — #1581's sentence, byte for byte, because it is true there. An unknown version (null: the + read failed, or the pre-#3591 callers) gets the same text on purpose: it is the sentence that costs + nothing when wrong on a new store and everything when wrong on an old one. */ + var version = TimescaleSupport.ParseTimescaleVersion(extversion); + Assert.False(TimescaleSupport.SchedulerRecoversNegativeInfinity(version)); + + var arm = TimescaleSupport.ClassifyCompressionJob( + nextStartIsNegativeInfinity: true, jobStatus: "Scheduled", lastRunStartedAtUtc: s_now.AddHours(-1), + scheduleInterval: TimeSpan.FromHours(1), nowUtc: s_now, timescaleVersion: version, out var reason); + + Assert.Equal(StuckCompressionJobArm.NextStartNegativeInfinity, arm); + Assert.Equal("next_start is -infinity — the scheduler will never run it again", reason); + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityPermanentReason, reason); + } + + [Theory] + [InlineData("2.26.4")] + [InlineData("2.27.0")] + [InlineData("2.28.1")] + [InlineData("2.29.0-dev")] + public void NegInfinityArm_FromTheFix_SaysCrashBackoff_AndStillFires(string extversion) + { + /* Upstream #9360 shipped in 2.26.4 (the CHANGELOG lists it there, not under 2.27.0), so from 2.26.4 the + only persistent -infinity is a crashed run in the scheduler's crash backoff. Same arm — the verdict + does not move — different sentence, naming the fix so the reader knows where the claim comes from. */ + var version = TimescaleSupport.ParseTimescaleVersion(extversion); + Assert.NotNull(version); + Assert.True(TimescaleSupport.SchedulerRecoversNegativeInfinity(version)); + + var arm = TimescaleSupport.ClassifyCompressionJob( + nextStartIsNegativeInfinity: true, jobStatus: "Scheduled", lastRunStartedAtUtc: s_now.AddHours(-1), + scheduleInterval: TimeSpan.FromHours(1), nowUtc: s_now, timescaleVersion: version, out var reason); + + Assert.Equal(StuckCompressionJobArm.NextStartNegativeInfinity, arm); + Assert.Equal(TimescaleSupport.NextStartNegativeInfinityCrashBackoffReason, reason); + Assert.Contains("crash backoff", reason, StringComparison.Ordinal); + Assert.Contains("#9360", reason, StringComparison.Ordinal); + Assert.Contains("2.26.4", reason, StringComparison.Ordinal); + Assert.DoesNotContain("never", reason, StringComparison.Ordinal); + + /* And the boolean projection with the version agrees with the classifier. */ + Assert.True(TimescaleSupport.IsCompressionJobStuck( + nextStartIsNegativeInfinity: true, jobStatus: "Scheduled", lastRunStartedAtUtc: s_now.AddHours(-1), + scheduleInterval: TimeSpan.FromHours(1), nowUtc: s_now, timescaleVersion: version, out var boolReason)); + Assert.Equal(reason, boolReason); + } + + [Fact] + public void NegInfinityArm_TheVersionChangesTheSentenceOnly_NeverTheVerdict() + { + /* Every row shape the predicate knows, on both sides of the fix: the arm is identical, and only the + -infinity arm's text differs. The stuck-Running arm did not change upstream and reads the version + for nothing. */ + var old = new Version(2, 26, 3); + var fixedVersion = new Version(2, 28, 1); + foreach (var (negInf, status, started) in new (bool, string, DateTime?)[] + { + (true, "Scheduled", null), /* the dead-job / crash-backoff arm */ + (true, "Running", s_now.AddMinutes(-3)), /* mid-run marker */ + (true, "Running", s_now.AddHours(-30)), /* hung run carrying the marker */ + (false, "Scheduled", s_now.AddMinutes(-5)), /* healthy */ + (false, "Running", s_now.AddHours(-8)), /* hung run */ + (false, "Running", DateTime.MinValue), /* #1760 sentinel */ + }) + { + var armOld = TimescaleSupport.ClassifyCompressionJob(negInf, status, started, TimeSpan.FromHours(1), s_now, old, out var reasonOld); + var armNew = TimescaleSupport.ClassifyCompressionJob(negInf, status, started, TimeSpan.FromHours(1), s_now, fixedVersion, out var reasonNew); + var armNone = TimescaleSupport.ClassifyCompressionJob(negInf, status, started, TimeSpan.FromHours(1), s_now, out var reasonNone); + + Assert.Equal(armOld, armNew); + Assert.Equal(armOld, armNone); + Assert.Equal(reasonOld, reasonNone); /* version-less IS the old text */ + if (armOld == StuckCompressionJobArm.NextStartNegativeInfinity) + { + Assert.NotEqual(reasonOld, reasonNew); + } + else + { + Assert.Equal(reasonOld, reasonNew); + } + } + } + + [Fact] + public void TimescaleNextStartSanitizedFrom_Is_2_26_4() + { + /* Pinned to the release the upstream CHANGELOG lists #9360 under. The issue and the first brief said + 2.27.0; a check keyed there would have told every 2.26.4–2.26.x store the old lie. */ + Assert.Equal(new Version(2, 26, 4), TimescaleSupport.TimescaleNextStartSanitizedFrom); + } + + [Theory] + [InlineData("2.28.1", "2.28.1")] + [InlineData(" 2.26.4 ", "2.26.4")] + [InlineData("2.29.0-dev", "2.29.0")] + [InlineData("2.28", "2.28")] + public void ParseTimescaleVersion_TakesTheNumericPrefix(string raw, string expected) + { + Assert.Equal(Version.Parse(expected), TimescaleSupport.ParseTimescaleVersion(raw)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("2")] + [InlineData("dev")] + [InlineData("v2.28.1")] + public void ParseTimescaleVersion_AnythingElse_IsNull_WhichReadsAsOld(string? raw) + { + var parsed = TimescaleSupport.ParseTimescaleVersion(raw); + Assert.Null(parsed); + Assert.False(TimescaleSupport.SchedulerRecoversNegativeInfinity(parsed)); + } + [Fact] public void StuckRunningBound_UsesMaxOfTwiceIntervalAndFloor() { @@ -2051,7 +2186,13 @@ DETECTION logic is covered by the pure IsCompressionJobStuck unit tests. */ on one snapshot": next_start => now() makes the job immediately due, the scheduler picks it up, and from pickup to completion job_stats reads next_start = -infinity with status Running — the mid-run marker (measured live; the detector now defers that state to its elapsed-bound arm). A single - un-settled read raced the very run the re-arm triggered, which was this test's own flake. */ + un-settled read raced the very run the re-arm triggered, which was this test's own flake. + + Since #3575 the detector also re-reads five seconds later before it reports the -infinity arm, so + the run-instant EDGES (-infinity while the worker is not yet, or no longer, visible as Running — + the shape that paged a production store) clear inside one call rather than surfacing as a flagged + poll here. The wait stays: it is the assertion's contract, and a poll that lands on the edge now + costs five seconds of confirm rather than a flagged iteration. */ await WaitUntilDetectorReportsHealthyAsync(connection, jobId, ct); } diff --git a/Darling/Darling.Tests/TsqlConventionGuardTests.cs b/Darling/Darling.Tests/TsqlConventionGuardTests.cs index f47a3ecac..55d88ff6c 100644 --- a/Darling/Darling.Tests/TsqlConventionGuardTests.cs +++ b/Darling/Darling.Tests/TsqlConventionGuardTests.cs @@ -1439,6 +1439,19 @@ where the bound has nothing to compare against. */ "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs OutputFinding", "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStallProbeReader.cs TriggerMbPerSecond", "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStallProbeReader.cs TerminalSilenceMs", + /* #3582: two expression-bodied formatters of the same shape as the Figure/Count/Tally lines above. + Stamp strands the "o" round-trip format and "(unknown instant)"; Window strands the "0.##" format. + Neither is T-SQL and neither is a tempdb label, so no census reads a site of that kind here. */ + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs Stamp", + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs Window", + /* #3541 A12: four expression-bodied derivations on the growth row, each `Baseline is { } b ? … : null` + — the property pattern's braces are where the walk stops. Every one is arithmetic over the row's + own fields and strands NO string literal at all, so no census reads a site of that kind here. The + Lite twin (LocalDataService.FinOps.IndexObjects.cs, below) derives the same four the same way. */ + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs DailyGrowthRateMb", + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs Growth30dMb", + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs Growth7dMb", + "Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs GrowthOverAvailableHistoryMb", "Darling/PerformanceMonitor.Darling.Service/Targets/PostgresTargetProvider.cs WithDatabase", "Darling/PerformanceMonitor.Darling.Service/Targets/SqlServerTargetProvider.cs WithDatabase", "Darling/PerformanceMonitor.Darling.Viewer/MainWindow.ServerManagement.cs SelectedTabCollectorScope", @@ -1455,6 +1468,12 @@ where the bound has nothing to compare against. */ "Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.SystemEvents.cs Local", "Darling/PerformanceMonitor.Darling.Viewer/ViewerPostgresDisplay.cs Timestamp", "Lite/Services/LocalDataService.CollectionHealth.cs OutputFinding", + /* #3541 A12: the Lite twin of the four DarlingObjectStatsReader growth derivations above — the same + `is { } b ? … : null` shape, the same absence of any string literal. */ + "Lite/Services/LocalDataService.FinOps.IndexObjects.cs DailyGrowthRateMb", + "Lite/Services/LocalDataService.FinOps.IndexObjects.cs Growth30dMb", + "Lite/Services/LocalDataService.FinOps.IndexObjects.cs Growth7dMb", + "Lite/Services/LocalDataService.FinOps.IndexObjects.cs GrowthOverAvailableHistoryMb", ]; /* ───────────────────────── the resolver, pinned on arranged source ───────────────────────── */ diff --git a/Darling/Darling.Tests/UnmeasuredMetricsAreNotHealthyTests.cs b/Darling/Darling.Tests/UnmeasuredMetricsAreNotHealthyTests.cs index bf9256577..b4f911600 100644 --- a/Darling/Darling.Tests/UnmeasuredMetricsAreNotHealthyTests.cs +++ b/Darling/Darling.Tests/UnmeasuredMetricsAreNotHealthyTests.cs @@ -22,12 +22,19 @@ namespace Darling.Tests; /// The defect. A PostgreSQL target has no row in v_memory_grant_stats, /// v_blocked_process_reports / v_dmv_blocking_snapshots or v_deadlocks, so the /// per-metric reads handed its card zeros — and MemorySeverity(bool), -/// BlockingSeverity(int, double) and DeadlockSeverity(int) took non-nullable parameters, so +/// BlockingSeverity(int, double) and DeadlockSeverity(int) (their signatures at the time) took non-nullable parameters, so /// there was no way for a zero to mean "absent" rather than "calm". All three answered Healthy and painted a /// green dot. That is a positive claim of health, which is worse than the null beside it, and worse than /// what and /// already did by taking nullable inputs. /// +/// Deadlocks left the unmeasured set in #3539. A PostgreSQL target's deadlocks are now +/// counted from its own pg_stat_database.deadlocks counter (differenced over the window) and banded +/// through the shared rate tiers, so that row is a MEASUREMENT on both engines and this file asserts it +/// bands — Healthy at zero, Warning and Critical at the tiers — rather than reading Unknown. Memory pressure +/// and blocking stay unmeasured on PostgreSQL for the reasons gives, and +/// those two are what the Unknown pins below are about. +/// /// The invariant this is held to. A MEASURED metric must band exactly as it does today. That is /// the real constraint, and it is not the same as "leave the SQL Server path alone": the fix necessarily /// edits functions every SQL Server surface calls. is @@ -40,17 +47,41 @@ namespace Darling.Tests; /// existing code rather than a happy accident of this change, and /// pins it so an "improvement" that /// made Unknown escalate would fail here instead of silently reordering the fleet. +/// +/// #3539 A6, the two edges of the same family. The collectors row had a Healthy arm no other +/// metric here has: (failed 0, banded 0) — a server nothing had banded yet — read Healthy, a green +/// dot for a collection nobody had classified. And the fold over a card on which NOTHING was measured +/// answered Healthy, so an online server with six Unknowns counted in the fleet's healthy mass with a +/// "0 of 6 measured" qualifier as its only tell. Both now read Unknown, and the all-Unknown card bands +/// Warning — the never-collected server's band. Neutrality is unchanged wherever anything IS measured: +/// the pins below hold a one-of-six card exactly where #3528 left it. /// public sealed class UnmeasuredMetricsAreNotHealthyTests { private static readonly DateTime Now = new(2026, 9, 10, 12, 0, 0, DateTimeKind.Utc); + /// Collectors banded for this server, none failing — forty by default so + /// the card's collectors row is a MEASURED calm reading and the DMV metrics stay this file's only + /// variable. Zero is the #3539 A6 shape: nothing banded, nothing measured. + /// The SQL Server extended-event count — what a SQL Server card believes. + /// The PostgreSQL counter-difference count (#3539) — what a PostgreSQL card + /// believes. Both are always handed in so a test can assert the card took the ENGINE'S row and ignored + /// the other engine's structural zero. + /// How many differences that count was summed over — the PostgreSQL + /// arm's measured/not-measured test. Zero by default: a PostgreSQL card is UNMEASURED unless a test + /// says a difference was taken, which is the direction a forgotten argument must fail in. + /// The pg_database_stats collector's band, for the coverage + /// arm. private static FleetServerCard Card( string? engineKind, bool memoryPressure = false, int blocking = 0, long maxBlockingWaitMs = 0, - int deadlocks = 0) => + int deadlocks = 0, + int bandedCollectors = 40, + int pgDeadlocks = 0, + long pgDeadlockIntervals = 0, + string? pgDeadlockBand = null) => DarlingFleetReader.BuildCard( new DarlingFleetReader.FleetServerRow(1, "t", "t", null, engineKind, false), default, @@ -60,8 +91,9 @@ private static FleetServerCard Card( default, new DarlingFleetReader.BlockingRow(blocking, maxBlockingWaitMs, 0, 0), new DarlingFleetReader.DeadlockRow(deadlocks, deadlocks > 0 ? Now.AddMinutes(-5) : null), + new DarlingFleetReader.PgDeadlockRow(pgDeadlocks, pgDeadlocks > 0 ? Now.AddMinutes(-7) : null, pgDeadlockIntervals), Now.AddSeconds(-30), - default, + new DarlingFleetReader.CollectorCounts(bandedCollectors, 0, bandedCollectors, null, pgDeadlockBand), null, Now, /* #3368: a real one-hour window and the shipped tiers. This file's subject is the @@ -76,12 +108,15 @@ private static ServerSummaryItem ViewerCard( bool memoryPressure = false, int blocking = 0, long maxBlockingWaitMs = 0, - int deadlocks = 0) + int deadlocks = 0, + int bandedCollectors = 40) { var card = new ServerSummaryItem { ServerName = "t", ServerId = 1, + HealthyCollectorCount = bandedCollectors, + CollectorCount = bandedCollectors, MemoryWaiterCount = memoryPressure ? 3 : 0, BlockingCount = blocking, MaxBlockingWaitMs = maxBlockingWaitMs, @@ -90,6 +125,7 @@ private static ServerSummaryItem ViewerCard( subject is Postgres-vs-SQL-Server, so the window must not be the thing producing the Unknown. */ DeadlockWindow = TimeSpan.FromHours(1), + BlockingWindow = TimeSpan.FromHours(1), IsPostgres = MonitoredEngineKind.IsPostgres(engineKind), IsAurora = MonitoredEngineKind.IsAurora(engineKind), LastCollectionTime = Now.AddSeconds(-30), @@ -101,8 +137,12 @@ private static ServerSummaryItem ViewerCard( /* ─────────────────────────── the defect ─────────────────────────── */ /// - /// A PostgreSQL card's three DMV-sourced metrics read Unknown, not Healthy. Red against the unfixed - /// classifiers, which had no arm that could return anything else for a zero. + /// A PostgreSQL card's two DMV-sourced metrics read Unknown, not Healthy. Red against the unfixed + /// classifiers, which had no arm that could return anything else for a zero. Deadlocks read Unknown + /// here too, but for #3539's OWN reason rather than #3272's: the helper hands this card no counter + /// difference (zero intervals), and a difference of nothing is not a zero. The measured case — a + /// difference taken, zero deadlocks — is Healthy with a published 0.0/hr, and is asserted beside it so + /// the two readings of the same card cannot be confused. /// [Theory] [InlineData(MonitoredEngineKind.Postgres)] @@ -114,20 +154,26 @@ public void APostgresCard_ClaimsNoHealthForWhatItNeverMeasured(string engineKind Assert.Equal(HealthSeverity.Unknown, card.MemorySeverity); Assert.Equal(HealthSeverity.Unknown, card.BlockingSeverity); Assert.Equal(HealthSeverity.Unknown, card.DeadlockSeverity); - - /* And the RATE says the same thing the severity says. A rate of 0.0/hr on a target with no deadlock - source is a fabricated measurement, and both the card chip and the viewer detail line render this - field on non-null alone - so a structural zero here would contradict the Unknown above on the very - same card. */ Assert.Null(card.DeadlockRatePerHour); + Assert.False(card.DeadlockMeasured); /* Threads already reached Unknown on its own (a null ceiling), and CPU does since #3267. So after this the card makes NO unearned claim on any metric row - which is the property worth asserting, - rather than three separate arms that happen to agree today. */ + rather than separate arms that happen to agree today. */ Assert.Equal(HealthSeverity.Unknown, card.ThreadsSeverity); Assert.DoesNotContain( HealthSeverity.Healthy, new[] { card.MemorySeverity, card.BlockingSeverity, card.DeadlockSeverity, card.ThreadsSeverity, card.CpuSeverity }); + + /* #3539: ONE difference taken and the same zero is a measurement. Healthy is EARNED here - a zero + counter difference over a rateable hour - and the rate is published beside it, because the chip + and the viewer detail render the rate on non-null alone and a band with no rate beside it is the + #3368 defect. */ + var measured = Card(engineKind, pgDeadlockIntervals: 1); + Assert.True(measured.DeadlockMeasured); + Assert.Equal(HealthSeverity.Healthy, measured.DeadlockSeverity); + Assert.Equal(0.0, measured.DeadlockRatePerHour); + Assert.Equal(0, measured.DeadlockCount); } /// The viewer's card, same server, same answer — the #2473 rule. @@ -140,16 +186,24 @@ public void TheViewerCardAgrees(string engineKind) Assert.Equal(HealthSeverity.Unknown, card.MemorySeverity); Assert.Equal(HealthSeverity.Unknown, card.BlockingSeverity); + /* No difference taken (the helper leaves PgDeadlockIntervals at zero), so Unknown - and one + difference makes the same zero Healthy, as on the fleet card. */ Assert.Equal(HealthSeverity.Unknown, card.DeadlockSeverity); Assert.Null(card.DeadlockRatePerHour); + + var measured = ViewerCard(engineKind); + measured.PgDeadlockIntervals = 1; + Assert.Equal(HealthSeverity.Healthy, measured.DeadlockSeverity); + Assert.Equal(0.0, measured.DeadlockRatePerHour); } /// - /// The published COUNTS are deliberately unchanged, and #3017's disclosure still reads the same. The - /// fleet's total_deadlocks is summed from those zeros and deadlock_coverage is what - /// explains it; nulling them would leave that denominator describing nothing. So the band stopped - /// claiming health while the count kept saying what it counted — and the two now AGREE, where before - /// deadlock_source: PostgresTarget sat beside deadlock_severity: Healthy. + /// The published COUNTS are deliberately unchanged, and #3017's disclosure reads the collector state on + /// this engine too since #3539. The fleet's total_deadlocks is summed from the cards and + /// deadlock_coverage is what explains it; nulling the counts would leave that denominator + /// describing nothing. A PostgreSQL card whose pg_database_stats collector left no band is + /// UNCOVERED (silent), exactly as a SQL Server card with no deadlocks band is — the pre-#3539 + /// answer, PostgresTarget on the engine alone, would have called a server nothing read "counted". /// [Fact] public void TheCountsAndTheCoverageDisclosureAreUntouched() @@ -159,7 +213,86 @@ public void TheCountsAndTheCoverageDisclosureAreUntouched() Assert.Equal(0, card.BlockingCount); Assert.Equal(0, card.DeadlockCount); Assert.False(card.HasMemoryPressure); + Assert.Equal(FleetDeadlockSource.CollectorSilent, card.DeadlockSource); + + var counted = Card(MonitoredEngineKind.AuroraPostgres, pgDeadlockBand: CollectorHealthClassifier.Healthy); + Assert.Equal(FleetDeadlockSource.PostgresTarget, counted.DeadlockSource); + } + + /* ─────────────────────────── the PostgreSQL deadlock band (#3539) ─────────────────────────── */ + + /// + /// The PostgreSQL card bands its OWN count through the SAME tiers the SQL Server card uses: over the + /// helper's one-hour window the count is the rate, so 4 is under the shipped 5/hr Warning bar, 5 is + /// Warning, and 20 is the shipped Critical bar. The SQL Server row handed in alongside is a + /// structural zero for this engine and must be IGNORED — a card that summed the two would be right by + /// accident here and wrong the moment either read produced a non-zero for the wrong engine. + /// + [Theory] + [InlineData(0, HealthSeverity.Healthy)] + [InlineData(4, HealthSeverity.Healthy)] + [InlineData(5, HealthSeverity.Warning)] + [InlineData(19, HealthSeverity.Warning)] + [InlineData(20, HealthSeverity.Critical)] + public void APostgresCard_BandsItsCounterDifferenceThroughTheSharedTiers(int pgDeadlocks, HealthSeverity expected) + { + var card = Card(MonitoredEngineKind.Postgres, deadlocks: 999, pgDeadlocks: pgDeadlocks, + pgDeadlockIntervals: 59, pgDeadlockBand: CollectorHealthClassifier.Healthy); + + Assert.Equal(pgDeadlocks, card.DeadlockCount); + Assert.True(card.DeadlockMeasured); + Assert.Equal(expected, card.DeadlockSeverity); + Assert.Equal(pgDeadlocks, card.DeadlockRatePerHour); + Assert.Equal(pgDeadlocks > 0 ? Now.AddMinutes(-7) : null, card.DeadlockLastSeen); Assert.Equal(FleetDeadlockSource.PostgresTarget, card.DeadlockSource); + + /* The overall band follows, so the fleet's worst-first ranking sees a deadlocking PostgreSQL + server the way it sees a deadlocking SQL Server. */ + Assert.Equal(expected == HealthSeverity.Healthy ? FleetHealthBand.Healthy + : expected == HealthSeverity.Warning ? FleetHealthBand.Warning : FleetHealthBand.Critical, card.Band); + } + + /// The mirror image: a SQL Server card takes the extended-event row and ignores the PostgreSQL + /// row, and its collector band is the deadlocks collector's, not pg_database_stats'. + [Fact] + public void ASqlServerCard_IgnoresThePostgresRow() + { + var card = Card(MonitoredEngineKind.SqlServer, deadlocks: 2, pgDeadlocks: 999, pgDeadlockIntervals: 59, pgDeadlockBand: CollectorHealthClassifier.Healthy); + + Assert.Equal(2, card.DeadlockCount); + /* A SQL Server count is a measurement whatever the PostgreSQL row's interval count says - and + whatever its own collector band says, which is #3272's engine-not-collector rule. */ + Assert.True(card.DeadlockMeasured); + Assert.True(Card(MonitoredEngineKind.SqlServer).DeadlockMeasured); + Assert.Equal(Now.AddMinutes(-5), card.DeadlockLastSeen); + Assert.Equal(HealthSeverity.Healthy, card.DeadlockSeverity); + /* No deadlocks-collector band was handed in, so a SQL Server is silent whatever pg_database_stats + says about it. */ + Assert.Equal(FleetDeadlockSource.CollectorSilent, card.DeadlockSource); + } + + /// + /// #3528's label counts the row as measured on a PostgreSQL card now: the fold over the six metric + /// severities finds one more non-Unknown than it did, so "Healthy — N of 6 measured" moves by one on + /// every PostgreSQL card. The re-scoring bundle carries the count as a reading too, so the worst-first + /// rank sees the same measurement the dot does. + /// + [Fact] + public void ThePostgresCardsMeasuredMetricCountIncludesDeadlocks() + { + var card = Card(MonitoredEngineKind.Postgres, pgDeadlocks: 1, pgDeadlockIntervals: 59); + + /* CPU (no source), Threads, Memory and Blocking are Unknown; Deadlocks and Collectors (the + helper's forty) are measured. Written as the two numbers rather than "one more than before" so + a regression that un-measured a different row could not pass by coincidence. */ + Assert.Equal(2, card.MeasuredMetricCount); + Assert.Equal(6, card.MetricCount); + Assert.Equal((long?)1L, card.ToHealthMetricsValue.DeadlockCount); + + /* And with no difference taken the re-band bundle carries null, the way the card banded - so the + worst-first score cannot see a measurement the dot did not. */ + Assert.Null(Card(MonitoredEngineKind.Postgres).ToHealthMetricsValue.DeadlockCount); + Assert.Equal(1, Card(MonitoredEngineKind.Postgres).MeasuredMetricCount); } /* ─────────────────────────── the invariant ─────────────────────────── */ @@ -178,11 +311,14 @@ public void TheCountsAndTheCoverageDisclosureAreUntouched() [InlineData(false, 0, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Healthy, HealthSeverity.Healthy, FleetHealthBand.Healthy)] /* memory pressure -> Critical */ [InlineData(true, 0, 0L, 0, HealthSeverity.Critical, HealthSeverity.Healthy, HealthSeverity.Healthy, FleetHealthBand.Critical)] - /* one blocking event -> Warning */ - [InlineData(false, 1, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Warning, HealthSeverity.Healthy, FleetHealthBand.Warning)] - /* two events -> Warning; five -> Critical */ - [InlineData(false, 2, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Warning, HealthSeverity.Healthy, FleetHealthBand.Warning)] - [InlineData(false, 5, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Critical, HealthSeverity.Healthy, FleetHealthBand.Critical)] + /* The blocking COUNT arm bands on a RATE as of #3539 A3, so these rows track the CURRENT band the way + the deadlock rows below do. Both helpers window one hour, so the count IS the per-hour rate: one or + two reports is the measured quiet mode and Healthy by count, five is the Warning tier, twenty the + Critical one — where the old count ladder called one Warning and five Critical. */ + [InlineData(false, 1, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Healthy, HealthSeverity.Healthy, FleetHealthBand.Healthy)] + [InlineData(false, 2, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Healthy, HealthSeverity.Healthy, FleetHealthBand.Healthy)] + [InlineData(false, 5, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Warning, HealthSeverity.Healthy, FleetHealthBand.Warning)] + [InlineData(false, 20, 0L, 0, HealthSeverity.Healthy, HealthSeverity.Critical, HealthSeverity.Healthy, FleetHealthBand.Critical)] /* a long wait alone -> Warning at 10s, Critical at 60s, with a count of 1 */ [InlineData(false, 1, 10_000L, 0, HealthSeverity.Healthy, HealthSeverity.Warning, HealthSeverity.Healthy, FleetHealthBand.Warning)] [InlineData(false, 1, 60_000L, 0, HealthSeverity.Healthy, HealthSeverity.Critical, HealthSeverity.Healthy, FleetHealthBand.Critical)] @@ -227,19 +363,24 @@ public void TheClassifiersMeasuredArmsAreUnchanged() Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.MemorySeverity(true)); Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.MemorySeverity(null)); - Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(0, 0)); - Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(1, 0)); - Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(5, 0)); - Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 0)); + /* #3539 A3 re-banded the blocking COUNT arm on a RATE, so, as for deadlocks below, "unchanged" + holds for the null arm this file is about and the measured arms are asserted at their + post-#3539 values over a rateable hour: 0/hr Healthy, 5/hr Warning, 20/hr Critical. */ + var hour = TimeSpan.FromHours(1); + Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.BlockingSeverity(0, 0, hour)); + Assert.Equal(HealthSeverity.Warning, ServerHealthClassifier.BlockingSeverity(5, 0, hour)); + Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.BlockingSeverity(20, 0, hour)); + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 0, hour)); /* A max wait with no population is still Unknown - the count is the only spelling of "unmeasured", - so a stray duration cannot smuggle a band back in. */ - Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 600)); + so a stray duration cannot smuggle a band back in — at any window length, including none. */ + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 600, hour)); + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 600, TimeSpan.Zero)); + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.BlockingSeverity(null, 600, TimeSpan.FromHours(168))); /* #3368 re-banded this one on a RATE, so "unchanged" holds only for the null arm this file is about. The measured arms are asserted at their post-#3368 values, over a window a rate can be computed on: 0/hr Healthy, 30/hr Critical. That the NULL arm still answers Unknown at every window length is the claim that belongs here. */ - var hour = TimeSpan.FromHours(1); Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.DeadlockSeverity(0, hour, DeadlockRateThresholds.Default)); Assert.Equal(HealthSeverity.Critical, ServerHealthClassifier.DeadlockSeverity(30, hour, DeadlockRateThresholds.Default)); Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.DeadlockSeverity(null, hour, DeadlockRateThresholds.Default)); @@ -275,6 +416,7 @@ public void UnknownIsBandAndRankNeutral_SoTheFixCannotReorderTheFleet(int blocki HasMemoryPressure = false, BlockingCount = blocking, MaxBlockedSeconds = maxWaitMs / 1000.0, + BlockingWindow = TimeSpan.FromHours(1), DeadlockCount = 0, TotalThreads = 512, AvailableThreads = 500, @@ -302,16 +444,124 @@ public void UnknownIsBandAndRankNeutral_SoTheFixCannotReorderTheFleet(int blocki /// /// default(ServerHealthMetrics) flipped from "all zero, therefore Healthy" to "all null, - /// therefore Unknown" when the fields became nullable. That is only safe BECAUSE Unknown is inert, so it - /// is pinned rather than assumed: a bundle nobody populated still bands and scores as it did. + /// therefore Unknown" when the fields became nullable, and #3539 A6 closed the last gap: its collectors + /// row (0, 0) is Unknown too, so a bundle nobody populated measures NOTHING — and the fold says + /// so rather than answering Healthy from six Unknowns. Its band is the never-collected server's Warning, + /// and the score is that band's rank alone: the magnitude terms still skip Unknown, so it cannot climb + /// within the band on readings it does not have. /// [Fact] - public void AnUnpopulatedMetricBundleBandsAndScoresAsItAlwaysDid() + public void AnUnpopulatedMetricBundleMeasuresNothing_AndIsNotHealthy() { var empty = default(ServerHealthMetrics); - Assert.Equal(HealthSeverity.Healthy, ServerHealthClassifier.OverallMetricSeverity(empty)); - Assert.Equal(0L, ServerHealthClassifier.FleetHealthScore(FleetHealthBand.Healthy, empty)); + Assert.Equal((0, 6), ServerHealthClassifier.MeasuredMetricCounts(empty)); + Assert.Equal(HealthSeverity.Unknown, ServerHealthClassifier.OverallMetricSeverity(empty)); + + var band = ServerHealthClassifier.ClassifyBand(true, false, false, HealthSeverity.Unknown); + Assert.Equal(FleetHealthBand.Warning, band); + /* The Warning rank step and nothing else: no Critical/Warning metric to add magnitude, no incident + count — so an all-Unknown card sorts at the foot of the Warning band, under any card with a real + amber reading. */ + Assert.Equal(2000L, ServerHealthClassifier.FleetHealthScore(band, empty)); + } + + /* ─────────────────────────── #3539 A6: nothing banded, nothing measured ─────────────────────────── */ + + /// + /// The collectors row with NO collector banded reads Unknown on both cards, not Healthy — the #3539 A6 + /// sibling. The viewer's offline arm and the web's is_online === false chip cover a KNOWN-dark + /// server; this is a reachable one whose collection nobody has classified, and "nothing failing" is not + /// a health claim when nothing could have failed. Held on a SQL Server card so the other five rows are + /// measured and the collectors row is the only thing that changed. + /// + [Fact] + public void ZeroBandedCollectors_ReadUnknownNotHealthy_OnBothCards() + { + var card = Card(MonitoredEngineKind.SqlServer, bandedCollectors: 0); + Assert.Equal(0, card.CollectorCount); + Assert.Equal(0, card.FailedCollectorCount); + Assert.Equal(HealthSeverity.Unknown, card.CollectorSeverity); + + var viewer = ViewerCard(MonitoredEngineKind.SqlServer, bandedCollectors: 0); + Assert.Equal(HealthSeverity.Unknown, viewer.CollectorSeverity); + /* The word beside the dot agrees with it: "--" is the card's spelling of "no reading", where "OK" + was a green word under what is now a grey dot. */ + Assert.Equal("--", viewer.CollectorDisplay); + + /* And with ONE collector banded the arm is Healthy again, on both — the fix is the zero, not the + count. */ + Assert.Equal(HealthSeverity.Healthy, Card(MonitoredEngineKind.SqlServer, bandedCollectors: 1).CollectorSeverity); + Assert.Equal(HealthSeverity.Healthy, ViewerCard(MonitoredEngineKind.SqlServer, bandedCollectors: 1).CollectorSeverity); + Assert.Equal("OK", ViewerCard(MonitoredEngineKind.SqlServer, bandedCollectors: 1).CollectorDisplay); + + /* The collectors row is one of the six the coverage counts fold over: this helper's SQL Server card + measures memory, blocking and deadlocks (no CPU or threads row is handed in), so it says "3 of 6 + measured" with nothing banded and "4 of 6" with one collector — the row moved, and only the row. */ + Assert.Equal(3, card.MeasuredMetricCount); + Assert.Equal(6, card.MetricCount); + Assert.Equal(4, Card(MonitoredEngineKind.SqlServer, bandedCollectors: 1).MeasuredMetricCount); + Assert.Equal(FleetHealthBand.Healthy, card.Band); + } + + /// + /// The A6 card itself: an ONLINE PostgreSQL target with nothing banded — no CPU source, no threads, the + /// three DMV rows structurally null, zero collectors — measures nothing, and is NOT Healthy. Before this + /// it banded Healthy with "0 of 6 measured" as its only tell and counted in healthy_count on the + /// web fleet page, get_fleet_overview and the viewer's rollup. Now it bands Warning like a server + /// awaiting its first collection, leaves the healthy mass on every one of those surfaces, and the + /// ranking's reason says why in words rather than falling to "Needs attention". + /// + [Theory] + [InlineData(MonitoredEngineKind.Postgres)] + [InlineData(MonitoredEngineKind.AuroraPostgres)] + public void AnOnlineCardMeasuringNothing_IsNotInTheHealthyMass_OnAnySurface(string engineKind) + { + var card = Card(engineKind, bandedCollectors: 0); + Assert.True(card.IsOnline); + Assert.False(card.AwaitingFirstCollection); + Assert.Equal(0, card.MeasuredMetricCount); + Assert.Equal(6, card.MetricCount); + Assert.Equal(HealthSeverity.Unknown, card.OverallMetricSeverity); + Assert.Equal(FleetHealthBand.Warning, card.Band); + + /* The service's rollup — /api/fleet and get_fleet_overview read this: zero healthy, one warning. */ + var rollup = DarlingFleetReader.BuildRollup(new[] { card }, Now, Now.AddHours(-1), Now); + Assert.Equal(0, rollup.HealthyCount); + Assert.Equal(1, rollup.WarningCount); + var ranked = Assert.Single(rollup.WorstServers); + Assert.Equal(DarlingFleetReader.NoMetricMeasuredReason, ranked.Reason); + + /* The viewer's card and rollup, same server, same answer (#2473). */ + var viewer = ViewerCard(engineKind, bandedCollectors: 0); + Assert.Equal(0, viewer.MeasuredMetricCount); + Assert.Equal(HealthSeverity.Unknown, viewer.OverallMetricSeverity); + Assert.Equal(FleetHealthBand.Warning, FleetRollup.ClassifyBand(viewer)); + Assert.Equal(FleetRollup.NoMetricMeasuredReason, FleetRollup.BuildReason(viewer)); + Assert.Equal(DarlingFleetReader.NoMetricMeasuredReason, FleetRollup.NoMetricMeasuredReason); + + var viewerRollup = FleetRollup.Build(new[] { viewer }, new FleetTotals()); + Assert.Equal(0, viewerRollup.HealthyCount); + Assert.Equal(1, viewerRollup.WarningCount); + Assert.Contains(viewer, FleetRollup.NeedsAttention(new[] { viewer })); + + /* The tooltip's headline names the band and the reason once — not "Warning — no metric measured yet + · 0 of 6 measured", which would say the same thing twice. */ + Assert.StartsWith("Warning — " + FleetRollup.NoMetricMeasuredReason, viewer.StatusTooltip, StringComparison.Ordinal); + Assert.DoesNotContain("0 of 6", viewer.StatusTooltip, StringComparison.Ordinal); + + /* The border agrees with the band: the awaiting-first-collection amber, not the calm dark. */ + Assert.Equal("#FFFFD54F", viewer.CardBorderBrush.Color.ToString()); + + /* ONE banded collector and the same card is #3528's "Healthy — 1 of 6 measured", exactly where + that issue left it: the healthy mass loses only the cards that measured nothing. */ + var one = Card(engineKind, bandedCollectors: 1); + Assert.Equal(1, one.MeasuredMetricCount); + Assert.Equal(FleetHealthBand.Healthy, one.Band); + Assert.Equal(1, DarlingFleetReader.BuildRollup(new[] { one }, Now, Now.AddHours(-1), Now).HealthyCount); + var oneViewer = ViewerCard(engineKind, bandedCollectors: 1); + Assert.Equal(FleetHealthBand.Healthy, FleetRollup.ClassifyBand(oneViewer)); + Assert.StartsWith("Healthy — 1 of 6 measured", oneViewer.StatusTooltip, StringComparison.Ordinal); } /* ─────────────────────────── the ordering the guards depend on ─────────────────────────── */ diff --git a/Darling/Darling.Tests/ViewerAlertToastCoordinatorTests.cs b/Darling/Darling.Tests/ViewerAlertToastCoordinatorTests.cs index 618724075..62f54b0a4 100644 --- a/Darling/Darling.Tests/ViewerAlertToastCoordinatorTests.cs +++ b/Darling/Darling.Tests/ViewerAlertToastCoordinatorTests.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Linq; using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; using Xunit; namespace Darling.Tests; @@ -224,4 +225,159 @@ public void SelectToasts_SeenRowBeyondRetention_IsPruned_ThenReToasts() Assert.Single(toasts); } + + /* ---------------- #3570: the viewer honors mute rules for its own channel ---------------- */ + + /// + /// The report, as a pin. The service re-fires "Agent Not Running" with muted = false — because it has + /// not reloaded its cache yet, or its reload failed, or the beacon never reached it — and the viewer holds + /// the rule the operator's Snooze just wrote. The toast must not appear: the tray is the viewer's channel and + /// the viewer's rule set decides. Before #3570 only row.Muted was consulted and this toasted. + /// + [Fact] + public void SelectToasts_RowCoveredByAViewerRule_IsNotToasted_EvenWhenTheServiceLeftItUnmuted() + { + var coordinator = new AlertToastCoordinator(Retention); + var refire = Row(T0.AddMinutes(5), 1, "Agent Not Running", muted: false); + var snooze = ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromHours(4), T0); + + var toasts = coordinator.SelectToasts(new[] { refire }, T0.AddMinutes(5), Cooldown, new[] { snooze }); + + Assert.Empty(toasts); + } + + /// Null and empty rule sets are the pre-#3570 behavior exactly: the service's flag alone decides. + [Fact] + public void SelectToasts_NoViewerRules_LeavesAnUnmutedRowToasting() + { + var row = Row(T0, 1, "Agent Not Running"); + + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, muteRules: null)); + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, Array.Empty())); + } + + /// + /// The viewer-side judgement is on the coordinator's injected clock: a rule + /// whose expiry has passed suppresses nothing, judged at the SAME instant the rest of the decision is. + /// + [Fact] + public void SelectToasts_ExpiredViewerRule_DoesNotSuppress() + { + var coordinator = new AlertToastCoordinator(Retention); + var snooze = ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromMinutes(15), T0); + + /* Judged one second after the 15 m snooze lapsed. */ + var toasts = coordinator.SelectToasts( + new[] { Row(T0.AddMinutes(16), 1, "Agent Not Running") }, T0.AddMinutes(15).AddSeconds(1), Cooldown, new[] { snooze }); + + Assert.Single(toasts); + } + + [Fact] + public void SelectToasts_DisabledViewerRule_DoesNotSuppress() + { + var coordinator = new AlertToastCoordinator(Retention); + var rule = ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromHours(4), T0); + rule.Enabled = false; + + var toasts = coordinator.SelectToasts(new[] { Row(T0, 1, "Agent Not Running") }, T0, Cooldown, new[] { rule }); + + Assert.Single(toasts); + } + + /// + /// Scope is the shared matcher's: server name case-insensitive, metric exact (by name), a rule for another + /// server or another metric leaves this row alone. Pinned here so the tray can never be broader OR narrower + /// than the channels the service mutes with the same rule. + /// + [Fact] + public void SelectToasts_ViewerRuleScope_IsTheSharedMatchersScope() + { + var row = Row(T0, 1, "Agent Not Running"); + + var otherServer = ViewerDataService.BuildTraySnoozeRule("Server2", "Agent Not Running", TimeSpan.FromHours(4), T0); + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, new[] { otherServer })); + + var otherMetric = ViewerDataService.BuildTraySnoozeRule("Server1", "Failed Agent Job", TimeSpan.FromHours(4), T0); + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, new[] { otherMetric })); + + var differentCase = ViewerDataService.BuildTraySnoozeRule("SERVER1", "agent not running", TimeSpan.FromHours(4), T0); + Assert.Empty(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, new[] { differentCase })); + + /* A whole-server silence (no metric) covers every metric on that server — the sidebar's one-click rule. */ + var silence = ViewerDataService.BuildServerSilenceRule("Server1"); + Assert.Empty(new AlertToastCoordinator(Retention).SelectToasts(new[] { row }, T0, Cooldown, new[] { silence })); + } + + /// + /// A pattern-scoped rule is judged over the dimensions parses out + /// of the row's detail text — the same pre-fill the "Mute This Alert" dialog reads — so a database-scoped + /// mute the operator authored FROM a row covers that row's toast, and only rows about that database. + /// + [Fact] + public void SelectToasts_PatternScopedViewerRule_IsJudgedOverTheRowsDetailText() + { + var rule = new MuteRule { MetricName = "Blocking Detected", DatabasePattern = "Sales" }; + + var salesRow = Row(T0, 1, "Blocking Detected", detail: "Blocking detected\n Database: SalesDb\n Wait Type: LCK_M_S"); + Assert.Empty(new AlertToastCoordinator(Retention).SelectToasts(new[] { salesRow }, T0, Cooldown, new[] { rule })); + + var otherDbRow = Row(T0, 1, "Blocking Detected", detail: "Blocking detected\n Database: Payroll"); + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { otherDbRow }, T0, Cooldown, new[] { rule })); + + /* No detail text at all: the pattern dimension is unknown, the rule cannot claim it, the row toasts. */ + var bareRow = Row(T0, 1, "Blocking Detected"); + Assert.Single(new AlertToastCoordinator(Retention).SelectToasts(new[] { bareRow }, T0, Cooldown, new[] { rule })); + } + + /// + /// A row the viewer's rule suppressed is marked seen exactly like a service-muted one, so when the snooze + /// lapses the rows it covered do not replay as a storm — only rows that arrive AFTER expiry can toast. + /// + [Fact] + public void SelectToasts_ViewerSuppressedRow_IsMarkedSeen_SoRuleExpiryDoesNotReplayIt() + { + var coordinator = new AlertToastCoordinator(Retention); + var snooze = ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromMinutes(15), T0); + var covered = Row(T0.AddMinutes(5), 1, "Agent Not Running"); + + Assert.Empty(coordinator.SelectToasts(new[] { covered }, T0.AddMinutes(5), Cooldown, new[] { snooze })); + + /* The snooze has lapsed and the same row is re-read (the poll window overlaps): still nothing. */ + Assert.Empty(coordinator.SelectToasts(new[] { covered }, T0.AddMinutes(16), Cooldown, new[] { snooze })); + + /* A NEW row after expiry toasts — the condition is live again and the operator asked for 15 m, not forever. */ + Assert.Single(coordinator.SelectToasts(new[] { Row(T0.AddMinutes(17), 1, "Agent Not Running") }, T0.AddMinutes(17), Cooldown, new[] { snooze })); + } + + /// + /// The viewer's rule set and the service's flag are ORed: either alone suppresses, and a rule covering row A + /// says nothing about row B on another server in the same poll. + /// + [Fact] + public void SelectToasts_ViewerRulesAndServiceFlag_AreIndependentPerRow() + { + var coordinator = new AlertToastCoordinator(Retention); + var snooze = ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromHours(4), T0); + + var coveredByRule = Row(T0, 1, "Agent Not Running"); + var mutedByService = Row(T0, 2, "High CPU", muted: true); + var neither = Row(T0, 3, "Agent Not Running"); + + var toasts = coordinator.SelectToasts(new[] { coveredByRule, mutedByService, neither }, T0, Cooldown, new[] { snooze }); + + var only = Assert.Single(toasts); + Assert.Equal(3, only.ServerId); + } + + /// A null entry in the rule list is skipped rather than thrown on — the filter runs inside the refresh loop. + [Fact] + public void IsMutedByViewerRules_SkipsNullEntries() + { + var row = Row(T0, 1, "Agent Not Running"); + var rules = new MuteRule[] { null!, ViewerDataService.BuildTraySnoozeRule("Server1", "Agent Not Running", TimeSpan.FromHours(1), T0) }; + + Assert.True(AlertToastCoordinator.IsMutedByViewerRules(row, rules, T0)); + Assert.False(AlertToastCoordinator.IsMutedByViewerRules(row, new MuteRule[] { null! }, T0)); + } } diff --git a/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs b/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs index 70093fb9f..f870fa25f 100644 --- a/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs +++ b/Darling/Darling.Tests/ViewerControlPlaneStage3bTests.cs @@ -437,6 +437,49 @@ public void ServerHasOverride_ReflectsTheServersRows() Assert.True(CollectorScheduleOverlay.ServerHasOverride(overrides, 7)); Assert.False(CollectorScheduleOverlay.ServerHasOverride(overrides, 8)); } + + /// + /// #3532: the editor refuses a delta-family cadence past the shared gap-policy cap before the write — + /// past it every cycle exceeds , re-baselines, + /// and stores zeros forever. The refusal names the cap and the policy; snapshot collectors stay exempt. + /// + [Fact] + public void ValidateSchedule_RefusesADeltaCadencePastTheCap_NamingThePolicy() + { + var edited = CollectorSchedulePresets.BuildDefaultSchedule(); + edited.First(s => s.Name == "wait_stats").FrequencyMinutes = 90; + + Assert.False(CollectorScheduleOverlay.ValidateSchedule(edited, out var error)); + Assert.Contains("wait_stats", error); + Assert.Contains(CollectorDeltaCalculator.MaxDeltaFrequencyMinutes.ToString(), error); + Assert.Contains($"{CollectorDeltaCalculator.DefaultMaxGapSeconds / 60}-minute delta gap policy", error); + } + + [Fact] + public void ValidateSchedule_AllowsTheCapSnapshotLongCadences_AndTheShippedDefaults() + { + /* The shipped defaults must validate as-is (index_object_stats ships at 1440 — snapshot, exempt). */ + var edited = CollectorSchedulePresets.BuildDefaultSchedule(); + Assert.True(CollectorScheduleOverlay.ValidateSchedule(edited, out _)); + + edited.First(s => s.Name == "wait_stats").FrequencyMinutes = CollectorDeltaCalculator.MaxDeltaFrequencyMinutes; + edited.First(s => s.Name == "database_size_stats").FrequencyMinutes = 90; + Assert.True(CollectorScheduleOverlay.ValidateSchedule(edited, out _)); + } + + [Fact] + public void ValidateSchedule_StillRefusesNegativeFrequency_AndSubDayRetention() + { + var edited = CollectorSchedulePresets.BuildDefaultSchedule(); + edited.First(s => s.Name == "wait_stats").FrequencyMinutes = -1; + Assert.False(CollectorScheduleOverlay.ValidateSchedule(edited, out var negativeError)); + Assert.Contains("can't be negative", negativeError); + + edited.First(s => s.Name == "wait_stats").FrequencyMinutes = 1; + edited.First(s => s.Name == "wait_stats").RetentionDays = 0; + Assert.False(CollectorScheduleOverlay.ValidateSchedule(edited, out var retentionError)); + Assert.Contains("at least 1", retentionError); + } } /// The viewer's control commands agree with the service executor's dispatch (the two ends must use the diff --git a/Darling/Darling.Tests/ViewerDailyHealthTests.cs b/Darling/Darling.Tests/ViewerDailyHealthTests.cs index 0284f26f9..ead6358e2 100644 --- a/Darling/Darling.Tests/ViewerDailyHealthTests.cs +++ b/Darling/Darling.Tests/ViewerDailyHealthTests.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Npgsql; using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Storage; using PerformanceMonitor.Darling.Viewer; using Xunit; @@ -58,14 +59,26 @@ the SAME source the count came from (BPR preferred, DMV fallback) so it reconcil Assert.Contains("MAX(wait_time_ms) AS max_wait_ms", sql, StringComparison.Ordinal); Assert.Contains("CASE WHEN COALESCE(b.c, 0) > 0 THEN b.max_wait_ms ELSE dm.max_wait_ms END", sql, StringComparison.Ordinal); - /* High-CPU count uses total host CPU = SQL + other-process (Linux NULL → 0), threshold 80, via FILTER. */ + /* High-CPU count uses total host CPU = SQL + other-process (Linux NULL → 0), threshold 80, via FILTER. + The 80 is the card band's Warning bar restated as a SQL literal (#3539 A2) — pinned against the + constant so the day cell and the card cannot drift on what "high CPU" means, and NOT against the + alert engine's knob, which would recolour every past day when retuned. */ Assert.Contains("(sqlserver_cpu_utilization + COALESCE(other_process_cpu_utilization, 0)) >= 80", sql, StringComparison.Ordinal); + Assert.Equal(80.0, ServerHealthThresholds.CpuWarningPercent); + Assert.Contains( + ">= " + ServerHealthThresholds.CpuWarningPercent.ToString("0", System.Globalization.CultureInfo.InvariantCulture) + ")", + sql, StringComparison.Ordinal); Assert.Contains("FROM v_cpu_utilization_stats", sql, StringComparison.Ordinal); Assert.Contains("FILTER (WHERE", sql, StringComparison.Ordinal); - /* Collect errors off the collection_log ERROR rows; all-status runs mark the day collected. */ + /* Collect errors off the collection_log ERROR rows; all-status runs mark the day collected AND are + projected as the error share's denominator (#3539 A2), appended last so no ordinal moved. */ Assert.Contains("status = 'ERROR'", sql, StringComparison.Ordinal); Assert.Contains("FROM v_collection_log", sql, StringComparison.Ordinal); + Assert.Contains("COALESCE(cl.runs, 0) AS collection_runs", sql, StringComparison.Ordinal); + Assert.True( + sql.IndexOf("AS peak_block_wait_ms", StringComparison.Ordinal) < sql.IndexOf("AS collection_runs", StringComparison.Ordinal), + "collection_runs must be the trailing column so the eleven positional reads before it stay put"); /* Memory pressure (process OR system indicator >= 2) and the severe escalation (process >= 3). */ Assert.Contains("FROM v_memory_pressure_events", sql, StringComparison.Ordinal); @@ -482,7 +495,12 @@ public async Task DailySummary_RollsUpEverySource_ForTheSelectedDay_AgainstDevPo Assert.Equal(1, summary.BlockingEvents); // XE report count (fallback not used) Assert.Equal(1, summary.HighCpuEvents); // only the 90% sample Assert.Equal(1, summary.CollectionErrors); - Assert.Equal("Critical", summary.OverallHealth); // deadlocks -> Critical composite band + Assert.Equal(1, summary.CollectionRuns); // #3539 A2: the trailing collection_runs column + /* The composite band, on a finished 24-hour day (#3525, #3539 A2): one deadlock is 0.04/hr and + one blocking event 0.04/hr (both Healthy by rate), one hot sample is under the day's six, and + the one collector run that ERRORED is a 100% error share — past the 20% bar, which is the + Warning arm and never Critical. */ + Assert.Equal("Warning", summary.OverallHealth); bodySucceeded = true; } @@ -514,8 +532,10 @@ public async Task DailySummary_BlockingEvents_FallBackToTheDmvSnapshotCount_Agai var inDay = day.AddHours(9); /* No XE blocked-process reports; two DMV snapshots → the COALESCE(NULLIF(...)) falls back to - the DMV count. No deadlocks / sustained CPU / heavy blocking, but 2 blocking events is - "some blocking" → the composite band is Warning. */ + the DMV count. Two snapshots over a finished 24-hour day is 0.08/hr — under the 5/hr Warning + tier (#3539 A2/A3) — and the rows carry no wait time for the wait arm, so the day bands + Healthy with the blocking still counted; "any blocking is a Warning day" was the count + trigger this replaced. */ await InsertDmvBlockingAsync(connection, SummaryServerId, inDay); await InsertDmvBlockingAsync(connection, SummaryServerId, inDay); @@ -524,7 +544,8 @@ public async Task DailySummary_BlockingEvents_FallBackToTheDmvSnapshotCount_Agai Assert.NotNull(summary); Assert.Equal(2, summary!.BlockingEvents); Assert.Equal(0, summary.DeadlockCount); - Assert.Equal("Warning", summary.OverallHealth); + Assert.Equal("Healthy", summary.OverallHealth); + Assert.Contains("2 blocking events (0.1/hr)", summary.SignalsTooltip, StringComparison.Ordinal); bodySucceeded = true; } diff --git a/Darling/Darling.Tests/ViewerFileIoBlockingTests.cs b/Darling/Darling.Tests/ViewerFileIoBlockingTests.cs index 50018b71e..9c238e345 100644 --- a/Darling/Darling.Tests/ViewerFileIoBlockingTests.cs +++ b/Darling/Darling.Tests/ViewerFileIoBlockingTests.cs @@ -62,12 +62,15 @@ public void FileIoThroughputTrendSql_LagInterval_PerSecondMbRate() Assert.Contains("WITH top_files AS", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); Assert.Contains("ORDER BY SUM(delta_read_bytes + delta_write_bytes) DESC", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); - /* The per-file label is database.file, and the interval comes from LAG over collection_time. */ + /* The per-file label is database.file, and the interval is the row's STORED sample_interval_seconds + (0 → NULL) with the LAG over collection_time only for pre-V127 rows (#3540). */ Assert.Contains("f.database_name || '.' || f.file_name AS file_label", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); + Assert.Contains("CASE WHEN f.sample_interval_seconds IS NULL", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); Assert.Contains("LAG(f.collection_time)", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); Assert.Contains("EXTRACT(EPOCH FROM", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(f.sample_interval_seconds, 0)", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); - /* Bytes / interval-seconds / 1 MiB, and the NULL-interval first row per file is dropped. */ + /* Bytes / interval-seconds / 1 MiB, and the NULL-interval rows (first per file, or unknowable) are dropped. */ Assert.Contains("/ interval_seconds / 1048576.0", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); Assert.Contains("WHERE interval_seconds IS NOT NULL AND interval_seconds > 0", ViewerDataService.FileIoThroughputTrendSql, StringComparison.Ordinal); } @@ -116,8 +119,19 @@ public void LockWaitTrendSql_FiltersLckPrefix_PerSecondRateViaLag() { Assert.Contains("FROM v_wait_stats", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); Assert.Contains("wait_type LIKE 'LCK%'", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); - Assert.Contains("LAG(collection_time)", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); - Assert.Contains("CAST(delta_wait_time_ms AS double precision) / interval_seconds", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); + ViewerLatchSpinlockSqlTests.AssertStoredIntervalIdiom(ViewerDataService.LockWaitTrendSql, "wait_type"); + Assert.Contains("CAST(delta_wait_time_ms AS double precision) / interval_seconds END AS wait_time_ms_per_second", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", ViewerDataService.LockWaitTrendSql, StringComparison.Ordinal); + } + + /// #3540: the latency reads drop rows whose stored interval is 0 — the calculator's "no delta + /// knowable" marker — so a restart renders as an absent point, never "0.00 ms". IS DISTINCT FROM 0 keeps + /// pre-V127 rows (NULL). Both the File I/O tab's read and the tempdb tab's file read. + [Fact] + public void FileIoLatencyReads_DropTheUnknowableMarker_KeepPreV127Rows() + { + Assert.Contains("AND f.sample_interval_seconds IS DISTINCT FROM 0", ViewerDataService.FileIoLatencyTrendSql, StringComparison.Ordinal); + Assert.Contains("AND sample_interval_seconds IS DISTINCT FROM 0", ViewerDataService.TempDbFileIoTrendSql, StringComparison.Ordinal); } [Fact] @@ -447,21 +461,24 @@ public async Task LockWaitTrend_FiltersLckPrefix_PerSecondRate_AgainstDevPostgre { var t1 = TruncateToSeconds(DateTime.UtcNow.AddMinutes(-10)); var t2 = t1.AddSeconds(60); + var t3 = t2.AddSeconds(60); - /* Two collections of LCK_M_S 60s apart + a non-LCK wait that must be filtered out. */ + /* Three collections of LCK_M_S 60s apart + a non-LCK wait that must be filtered out. t1/t2 are + pre-V127 rows (NULL interval); t3 stores the unknowable marker (#3540). */ await InsertWaitStatAsync(connection, LockWaitServerId, t1, "LCK_M_S", deltaWaitTimeMs: 3000); await InsertWaitStatAsync(connection, LockWaitServerId, t2, "LCK_M_S", deltaWaitTimeMs: 6000); await InsertWaitStatAsync(connection, LockWaitServerId, t2, "SOS_SCHEDULER_YIELD", deltaWaitTimeMs: 99999); + await InsertWaitStatAsync(connection, LockWaitServerId, t3, "LCK_M_S", deltaWaitTimeMs: 0, sampleIntervalSeconds: 0); - var rows = await viewer.GetLockWaitTrendAsync(LockWaitServerId, t1.AddMinutes(-1), t2.AddMinutes(1)); + var rows = await viewer.GetLockWaitTrendAsync(LockWaitServerId, t1.AddMinutes(-1), t3.AddMinutes(1)); - /* Only the two LCK_M_S rows survive the LIKE 'LCK%' filter. */ - Assert.Equal(2, rows.Count); - Assert.All(rows, r => Assert.Equal("LCK_M_S", r.WaitType)); - /* First collection has no prior sample → interval NULL → CASE ELSE 0. */ - Assert.Equal(0.0, rows[0].WaitTimeMsPerSecond, precision: 3); - /* Second: 6000 ms / 60 s = 100 ms/sec. */ - Assert.Equal(100.0, rows[1].WaitTimeMsPerSecond, precision: 3); + /* One row: t1 has no prior sample and no stored interval (it used to plot as 0.00), t3 is a + restart's unknowable marker (absent, never 0.00), and SOS_SCHEDULER_YIELD fails LIKE 'LCK%'. */ + var row = Assert.Single(rows); + Assert.Equal("LCK_M_S", row.WaitType); + Assert.Equal(t2.Ticks, row.CollectionTime.Ticks); + /* 6000 ms / 60 s = 100 ms/sec. */ + Assert.Equal(100.0, row.WaitTimeMsPerSecond, precision: 3); bodySucceeded = true; } @@ -664,12 +681,14 @@ INSERT INTO deadlocks } private static async Task InsertWaitStatAsync( - NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, string waitType, long deltaWaitTimeMs) + NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, string waitType, long deltaWaitTimeMs, + int? sampleIntervalSeconds = null) { + /* sample_interval_seconds NULL by default — a pre-V127 row; 0 is the unknowable marker (#3540). */ using var command = new NpgsqlCommand(@" INSERT INTO wait_stats - (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms) -VALUES ($1, $2, $3, $4, $5, $6, $7)", connection); + (collection_id, collection_time, server_id, server_name, wait_type, delta_waiting_tasks, delta_wait_time_ms, sample_interval_seconds) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", connection); command.Parameters.AddWithValue(1L); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(serverId); @@ -677,6 +696,7 @@ INSERT INTO wait_stats command.Parameters.AddWithValue(waitType); command.Parameters.AddWithValue(1L); command.Parameters.AddWithValue(deltaWaitTimeMs); + command.Parameters.Add(new NpgsqlParameter { Value = (object?)sampleIntervalSeconds ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer }); await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); } diff --git a/Darling/Darling.Tests/ViewerFleetRollupTests.cs b/Darling/Darling.Tests/ViewerFleetRollupTests.cs index aaf3683cf..ea35eb9f7 100644 --- a/Darling/Darling.Tests/ViewerFleetRollupTests.cs +++ b/Darling/Darling.Tests/ViewerFleetRollupTests.cs @@ -60,6 +60,29 @@ public void FleetTotalsSql_DeadlocksAreACrossServerCount_OverTheWindow() Assert.Contains("deadlock_time <= $2", sql, StringComparison.Ordinal); } + /// + /// #3539: the PostgreSQL half of the deadlock total is a per-(server_id, database_name) counter + /// DIFFERENCE, clamped at zero, summed, and windowed on both bounds — added to the graph count so the + /// total reconciles with the sum of the card counts on both engines. Pinned on the text because + /// SUM(deadlocks) is the plausible one-liner that returns a lifetime counter times the sample + /// count, and nothing else in the build would notice. + /// + [Fact] + public void FleetTotalsSql_AddsThePostgresCounterDifferences_NeverTheRawColumn() + { + var sql = ViewerDataService.FleetTotalsSql; + Assert.Contains("FROM pg_database_stats", sql, StringComparison.Ordinal); + Assert.Contains("deadlocks - LAG(deadlocks) OVER (PARTITION BY server_id, database_name ORDER BY collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("SUM(GREATEST(sampled.raw_delta, 0))", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $1", sql, StringComparison.Ordinal); + Assert.Contains("collection_time <= $2", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(deadlocks)", sql, StringComparison.Ordinal); + + /* The two halves are ONE column: a second column would let a reader that indexes the deadlock + total positionally pick up only the graph count and silently drop the PostgreSQL half. */ + Assert.Equal(1, CountOccurrences(sql, "AS total_deadlocks")); + } + [Fact] public void FleetTotalsSql_WindowsEverySource_OnBothBounds() { @@ -658,12 +681,13 @@ private static async Task DeleteFleetRowsAsync(NpgsqlConnection connection, Syst } /// -/// The denominator beside the Overview's deadlock total (#3029). FleetTotalsSql's -/// SELECT COUNT(*) FROM v_deadlocks reads the SQL Server extended-event capture and nothing else — -/// a PostgreSQL target's deadlocks go to pg_deadlocks, which nothing joins in — so on a PostgreSQL -/// fleet that total is structurally zero forever. Zero is also exactly what a genuinely quiet SQL Server -/// fleet reports, so the reading that needs no action and the reading that does not cover the fleet had the -/// same character, and the tile could not tell an operator which one they were looking at. +/// The denominator beside the Overview's deadlock total (#3029). A server whose deadlock-source collector +/// is silent or denied contributes a structural zero to FleetTotalsSql's total, and zero is also +/// exactly what a genuinely quiet fleet reports, so the reading that needs no action and the reading that +/// does not cover the fleet had the same character, and the tile could not tell an operator which one they +/// were looking at. Until #3539 every PostgreSQL target was such a zero (the total read v_deadlocks, +/// the SQL Server capture, and nothing else); the total now adds the PostgreSQL server counter's differences, +/// and a PostgreSQL target is covered on its own collector's terms. /// /// Both directions, deliberately. The failure mode of a coverage figure is over-exclusion: a /// denominator that quietly shrinks reads as a smaller fleet, which is a new wrong number rather than a fix. @@ -679,6 +703,10 @@ public sealed class ViewerFleetDeadlockCoverageTests { private static readonly FleetTotals NoTotals = new(); + /// The ENGINE'S deadlock-source collector band (#3539): set on the card's + /// pg_database_stats slot for a PostgreSQL target and on its deadlocks slot otherwise, the + /// way the loader's two bands land — so a PostgreSQL card here is covered on the same terms the + /// production card is. private static ServerSummaryItem Card(int id, bool isPostgres = false, string? band = null) => new() { @@ -686,7 +714,8 @@ private static ServerSummaryItem Card(int id, bool isPostgres = false, string? b DisplayName = "target-" + id.ToString(CultureInfo.InvariantCulture), IsOnline = true, IsPostgres = isPostgres, - DeadlockCollectorBand = band, + DeadlockCollectorBand = isPostgres ? null : band, + PgDeadlockCollectorBand = isPostgres ? band : null, }; // ── The card's own reading of whether its deadlock count read anything ────────────────────────── @@ -718,17 +747,26 @@ public void ADeadlockReaderThatReadNothing_DoesNotCount_AndNamesItsCause(string? => Assert.Equal(expected, Card(1, band: band).DeadlockSource); /// - /// The issue's own case: a PostgreSQL target is never covered, and its collector's band cannot change - /// that. pg_deadlocks can be perfectly HEALTHY on all fifty targets and this total still counts - /// none of it — the rows are in a different table. That is why PostgreSQL is asked before any band. + /// #3539: a PostgreSQL target is covered on its OWN collector's terms — pg_database_stats, whose + /// counter the card differences — and lands on the PostgresTarget arm when that collector read, + /// the silent/denied arms when it did not. The card picks the PostgreSQL band because IsPostgres + /// says so; a deadlocks band on the same card is ignored, because that engine has no such + /// collector and a stray value there must not make it read. /// [Theory] - [InlineData(CollectorHealthClassifier.Healthy)] - [InlineData(CollectorHealthClassifier.NoPermissions)] - [InlineData(CollectorHealthClassifier.Stopped)] - [InlineData(null)] - public void APostgresTarget_IsNeverCovered_WhateverItsCollectorSays(string? band) - => Assert.Equal(FleetDeadlockSource.PostgresTarget, Card(1, isPostgres: true, band: band).DeadlockSource); + [InlineData(CollectorHealthClassifier.Healthy, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.Failing, FleetDeadlockSource.PostgresTarget)] + [InlineData(CollectorHealthClassifier.NoPermissions, FleetDeadlockSource.CollectorDenied)] + [InlineData(CollectorHealthClassifier.Stopped, FleetDeadlockSource.CollectorSilent)] + [InlineData(null, FleetDeadlockSource.CollectorSilent)] + public void APostgresTarget_IsCoveredOnItsOwnCollectorsTerms(string? band, FleetDeadlockSource expected) + { + Assert.Equal(expected, Card(1, isPostgres: true, band: band).DeadlockSource); + + var strayDeadlocksBand = Card(1, isPostgres: true, band: band); + strayDeadlocksBand.DeadlockCollectorBand = CollectorHealthClassifier.Healthy; + Assert.Equal(expected, strayDeadlocksBand.DeadlockSource); + } /// /// A card that sets nothing reads as UNCOVERED, and that is the load-bearing default. @@ -765,29 +803,32 @@ public void Build_ReducesCoverageFromTheCards_WithEveryCauseAttributed() { Card(1, band: CollectorHealthClassifier.Healthy), Card(2, band: CollectorHealthClassifier.Failing), - Card(3, isPostgres: true), - Card(4, isPostgres: true), + Card(3, isPostgres: true, band: CollectorHealthClassifier.Healthy), + Card(4, isPostgres: true, band: CollectorHealthClassifier.Stale), Card(5, band: CollectorHealthClassifier.Stopped), Card(6, band: CollectorHealthClassifier.NoPermissions), Card(7, band: null), + /* #3539: a PostgreSQL target whose pg_database_stats collector left no band is SILENT. */ + Card(8, isPostgres: true, band: null), }, NoTotals); var coverage = rollup.DeadlockCoverage; - Assert.Equal(2, coverage.ServersRead); - Assert.Equal(7, coverage.ServersTotal); + Assert.Equal(4, coverage.ServersRead); + Assert.Equal(8, coverage.ServersTotal); Assert.Equal(2, coverage.PostgresServers); - Assert.Equal(2, coverage.ServersCollectorSilent); // STOPPED + the null band + Assert.Equal(3, coverage.ServersCollectorSilent); // STOPPED + the null band + the bandless PostgreSQL target Assert.Equal(1, coverage.ServersCollectorDenied); - /* With every registered server loaded, the four causes account for the fleet exactly once — an + /* With every registered server loaded, the causes account for the fleet exactly once — an unattributed server would mean coverage reporting a gap it cannot explain, which is the same - shape as a total reporting no denominator. */ + shape as a total reporting no denominator. Three terms since #3539: postgres_servers is a + SUBSET of servers_read, not a bucket beside it. */ Assert.Equal( coverage.ServersTotal, - coverage.ServersRead + coverage.PostgresServers - + coverage.ServersCollectorSilent + coverage.ServersCollectorDenied); + coverage.ServersRead + coverage.ServersCollectorSilent + coverage.ServersCollectorDenied); + Assert.True(coverage.PostgresServers <= coverage.ServersRead); /* And it agrees with the count the panel already shows beside it. */ Assert.Equal(rollup.TotalServers, coverage.ServersTotal); @@ -810,7 +851,7 @@ public void Build_CoverageDenominator_IsTheRegisteredFleet_AndTheShortfallIsTheU var loaded = new[] { Card(1, band: CollectorHealthClassifier.Healthy), - Card(2, isPostgres: true), + Card(2, isPostgres: true, band: CollectorHealthClassifier.Healthy), }; var rollup = FleetRollup.Build(loaded, NoTotals, totalServerCount: 5); @@ -818,7 +859,8 @@ public void Build_CoverageDenominator_IsTheRegisteredFleet_AndTheShortfallIsTheU Assert.Equal(5, coverage.ServersTotal); Assert.NotEqual(loaded.Length, coverage.ServersTotal); - Assert.Equal(1, coverage.ServersRead); + /* Both loaded cards are covered (#3539); the PostgreSQL one is also named in its sub-count. */ + Assert.Equal(2, coverage.ServersRead); Assert.Equal(1, coverage.PostgresServers); Assert.Equal(3, rollup.UnknownCount); @@ -828,7 +870,7 @@ public void Build_CoverageDenominator_IsTheRegisteredFleet_AndTheShortfallIsTheU Assert.Equal( coverage.ServersTotal, - coverage.ServersRead + coverage.PostgresServers + coverage.ServersCollectorSilent + coverage.ServersRead + coverage.ServersCollectorSilent + coverage.ServersCollectorDenied + rollup.UnknownCount); } @@ -866,13 +908,16 @@ public void Build_WithNoFleetAtAll_HasNothingToQualify() } /// - /// Only counts as read, and the enum cannot grow unnoticed. + /// Only the two covered arms count as read, and the enum cannot grow unnoticed. /// Reflected off the type rather than listed by hand: a pin that enumerates the kinds by hand cannot see /// the set grow, and a source kind added later that landed in the read bucket by default would restore /// exactly the defect this figure exists to fix. If this count moves, whoever moved it decides here. + /// Since #3539 the PostgreSQL arm is one of the covered two — the reducer counts it through the shared + /// , so this file and the service's roll-up cannot + /// disagree about which arms are read. /// [Fact] - public void EverySourceKind_IsProducible_AndOnlyReadCountsAsRead() + public void EverySourceKind_IsProducible_AndOnlyTheCoveredArmsCountAsRead() { var kinds = Enum.GetValues(); Assert.Equal(4, kinds.Length); @@ -882,7 +927,7 @@ public void EverySourceKind_IsProducible_AndOnlyReadCountsAsRead() var producible = new[] { (IsPostgres: false, Band: (string?)CollectorHealthClassifier.Healthy), - (IsPostgres: true, Band: (string?)null), + (IsPostgres: true, Band: (string?)CollectorHealthClassifier.Healthy), (IsPostgres: false, Band: (string?)CollectorHealthClassifier.Stopped), (IsPostgres: false, Band: (string?)CollectorHealthClassifier.NoPermissions), } @@ -892,18 +937,21 @@ public void EverySourceKind_IsProducible_AndOnlyReadCountsAsRead() Assert.Equal(kinds.Length, producible.Count); Assert.All(kinds, k => Assert.Contains(k, producible)); - /* And the reducer counts exactly one of them as read: one card of each kind, ServersRead == 1. */ + /* And the reducer counts exactly the covered two as read: one card of each kind, ServersRead == 2, + with the PostgreSQL one also in its own sub-count. */ var coverage = FleetRollup.ReduceDeadlockCoverage( new[] { Card(1, band: CollectorHealthClassifier.Healthy), - Card(2, isPostgres: true), + Card(2, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(3, band: CollectorHealthClassifier.Stopped), Card(4, band: CollectorHealthClassifier.NoPermissions), }, registeredTotal: 4); - Assert.Equal(1, coverage.ServersRead); + Assert.Equal(2, coverage.ServersRead); + Assert.Equal(1, coverage.PostgresServers); + Assert.Equal(Enum.GetValues().Count(FleetDeadlockCoverage.IsCovered), coverage.ServersRead); } // ── What the panel actually renders ──────────────────────────────────────────────────────────── @@ -947,20 +995,39 @@ public void TheCoverageLine_TracksCoverage_NotTheDeadlockCount() Assert.Equal("Deadlock coverage: read all 2 servers", rollup.DeadlockCoverageText); } - /// The issue's measured case: a PostgreSQL-only fleet reporting zero, now saying so. + /// The issue's measured case, reversed by #3539: a PostgreSQL-only fleet whose + /// pg_database_stats collectors run is FULLY covered, and the tooltip names the instrument + /// rather than saying the total cannot count it. A PostgreSQL-only fleet whose collectors are all silent + /// still reads "0 of 3" — through the silent cause, which is the one that names an action. [Fact] - public void APostgresOnlyFleet_ReportsZeroCoverage_AndNamesWhereThoseDeadlocksAre() + public void APostgresOnlyFleet_IsCovered_AndNamesTheInstrument() { var rollup = FleetRollup.Build( + new[] + { + Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy), + Card(2, isPostgres: true, band: CollectorHealthClassifier.Healthy), + Card(3, isPostgres: true, band: CollectorHealthClassifier.Warning), + }, + new FleetTotals { TotalDeadlocks = 4 }); + + Assert.Equal(4, rollup.TotalDeadlocks); + Assert.Equal(3, rollup.DeadlockCoverage.ServersRead); + Assert.Equal(3, rollup.DeadlockCoverage.PostgresServers); + Assert.False(rollup.DeadlockCoverageIsPartial); + Assert.Equal("Deadlock coverage: read all 3 servers", rollup.DeadlockCoverageText); + Assert.Contains("3 servers: " + FleetRollup.DeadlockPostgresCause, rollup.DeadlockCoverageTooltip, StringComparison.Ordinal); + Assert.DoesNotContain("cannot count", rollup.DeadlockCoverageTooltip, StringComparison.Ordinal); + + var silent = FleetRollup.Build( new[] { Card(1, isPostgres: true), Card(2, isPostgres: true), Card(3, isPostgres: true) }, new FleetTotals { TotalDeadlocks = 0 }); - Assert.Equal(0, rollup.TotalDeadlocks); - Assert.Equal(0, rollup.DeadlockCoverage.ServersRead); - Assert.Equal(3, rollup.DeadlockCoverage.PostgresServers); - Assert.True(rollup.DeadlockCoverageIsPartial); - Assert.Equal("Deadlock coverage: read 0 of 3 servers", rollup.DeadlockCoverageText); - Assert.Contains(FleetRollup.DeadlockPostgresCause, rollup.DeadlockCoverageTooltip, StringComparison.Ordinal); + Assert.Equal(0, silent.DeadlockCoverage.ServersRead); + Assert.Equal(0, silent.DeadlockCoverage.PostgresServers); + Assert.Equal(3, silent.DeadlockCoverage.ServersCollectorSilent); + Assert.Equal("Deadlock coverage: read 0 of 3 servers", silent.DeadlockCoverageText); + Assert.Contains("3 servers: " + FleetRollup.DeadlockCollectorSilentCause, silent.DeadlockCoverageTooltip, StringComparison.Ordinal); } /// A one-server fleet says "server", not "servers" — both ways round. @@ -973,7 +1040,7 @@ public void TheCoverageLine_AgreesWithItselfOnNumber() Assert.Equal( "Deadlock coverage: read 0 of 1 server", - FleetRollup.Build(new[] { Card(1, isPostgres: true) }, NoTotals).DeadlockCoverageText); + FleetRollup.Build(new[] { Card(1, band: CollectorHealthClassifier.Stopped) }, NoTotals).DeadlockCoverageText); } /// @@ -1002,8 +1069,11 @@ public void TheTooltip_NamesEachWindow_AndClaimsNeitherForTheOther() /* And the disclaimer that keeps the first from being read as the second. */ Assert.Contains("makes no claim about what was read in the last hour", tooltip, StringComparison.Ordinal); - /* What the total is assembled from — the fact that makes a PostgreSQL zero structural. */ - Assert.Contains("SQL Server extended-event capture and nothing else", tooltip, StringComparison.Ordinal); + /* What the total is assembled from - both engines' instruments since #3539, and the collector + state as the thing that makes a zero structural. */ + Assert.Contains("SQL Server extended-event capture", tooltip, StringComparison.Ordinal); + Assert.Contains("deadlock counter differenced over the window", tooltip, StringComparison.Ordinal); + Assert.DoesNotContain("and nothing else", tooltip, StringComparison.Ordinal); } /// @@ -1013,7 +1083,7 @@ public void TheTooltip_NamesEachWindow_AndClaimsNeitherForTheOther() public void TheTooltip_CarriesOnlyTheCausesThatApply() { var tooltip = FleetRollup.Build( - new[] { Card(1, isPostgres: true), Card(2, band: CollectorHealthClassifier.Healthy) }, + new[] { Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(2, band: CollectorHealthClassifier.Healthy) }, NoTotals).DeadlockCoverageTooltip; Assert.Contains("1 server: " + FleetRollup.DeadlockPostgresCause, tooltip, StringComparison.Ordinal); @@ -1034,7 +1104,7 @@ public void TheTooltip_AgreesWithItselfOnNumber_ForEveryCause() var singular = FleetRollup.Build( new[] { - Card(1, isPostgres: true), + Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(2, band: CollectorHealthClassifier.Stopped), Card(3, band: CollectorHealthClassifier.NoPermissions), }, @@ -1049,7 +1119,7 @@ public void TheTooltip_AgreesWithItselfOnNumber_ForEveryCause() var plural = FleetRollup.Build( new[] { - Card(1, isPostgres: true), Card(2, isPostgres: true), + Card(1, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(2, isPostgres: true, band: CollectorHealthClassifier.Healthy), Card(3, band: CollectorHealthClassifier.Stopped), Card(4, band: CollectorHealthClassifier.NeverRun), Card(5, band: CollectorHealthClassifier.NoPermissions), Card(6, band: CollectorHealthClassifier.NoPermissions), }, @@ -1134,9 +1204,11 @@ public void TheSummaryRead_RetainsTheDeadlockCollectorsBand_MatchedFromItsOwnNam var source = ReadRepoFile("Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Overview.cs"); /* Anchored on the DECLARATION, which is itself the shape being pinned: the helper hands the band - back beside the tallies rather than returning a pair the caller has to re-read the store for. */ + back beside the tallies rather than returning a pair the caller has to re-read the store for. + The tuple grew a Total for #3539 A8d (the share's denominator) — a fourth tally, same shape — + and the pg_database_stats collector's band for #3539's PostgreSQL deadlock arm, a fifth. */ var start = source.IndexOf( - "private async Task<(int Healthy, int Failing, string? DeadlockBand)> GetCollectorHealthCountsAsync", + "private async Task<(int Healthy, int Failing, int Total, string? DeadlockBand, string? PgDeadlockBand)> GetCollectorHealthCountsAsync", StringComparison.Ordinal); Assert.True(start > 0, "the collector-health helper does not hand back the deadlock band"); var end = source.IndexOf("private static int? MinutesAgo", start, StringComparison.Ordinal); @@ -1145,11 +1217,15 @@ back beside the tallies rather than returning a pair the caller has to re-read t var body = source[start..end]; Assert.Contains("DeadlocksCollector.Instance.Name", body, StringComparison.Ordinal); + /* #3539: the PostgreSQL deadlock-source collector's band, matched the same way for the same reason. */ + Assert.Contains("PgDatabaseStatsCollector.Instance.Name", body, StringComparison.Ordinal); /* The literal is the shape a careless match takes, and it is what the collector name is TODAY — so this is a real trap rather than a hypothetical one. */ Assert.DoesNotContain("\"deadlocks\"", body, StringComparison.Ordinal); + Assert.DoesNotContain("\"pg_database_stats\"", body, StringComparison.Ordinal); Assert.Equal("deadlocks", DeadlocksCollector.Instance.Name); + Assert.Equal("pg_database_stats", PgDatabaseStatsCollector.Instance.Name); } private static int CountOccurrences(string haystack, string needle) diff --git a/Darling/Darling.Tests/ViewerHistoryWindowTests.cs b/Darling/Darling.Tests/ViewerHistoryWindowTests.cs index 34bb065fa..5629852d3 100644 --- a/Darling/Darling.Tests/ViewerHistoryWindowTests.cs +++ b/Darling/Darling.Tests/ViewerHistoryWindowTests.cs @@ -46,7 +46,7 @@ public void QueryStatsHistorySql_FiltersOneQueryHashOverTheWindow_OrderedByTime( } [Fact] - public void ProcStatsHistorySql_FiltersOneSchemaObjectOverTheWindow_AndDerivesTheInterval() + public void ProcStatsHistorySql_FiltersOneSchemaObjectOverTheWindow_AndPrefersTheStoredInterval() { var sql = ViewerDataService.ProcStatsHistorySql; Assert.Contains("FROM procedure_stats", sql, StringComparison.Ordinal); @@ -57,7 +57,12 @@ public void ProcStatsHistorySql_FiltersOneSchemaObjectOverTheWindow_AndDerivesTh Assert.Contains("collection_time >= $5", sql, StringComparison.Ordinal); Assert.Contains("collection_time <= $6", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY collection_time", sql, StringComparison.Ordinal); - /* procedure_stats has no sample_interval_seconds column — it's derived from the previous row's gap. */ + /* #3540 (V128): procedure_stats carries sample_interval_seconds now. The STORED value is shown where + the row has one — a 0 included, the Interval (sec) 0 the query-stats history grid has always shown + for an unknowable row — and a pre-V128 row (NULL) keeps the interval this read always derived from + the previous row's gap. COALESCE, not CASE: a displayed interval is not a rate, so the stored 0 + stays a 0 here rather than becoming NULL. */ + Assert.Contains("COALESCE(sample_interval_seconds, CAST(extract(epoch FROM", sql, StringComparison.Ordinal); Assert.Contains("LAG(collection_time)", sql, StringComparison.Ordinal); Assert.Contains("total_spills", sql, StringComparison.Ordinal); AssertPgPositionalDialect(sql); diff --git a/Darling/Darling.Tests/ViewerIndexLockingRowFullNameTests.cs b/Darling/Darling.Tests/ViewerIndexLockingRowFullNameTests.cs new file mode 100644 index 000000000..a6c193add --- /dev/null +++ b/Darling/Darling.Tests/ViewerIndexLockingRowFullNameTests.cs @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3576, the viewer's half. The FinOps Locking & Contention grid binds +/// (schema.table) instead of the bare table name, so two tables sharing a name across schemas no longer +/// read as one. The viewer's row model is a hand-kept copy of Lite's, so the property is pinned here against +/// the copy the viewer actually compiles; the grid XAML of both SKUs is pinned from Lite.Tests +/// (IndexLockingGridQualifiedNameTests), which reads both files from source. +/// +public sealed class ViewerIndexLockingRowFullNameTests +{ + [Fact] + public void FullName_IsSchemaDotTable_WhenSchemaIsPresent() + { + var row = new IndexLockingRow { SchemaName = "archive", TableName = "Orders" }; + + Assert.Equal("archive.Orders", row.FullName); + } + + [Fact] + public void FullName_FallsBackToBareTable_WhenSchemaIsEmpty() + { + /* The reader writes "" (never null) for a NULL schema_name, so "" is the real fallback input. */ + var row = new IndexLockingRow { SchemaName = "", TableName = "Orders" }; + + Assert.Equal("Orders", row.FullName); + } + + [Fact] + public void ColumnFilter_OnFullName_ActuallyFilters() + { + /* The popup filter resolves its button's Tag to a row property by reflection; a Tag naming a property + the row lacks makes MatchesFilter return true for every row — a filter that filters nothing. */ + Assert.NotNull(typeof(IndexLockingRow).GetProperty("FullName")); + + var filter = new ColumnFilterState { ColumnName = "FullName", Operator = FilterOperator.Contains, Value = "archive." }; + + Assert.True(ColumnFilterMatcher.MatchesFilter(new IndexLockingRow { SchemaName = "archive", TableName = "Orders" }, filter)); + Assert.False(ColumnFilterMatcher.MatchesFilter(new IndexLockingRow { SchemaName = "dbo", TableName = "Orders" }, filter)); + } +} diff --git a/Darling/Darling.Tests/ViewerLatchSpinlockTests.cs b/Darling/Darling.Tests/ViewerLatchSpinlockTests.cs index b714b86d7..9a64bc3b7 100644 --- a/Darling/Darling.Tests/ViewerLatchSpinlockTests.cs +++ b/Darling/Darling.Tests/ViewerLatchSpinlockTests.cs @@ -21,9 +21,9 @@ namespace Darling.Tests; /// /// Pins the Latches & Spinlocks tab's four reads against the Darling store contract (no live /// Postgres): the two per-second trend reads (top-5 by delta, normalized to ms/sec and collisions/sec via -/// the per-contender LAG interval — Darling's cumulative-delta tables carry no stored -/// sample_interval_seconds, so the seconds come from the same truncate-then-diff epoch idiom the wait -/// trend uses) and the two latest-snapshot grid reads (most recent collection in the window, ordered by +/// each row's stored sample_interval_seconds since V127/#3540, with the per-contender LAG interval — the +/// same truncate-then-diff epoch idiom the wait trend uses — as the fallback for pre-V127 rows that never +/// recorded one) and the two latest-snapshot grid reads (most recent collection in the window, ordered by /// recent delta). All four run on the v_latch_stats / v_spinlock_stats passthrough views. /// public sealed class ViewerLatchSpinlockSqlTests @@ -44,13 +44,26 @@ public void LatchTrendSql_TopFiveByDeltaWait_PerClassLagPerSecond_OverTheWindow( Assert.Contains("LIMIT 5", sql, StringComparison.Ordinal); Assert.Contains("latch_class IN (SELECT latch_class FROM top_latches)", sql, StringComparison.Ordinal); - /* Per-class LAG interval → ms/sec, the wait-stats truncate-then-diff epoch idiom. */ - Assert.Contains("LAG(collection_time) OVER (PARTITION BY latch_class ORDER BY collection_time)", sql, StringComparison.Ordinal); - Assert.Contains("extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time)", sql, StringComparison.Ordinal); - Assert.Contains("CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first; the per-class LAG interval (the wait-stats truncate-then-diff + epoch idiom) only for pre-V127 rows; 0 → NULL through NULLIF; no ELSE 0 on the rate. */ + AssertStoredIntervalIdiom(sql, "latch_class"); + Assert.Contains("CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds END AS wait_time_ms_per_second", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0 END AS wait_time_ms_per_second", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY latch_class, collection_time", sql, StringComparison.Ordinal); } + /// The #3540 reader idiom every per-second read over the four interval-carrying delta families + /// shares: the stored interval when the row has one (0, the calculator's unknowable marker, mapped to + /// NULL), the LAG derivation only when it does not (a pre-V127 row). + internal static void AssertStoredIntervalIdiom(string sql, string partition) + { + Assert.Contains("CASE WHEN sample_interval_seconds IS NULL", sql, StringComparison.Ordinal); + Assert.Contains($"LAG(collection_time) OVER (PARTITION BY {partition} ORDER BY collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(sample_interval_seconds, 0)", sql, StringComparison.Ordinal); + Assert.Contains("END AS interval_seconds", sql, StringComparison.Ordinal); + } + [Fact] public void LatchSnapshotSql_LatestCollectionInWindow_OrderedByRecentDelta_CapAt20() { @@ -73,8 +86,9 @@ public void SpinlockTrendSql_TopFiveByDeltaCollisions_PerNameLagPerSecond_OverTh Assert.Contains("ORDER BY SUM(delta_collisions) DESC", sql, StringComparison.Ordinal); Assert.Contains("LIMIT 5", sql, StringComparison.Ordinal); Assert.Contains("spinlock_name IN (SELECT spinlock_name FROM top_spinlocks)", sql, StringComparison.Ordinal); - Assert.Contains("LAG(collection_time) OVER (PARTITION BY spinlock_name ORDER BY collection_time)", sql, StringComparison.Ordinal); - Assert.Contains("CAST(delta_collisions AS DOUBLE PRECISION) / interval_seconds", sql, StringComparison.Ordinal); + AssertStoredIntervalIdiom(sql, "spinlock_name"); + Assert.Contains("CAST(delta_collisions AS DOUBLE PRECISION) / interval_seconds END AS collisions_per_second", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY spinlock_name, collection_time", sql, StringComparison.Ordinal); } @@ -188,17 +202,25 @@ public async Task LatchTrend_TopFiveByDelta_PerSecond_AgainstDevPostgres() { var t1 = TruncateToSeconds(DateTime.UtcNow.AddMinutes(-10)); var t2 = t1.AddMinutes(5); // 300 seconds later - - /* BUFFER across two collections: 300 ms delta over 300 s = 1.0 ms/sec at t2; 0 at t1 (no LAG). */ + var t3 = t2.AddMinutes(5); + var t4 = t3.AddMinutes(5); + + /* BUFFER across four collections (#3540). t1/t2 are pre-V127 rows (NULL interval): t1 has no + prior and is NOT a point (it used to plot as 0.00); t2 is 300 ms over the LAG's 300 s = 1.0. + t3 stores interval 0 — the calculator's "no delta knowable" marker, a restart — and must be + ABSENT rather than 0.00. t4 stores a measured 120 s beside a 600 ms delta = 5.0 ms/sec, and + the stored interval wins over the LAG (which would say 300 s → 2.0). */ await InsertLatchAsync(connection, 1, t1, "BUFFER", deltaWait: 100, deltaReqs: 5); await InsertLatchAsync(connection, 2, t2, "BUFFER", deltaWait: 300, deltaReqs: 3); + await InsertLatchAsync(connection, 3, t3, "BUFFER", deltaWait: 0, deltaReqs: 0, sampleIntervalSeconds: 0); + await InsertLatchAsync(connection, 4, t4, "BUFFER", deltaWait: 600, deltaReqs: 6, sampleIntervalSeconds: 120); - var trend = await viewer.GetLatchStatsTrendAsync(LatchServerId, t1.AddMinutes(-1), t2.AddMinutes(1)); + var trend = await viewer.GetLatchStatsTrendAsync(LatchServerId, t1.AddMinutes(-1), t4.AddMinutes(1)); var buffer = trend.Where(p => p.LatchClass == "BUFFER").OrderBy(p => p.CollectionTime).ToList(); - Assert.Equal(2, buffer.Count); - Assert.Equal(0.0, buffer[0].WaitTimeMsPerSecond, precision: 3); - Assert.Equal(1.0, buffer[1].WaitTimeMsPerSecond, precision: 3); + Assert.Equal(new[] { t2.Ticks, t4.Ticks }, buffer.Select(p => p.CollectionTime.Ticks).ToArray()); + Assert.Equal(1.0, buffer[0].WaitTimeMsPerSecond, precision: 3); + Assert.Equal(5.0, buffer[1].WaitTimeMsPerSecond, precision: 3); bodySucceeded = true; } @@ -253,14 +275,16 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup private static async Task InsertLatchAsync( NpgsqlConnection connection, long collectionId, DateTime collectionTimeUtc, - string latchClass, long deltaWait, long deltaReqs) + string latchClass, long deltaWait, long deltaReqs, int? sampleIntervalSeconds = null) { + /* sample_interval_seconds NULL by default — a pre-V127 row, the shape every pin above was written + against; a test that wants the V127 contract passes 0 (unknowable) or a measured value. */ using var command = new NpgsqlCommand(@" INSERT INTO latch_stats (collection_id, collection_time, server_id, server_name, latch_class, waiting_requests_count, wait_time_ms, max_wait_time_ms, - delta_waiting_requests_count, delta_wait_time_ms, delta_max_wait_time_ms) -VALUES ($1, $2, $3, $4, $5, 0, 0, 0, $6, $7, 0)", connection); + delta_waiting_requests_count, delta_wait_time_ms, delta_max_wait_time_ms, sample_interval_seconds) +VALUES ($1, $2, $3, $4, $5, 0, 0, 0, $6, $7, 0, $8)", connection); command.Parameters.AddWithValue(collectionId); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(LatchServerId); @@ -268,6 +292,7 @@ INSERT INTO latch_stats command.Parameters.AddWithValue(latchClass); command.Parameters.AddWithValue(deltaReqs); command.Parameters.AddWithValue(deltaWait); + command.Parameters.Add(new NpgsqlParameter { Value = (object?)sampleIntervalSeconds ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer }); await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); } diff --git a/Darling/Darling.Tests/ViewerOverviewExplainsItselfTests.cs b/Darling/Darling.Tests/ViewerOverviewExplainsItselfTests.cs index 783591671..5cc26e8ea 100644 --- a/Darling/Darling.Tests/ViewerOverviewExplainsItselfTests.cs +++ b/Darling/Darling.Tests/ViewerOverviewExplainsItselfTests.cs @@ -49,6 +49,7 @@ private static ServerSummaryItem Busy(string name = "b1", int id = 2) => CpuPercent = 96, BlockingCount = 6, MaxBlockingWaitMs = 70000, + CollectorCount = 40, // #3539 A6: the collectors row is measured only with a denominator declared }; private static ServerSummaryItem Stale(string name = "s1", int id = 3) => @@ -121,6 +122,82 @@ public void TheCardsTooltip_OnAHealthyCard_DoesNotClaimItNeedsAttention() Assert.Contains("Healthy", tooltip, StringComparison.Ordinal); } + // ── The band label carries the measured-metric qualifier (#3528 / #3563) ─────────────────────── + + /// A card whose six severities all carry real readings — the only shape that earns the + /// unqualified all-clear. Deadlocks need a non-zero window to band (a zero window reads Unknown). + private static ServerSummaryItem FullyMeasured(string name = "f1", int id = 7) => + new() + { + DisplayName = name, + ServerId = id, + IsOnline = true, + CpuPercent = 50, + TotalThreads = 512, + CurrentWorkers = 100, + DeadlockWindow = TimeSpan.FromHours(1), + BlockingWindow = TimeSpan.FromHours(1), // #3539 A3: a zero count is measured only over a window + CollectorCount = 40, // #3539 A6: zero failing is measured only with a denominator + }; + + /// + /// The #3528 card: the band's fold SKIPS Unknown, so an online PostgreSQL target with five of six + /// metrics structurally Unknown still bands Healthy — and this tooltip claimed "every metric on this + /// card is inside its threshold" for it, an affirmative statement about five readings that were never + /// taken. The web fleet card says "1 of 6 measured" (#3562); the WPF card's band label now says the + /// same, wording and gate alike, so the two surfaces read alike. + /// + [Fact] + public void TheCardsTooltip_QualifiesAHealthyBand_ThatFoldedOverUnmeasuredMetrics() + { + /* No CPU/threads snapshot, DMV-sourced memory/blocking/deadlocks nulled by the engine — only the + collector row measured, which since #3539 A6 means its denominator is declared: forty banded, + none failing. The same shape DarlingFleetReader's card serializes as 1-of-6. */ + var pg = Healthy(); + pg.IsPostgres = true; + pg.CollectorCount = 40; + + Assert.Equal(1, pg.MeasuredMetricCount); + Assert.Equal(6, pg.MetricCount); + Assert.StartsWith("Healthy — 1 of 6 measured", pg.StatusTooltip, StringComparison.Ordinal); + Assert.DoesNotContain("every metric on this card", pg.StatusTooltip, StringComparison.Ordinal); + + /* The counts are the shared classifier's own fold over the card's metrics — the service's + measured_metric_count / metric_count pair, not a viewer re-derivation. */ + Assert.Equal( + ServerHealthClassifier.MeasuredMetricCounts(pg.ToHealthMetrics()), + (pg.MeasuredMetricCount, pg.MetricCount)); + } + + /// The all-clear's "every metric" claim survives — but only where it is true. A fully-measured + /// healthy card is unchanged by #3563, which is what keeps the qualifier a qualifier rather than a new + /// line every green card carries. + [Fact] + public void TheCardsTooltip_KeepsTheUnqualifiedAllClear_WhenEveryMetricIsMeasured() + { + var card = FullyMeasured(); + + Assert.Equal(6, card.MeasuredMetricCount); + Assert.Equal(6, card.MetricCount); + Assert.StartsWith( + "Healthy — every metric on this card is inside its threshold", card.StatusTooltip, StringComparison.Ordinal); + Assert.DoesNotContain("measured", card.StatusTooltip, StringComparison.Ordinal); + } + + /// A partially-measured PROBLEM card keeps its reason and gains the qualifier beside it — the + /// web card appends the coverage to every card it is short on, not just the green ones, and the reason + /// must stay the ranking's sentence verbatim (the drift-prevention this file pins). + [Fact] + public void TheCardsTooltip_CarriesTheQualifierBesideTheReason_OnAPartiallyMeasuredProblemCard() + { + /* Busy(): CPU, memory, blocking and collectors measured; threads and deadlocks Unknown. */ + var card = Busy(); + + Assert.Equal(4, card.MeasuredMetricCount); + Assert.StartsWith("Critical — CPU 96%, Blocking 6 · 4 of 6 measured", card.StatusTooltip, StringComparison.Ordinal); + Assert.Contains(FleetRollup.BuildReason(card), card.StatusTooltip, StringComparison.Ordinal); + } + /// Offline and awaiting-first-collection already come back as whole sentences naming themselves, so /// the band label is not stamped in front of them a second time. [Fact] diff --git a/Darling/Darling.Tests/ViewerOverviewLanesTests.cs b/Darling/Darling.Tests/ViewerOverviewLanesTests.cs index 5ff22509a..26a26bb5c 100644 --- a/Darling/Darling.Tests/ViewerOverviewLanesTests.cs +++ b/Darling/Darling.Tests/ViewerOverviewLanesTests.cs @@ -7,6 +7,7 @@ */ using System; +using System.Linq; using System.Threading.Tasks; using Npgsql; using NpgsqlTypes; @@ -39,11 +40,15 @@ public void TotalWaitTrendSql_SumsAllTypesPerCollection_PerSecondFromLagInterval Assert.Contains("collection_time >= $2", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); Assert.Contains("collection_time <= $3", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); - /* SUM across ALL wait types per collection (the single-line total), the LAG-derived collection - interval, and the per-second division with the delta CAST to double for the typed reader. */ + /* SUM across ALL wait types per collection (the single-line total), the collection's STORED interval + (MAX over its rows, 0 → NULL) with the LAG-derived interval only for pre-V127 collections (#3540), + and the per-second division with the delta CAST to double for the typed reader — no ELSE 0. */ Assert.Contains("SUM(delta_wait_time_ms)", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); + Assert.Contains("CASE WHEN MAX(sample_interval_seconds) IS NULL", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); Assert.Contains("LAG(collection_time)", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); - Assert.Contains("CAST(total_delta_ms AS double precision) / interval_seconds", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(MAX(sample_interval_seconds), 0)", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); + Assert.Contains("CAST(total_delta_ms AS double precision) / interval_seconds END AS wait_time_ms_per_second", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); Assert.Contains("GROUP BY collection_time", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); Assert.Contains("ORDER BY collection_time", ViewerDataService.TotalWaitTrendSql, StringComparison.Ordinal); } @@ -117,8 +122,9 @@ public void MemoryTrendSql_ReadsColumnsThatExistInTheGeneratedMemoryTable() /// /// Gated (DARLING_TEST_PG) live round-trips for the W1d Overview-lanes reads: the total-wait per-second -/// rate (SUM across all types divided by the LAG-derived collection interval, first collection reading 0 -/// on a NULL interval), the four-MB memory read (numeric→double), and the per-lane baseline lookup +/// rate (SUM across all types divided by the collection's stored interval, LAG-derived for pre-V127 rows; +/// a first collection with no interval and a restart's unknowable collection are both ABSENT, #3540), the +/// four-MB memory read (numeric→double), and the per-lane baseline lookup /// (graceful on no history — the baseline COMPUTATION itself is /// covered by the Analysis suite; this pins the viewer's delegation). Shares the serialized /// "live-postgres" collection; uses negative sentinel server_ids and cleans up in finally. @@ -153,24 +159,28 @@ public async Task TotalWaitTrend_SumsAcrossTypes_PerSecondFromLagInterval_Agains { var t1 = TruncateToSeconds(DateTime.UtcNow.AddMinutes(-10)); var t2 = t1.AddSeconds(60); - - /* Two collections, two wait types each. The first collection has no prior LAG (interval - NULL → per-second rate 0); the second is 60 seconds later, so its total delta divides by - 60. Totals sum ACROSS types: t2 = (60 + 120) / 60 = 3.0 ms/sec. */ + var t3 = t2.AddSeconds(60); + var t4 = t3.AddSeconds(60); + + /* Four collections, two wait types each (#3540). t1/t2 are pre-V127 rows (NULL interval): t1 + has no prior and is NOT a point (it used to read as 0.00); t2 is 60 s later, so its total + divides by the LAG's 60: (60 + 120) / 60 = 3.0 ms/sec. t3 is a restart — every row stores + interval 0 — and must be ABSENT rather than 0.00. t4 stores a measured 30 s on one row and 0 + on the other (a wait type first seen this pass): MAX = 30 wins over the LAG's 60, so + (90 + 0) / 30 = 3.0 ms/sec. */ await InsertWaitRowAsync(connection, 1, t1, "WAIT_A", 100); await InsertWaitRowAsync(connection, 1, t1, "WAIT_B", 200); await InsertWaitRowAsync(connection, 2, t2, "WAIT_A", 60); await InsertWaitRowAsync(connection, 2, t2, "WAIT_B", 120); + await InsertWaitRowAsync(connection, 3, t3, "WAIT_A", 0, sampleIntervalSeconds: 0); + await InsertWaitRowAsync(connection, 3, t3, "WAIT_B", 0, sampleIntervalSeconds: 0); + await InsertWaitRowAsync(connection, 4, t4, "WAIT_A", 90, sampleIntervalSeconds: 30); + await InsertWaitRowAsync(connection, 4, t4, "WAIT_C", 0, sampleIntervalSeconds: 0); - var points = await viewer.GetTotalWaitTrendAsync(WaitServerId, t1.AddMinutes(-1), t2.AddMinutes(1)); - - Assert.Equal(2, points.Count); - - /* Ordered by collection_time; first collection's NULL interval reads as 0. */ - Assert.Equal(t1.Ticks, points[0].CollectionTime.Ticks); - Assert.Equal(0.0, points[0].WaitTimeMsPerSecond, precision: 3); + var points = await viewer.GetTotalWaitTrendAsync(WaitServerId, t1.AddMinutes(-1), t4.AddMinutes(1)); - Assert.Equal(t2.Ticks, points[1].CollectionTime.Ticks); + Assert.Equal(new[] { t2.Ticks, t4.Ticks }, points.Select(p => p.CollectionTime.Ticks).ToArray()); + Assert.Equal(3.0, points[0].WaitTimeMsPerSecond, precision: 3); Assert.Equal(3.0, points[1].WaitTimeMsPerSecond, precision: 3); bodySucceeded = true; @@ -251,20 +261,23 @@ so the lane renderer's guard skips the band. Proves the viewer's PgBaselineProvi } private static async Task InsertWaitRowAsync( - NpgsqlConnection connection, long collectionId, DateTime collectionTimeUtc, string waitType, long deltaWaitTimeMs) + NpgsqlConnection connection, long collectionId, DateTime collectionTimeUtc, string waitType, long deltaWaitTimeMs, + int? sampleIntervalSeconds = null) { + /* sample_interval_seconds NULL by default — a pre-V127 row; 0 is the unknowable marker (#3540). */ using var command = new NpgsqlCommand(@" INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, waiting_tasks_count, wait_time_ms, signal_wait_time_ms, - delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms) -VALUES ($1, $2, $3, $4, $5, 0, 0, 0, 0, $6, 0)", connection); + delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms, sample_interval_seconds) +VALUES ($1, $2, $3, $4, $5, 0, 0, 0, 0, $6, 0, $7)", connection); command.Parameters.AddWithValue(collectionId); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(WaitServerId); command.Parameters.AddWithValue(WaitServerName); command.Parameters.AddWithValue(waitType); command.Parameters.AddWithValue(deltaWaitTimeMs); + command.Parameters.Add(new NpgsqlParameter { Value = (object?)sampleIntervalSeconds ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer }); await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); } diff --git a/Darling/Darling.Tests/ViewerPgIndexUsageGridQualifiedNameTests.cs b/Darling/Darling.Tests/ViewerPgIndexUsageGridQualifiedNameTests.cs new file mode 100644 index 000000000..3c81e5383 --- /dev/null +++ b/Darling/Darling.Tests/ViewerPgIndexUsageGridQualifiedNameTests.cs @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; +using static Darling.Tests.RepoFile; + +namespace Darling.Tests; + +/// +/// #3576, the Postgres sibling. The viewer's Index Usage grid showed the bare table name while the five other +/// table-naming grids on the Postgres tab show Schema as a column, so public.events and +/// archive.events read as one table. The store's reader carried SchemaName from the start; the +/// drop happened in TWO places — the display row had no SchemaName member for the mapper to fill, and +/// the grid bound TableName — so this pins the schema through the REAL mapper () +/// rather than through a hand-built display row, and then pins the grid's binding from source. A pin on the +/// property alone would pass with the mapper still dropping the schema, which is exactly the shape that was +/// shipped. Darling-only: Lite has no Postgres tab and the deprecated Dashboard has no such grid. +/// +public sealed class ViewerPgIndexUsageGridQualifiedNameTests +{ + /* ---------------- through the mapper ---------------- */ + + [Fact] + public void Mapper_CarriesTheSchema_AndFullNameIsSchemaDotTable() + { + var projected = PgDisplay.IndexUsage(Row(schema: "archive", table: "events")); + + Assert.Equal("archive", projected.SchemaName); + Assert.Equal("events", projected.TableName); + Assert.Equal("archive.events", projected.FullName); + } + + [Fact] + public void Mapper_NullSchema_FallsBackToBareTable() + { + /* The reader's SchemaName is nullable; the mapper writes "" for null, and "" is the fallback input. */ + var projected = PgDisplay.IndexUsage(Row(schema: null, table: "events")); + + Assert.Equal("", projected.SchemaName); + Assert.Equal("events", projected.FullName); + } + + [Fact] + public void FullName_DistinguishesTheSameTableNameAcrossSchemas() + { + /* The reported defect in one assertion: these two rows used to render identically. */ + var publicRow = PgDisplay.IndexUsage(Row(schema: "public", table: "events")); + var archiveRow = PgDisplay.IndexUsage(Row(schema: "archive", table: "events")); + + Assert.Equal(publicRow.TableName, archiveRow.TableName); + Assert.NotEqual(publicRow.FullName, archiveRow.FullName); + } + + /* ---------------- the grid, from source ---------------- */ + + [Fact] + public void IndexUsageGrid_TableColumn_BindsFullName() + { + var xaml = ReadRepoFile(Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerServerTab.xaml")); + + var start = xaml.IndexOf("x:Name=\"PgIndexUsageGrid\"", StringComparison.Ordinal); + Assert.True(start >= 0, "ViewerServerTab.xaml: no element named PgIndexUsageGrid — the grid was renamed or removed."); + var end = xaml.IndexOf("", start, StringComparison.Ordinal); + Assert.True(end > start, "ViewerServerTab.xaml: PgIndexUsageGrid has no closing ."); + var grid = xaml.Substring(start, end - start); + + Assert.Contains("Header=\"Table\" Binding=\"{Binding FullName}\"", grid, StringComparison.Ordinal); + + /* The regression's fingerprint. */ + Assert.DoesNotContain("Binding=\"{Binding TableName}\"", grid, StringComparison.Ordinal); + } + + /// + /// The reader row with only the two facts under test varied. Every other argument is inert for the + /// mapper's schema/table handling; the droppability inputs are the shape DarlingPgIndexUsageReaderTests + /// uses for a plain, valid, non-constraint index. + /// + private static DarlingPgIndexUsageReader.PgIndexUsageRow Row(string? schema, string table) => + new( + DatabaseName: "appdb", + SchemaName: schema, + TableName: table, + IndexName: "events_idx", + MeasuredAt: DateTime.UtcNow, + TotalScans: 0, + ScansInWindow: 0, + TuplesRead: 0, + TuplesFetched: 0, + BlocksRead: 0, + BlocksHit: 0, + IndexBytes: 1_000_000, + TableBytes: 9_000_000, + IsUnique: false, + IsPrimaryKey: false, + IsValid: true, + IsReady: true, + IsReplicaIdentity: false, + IsPartial: false, + IsExpression: false, + SupportsConstraint: false, + IndexMethod: "btree", + ColumnCount: 1, + IndexDefinition: "CREATE INDEX events_idx ON archive.events (a)", + LastScan: null, + StatsReset: null, + FirstSeenAt: DateTime.UtcNow.AddDays(-30), + SampleCount: 5, + StatsWereResetInWindow: false); +} diff --git a/Darling/Darling.Tests/ViewerQueriesRestTests.cs b/Darling/Darling.Tests/ViewerQueriesRestTests.cs index 73090f708..b10372c1c 100644 --- a/Darling/Darling.Tests/ViewerQueriesRestTests.cs +++ b/Darling/Darling.Tests/ViewerQueriesRestTests.cs @@ -53,6 +53,33 @@ public void TrendSql_ComputesPerSecondRate_ViaLagInterval_BaseTable(string sqlNa Assert.Contains("ORDER BY collection_time", sql, StringComparison.Ordinal); } + /// + /// #3540 (V128): the procedure trend reads the collection's STORED interval — MAX over the collection's + /// rows, 0 → NULL through NULLIF so a restart's marker collection drops rather than plotting 0.00 — and + /// falls back to the LAG derivation only for a pre-V128 collection (NULL). No ELSE 0 anywhere in it: the + /// rate is NULL when the interval is unknowable or absent and the reader drops the point. Its + /// query-stats sibling deliberately keeps the LAG-only form (the A11a residual, reported not rewritten). + /// + [Fact] + public void ProcedureDurationTrendSql_PrefersTheStoredInterval_AndNeverFabricatesZero() + { + var sql = ViewerDataService.ProcedureDurationTrendSql; + Assert.Contains("CASE WHEN MAX(sample_interval_seconds) IS NULL", sql, StringComparison.Ordinal); + Assert.Contains("ELSE NULLIF(MAX(sample_interval_seconds), 0)", sql, StringComparison.Ordinal); + Assert.Contains("THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time))))", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", sql, StringComparison.Ordinal); + Assert.Contains("CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds END AS elapsed_ms_per_second", sql, StringComparison.Ordinal); + Assert.Contains("CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds END AS executions_per_second", sql, StringComparison.Ordinal); + + /* And the shared reader DROPS a NULL-rate row rather than reading it as 0 — the C# half of the idiom. */ + var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.QueryTrends.cs"); + var reader = source[source.IndexOf("private async Task> ReadDurationTrendAsync(", StringComparison.Ordinal)..]; + reader = reader[..reader.IndexOf("return items;", StringComparison.Ordinal)]; + Assert.Contains("if (reader.IsDBNull(1))", reader, StringComparison.Ordinal); + Assert.Contains("continue;", reader, StringComparison.Ordinal); + Assert.DoesNotContain("reader.IsDBNull(1) ? 0", reader, StringComparison.Ordinal); + } + [Fact] public void DurationTrendSql_SumsElapsedMs_ExecutionTrendSql_OnlyExecutions() { diff --git a/Darling/Darling.Tests/ViewerQueriesTests.cs b/Darling/Darling.Tests/ViewerQueriesTests.cs index 06ce9e2a2..e332f8422 100644 --- a/Darling/Darling.Tests/ViewerQueriesTests.cs +++ b/Darling/Darling.Tests/ViewerQueriesTests.cs @@ -317,6 +317,15 @@ public void SlicerSql_BucketsByHour_SevenColumnShape(string sqlName, string tabl /* The bucket key and the window filter agree, stated as the invariant rather than left implicit in two InlineData columns that a future edit could change one of. */ Assert.Contains(windowFilter, bucketExpression, StringComparison.Ordinal); + + /* #3556: ReadQueryStatsSlicerAsync maps the IO columns by ORDINAL (4 reads, 5 writes, 6 physical) + for all three slicers, so every SELECT must keep the three aliases in that relative order — a + reorder would silently swap series under the sort-driven metric labels. */ + var reads = sql.IndexOf("AS total_reads", StringComparison.Ordinal); + var writes = sql.IndexOf("AS total_writes", StringComparison.Ordinal); + var physical = sql.IndexOf("AS total_physical_reads", StringComparison.Ordinal); + Assert.True(reads >= 0 && writes > reads && physical > writes, + $"{sqlName}: expected total_reads, then total_writes, then total_physical_reads in the SELECT."); } /// @@ -902,6 +911,12 @@ await InsertQueryStoreAsync(connection, DedupServerId, bucketStart.AddMinutes(g. Assert.Equal(13.0, bucket.TotalCpu, 3); Assert.Equal(282.0, bucket.TotalElapsed, 3); + /* #3556: the insert helper's fixed per-execution averages (logical 100, physical 10) make the + two IO series distinct, so the QS slicer feeding physical from the logical ordinal — or vice + versa — goes red. 41 deduped executions x 100 / x 10. */ + Assert.Equal(4100.0, bucket.TotalReads, 3); + Assert.Equal(410.0, bucket.TotalPhysicalReads, 3); + /* ── the comparison ── this read groups by (database, query_hash), which is COARSER than the interval grain, and the seed gives both queries the same hash — so it is also the pin that dedup happens at the INTERVAL grain FIRST and only then re-aggregates up to the hash. @@ -933,6 +948,10 @@ agrees with the deduped bars it is drawn over instead of showing a rising stairc Assert.Equal(bucketStart.AddMinutes(15), point.PointTime); Assert.Equal(280.0, point.ElapsedMs, 3); /* 40 x 7,000us; un-deduped this is 3 points, 50/150/280 */ Assert.Equal(12.0, point.CpuMs, 3); /* 40 x 300us */ + /* #3556's overlay half: the physical-sorted bars now draw under a physical overlay, so the + timeline's logical/physical split gets the same distinct-value pin as the bars'. */ + Assert.Equal(4000.0, point.Reads, 3); /* 40 x 100 logical */ + Assert.Equal(400.0, point.PhysicalReads, 3); /* 40 x 10 physical */ /* ── the MCP / REST surface ── the same dedup, so an agent and the web dashboard see the grid's numbers rather than the inflated ones. */ @@ -1289,9 +1308,11 @@ public async Task QueryStatsSlicer_BucketsByHour_AgainstDevPostgres() try { await InsertQueryStatsAsync(connection, SlicerServerId, hour1.AddMinutes(5), "DB", "0xA", - deltaExec: 1, deltaWorker: 60_000, deltaElapsed: 120_000, deltaReads: 10, queryText: "a"); + deltaExec: 1, deltaWorker: 60_000, deltaElapsed: 120_000, deltaReads: 60, queryText: "a", + deltaWrites: 20, deltaPhysicalReads: 30); await InsertQueryStatsAsync(connection, SlicerServerId, hour1.AddMinutes(35), "DB", "0xB", - deltaExec: 1, deltaWorker: 60_000, deltaElapsed: 120_000, deltaReads: 10, queryText: "b"); + deltaExec: 1, deltaWorker: 60_000, deltaElapsed: 120_000, deltaReads: 50, queryText: "b", + deltaWrites: 10, deltaPhysicalReads: 20); await InsertQueryStatsAsync(connection, SlicerServerId, hour2.AddMinutes(5), "DB", "0xA", deltaExec: 1, deltaWorker: 30_000, deltaElapsed: 60_000, deltaReads: 10, queryText: "a"); @@ -1303,6 +1324,15 @@ await InsertQueryStatsAsync(connection, SlicerServerId, hour2.AddMinutes(5), "DB Assert.Equal(2, first.SessionCount); /* two distinct query hashes in hour1 */ Assert.Equal(120.0, first.TotalCpu, 3); /* (60000 + 60000) us / 1000 -> ms */ + /* #3556's distinct-per-column pin (Lite's ProcStatsSlicerReadTests twin, same 110/30/50 + signature): the three slicers share ReadQueryStatsSlicerAsync, so mapping any IO ordinal to + the wrong bucket field goes red here — equal fixture values are exactly how a swap would stay + invisible. TotalReads and TotalLogicalReads are deliberate aliases of the LOGICAL aggregate. */ + Assert.Equal(110.0, first.TotalReads, 3); + Assert.Equal(110.0, first.TotalLogicalReads, 3); + Assert.Equal(30.0, first.TotalWrites, 3); + Assert.Equal(50.0, first.TotalPhysicalReads, 3); + bodySucceeded = true; } finally @@ -1316,7 +1346,8 @@ await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, async (cleanup, cleanupCt) = private static async Task InsertQueryStatsAsync( NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, string databaseName, string queryHash, - long deltaExec, long deltaWorker, long deltaElapsed, long deltaReads, string queryText, string sqlHandle = "0xSQLHANDLE") + long deltaExec, long deltaWorker, long deltaElapsed, long deltaReads, string queryText, string sqlHandle = "0xSQLHANDLE", + long deltaWrites = 0, long deltaPhysicalReads = 0) { using var command = new NpgsqlCommand(@" INSERT INTO query_stats @@ -1344,8 +1375,8 @@ INSERT INTO query_stats command.Parameters.AddWithValue(deltaElapsed); command.Parameters.AddWithValue(deltaReads); command.Parameters.AddWithValue(0L); - command.Parameters.AddWithValue(0L); - command.Parameters.AddWithValue(0L); + command.Parameters.AddWithValue(deltaWrites); + command.Parameters.AddWithValue(deltaPhysicalReads); command.Parameters.AddWithValue(0L); command.Parameters.AddWithValue(0L); command.Parameters.AddWithValue(1L); diff --git a/Darling/Darling.Tests/ViewerRecommendationsTests.cs b/Darling/Darling.Tests/ViewerRecommendationsTests.cs index b9ca551f3..c495edc31 100644 --- a/Darling/Darling.Tests/ViewerRecommendationsTests.cs +++ b/Darling/Darling.Tests/ViewerRecommendationsTests.cs @@ -638,6 +638,82 @@ public void InsufficientData_UsesEngineMessageWhenPresent_ElseTheDefault() Assert.Equal(RecommendationsState.InsufficientData, RecommendationsViewModel.InsufficientData(null).State); } + // ── Window-empty vs all-clear state selection (#3524/#3551, the marker's false-with-a-message shape) ── + + [Fact] + public void FromFindings_ZeroFindings_WindowEmptyMarker_ShowsCollectionBroken_NotAllClear() + { + // window-empty + zero findings -> WindowEmpty ("collection appears broken"), NOT a false all-clear. + var vm = RecommendationsViewModel.FromFindings( + Array.Empty(), "SQL2022", utcOffsetMinutes: 0, + windowEmpty: true, + windowEmptyMessage: "No facts were collected in the analysis window."); + + Assert.Equal(RecommendationsState.WindowEmpty, vm.State); + Assert.Empty(vm.Sections); + Assert.StartsWith("No facts were collected in the analysis window.", vm.WindowEmptyMessage); + Assert.EndsWith(RecommendationsViewModel.WindowEmptyCollectionHealthPointer, vm.WindowEmptyMessage); + Assert.Equal(string.Empty, vm.InsufficientDataMessage); + } + + [Fact] + public void FromFindings_WindowEmptyMarker_ButFindingsPresent_FindingsWin_Loaded() + { + // Same rule as the insufficient marker: it only decides the zero-finding case. + var rows = new List { Row(1.6, "CPU is on fire", incidentId: "a") }; + + var vm = RecommendationsViewModel.FromFindings( + rows, "SQL2022", utcOffsetMinutes: 0, windowEmpty: true, windowEmptyMessage: "window empty"); + + Assert.Equal(RecommendationsState.Loaded, vm.State); + Assert.Single(vm.Sections); + Assert.Equal(string.Empty, vm.WindowEmptyMessage); + } + + [Fact] + public void WindowEmpty_UsesMarkerMessageWhenPresent_ElseTheDefault_AlwaysWithThePointer() + { + Assert.StartsWith( + RecommendationsViewModel.DefaultWindowEmptyMessage, + RecommendationsViewModel.WindowEmpty(null).WindowEmptyMessage); + Assert.StartsWith( + RecommendationsViewModel.DefaultWindowEmptyMessage, + RecommendationsViewModel.WindowEmpty(" ").WindowEmptyMessage); + Assert.StartsWith( + "engine says the window held nothing", + RecommendationsViewModel.WindowEmpty("engine says the window held nothing").WindowEmptyMessage); + Assert.EndsWith( + RecommendationsViewModel.WindowEmptyCollectionHealthPointer, + RecommendationsViewModel.WindowEmpty(null).WindowEmptyMessage); + Assert.Equal(RecommendationsState.WindowEmpty, RecommendationsViewModel.WindowEmpty(null).State); + } + + [Fact] + public void AnalysisStateMarker_WindowEmpty_IsExactlyTheFalseWithMessageShape() + { + // The marker encoding contract (#3551): false + message = window-empty, the shape only the + // worker's window-empty arm writes. Every other persisted shape must NOT read as window-empty — + // true + message is the span-gate miss, false + null/empty is a clean pass. + var at = new DateTime(2026, 9, 1, 12, 0, 0, DateTimeKind.Utc); + Assert.True(new AnalysisStateMarker(false, "window empty", at).WindowEmpty); + Assert.False(new AnalysisStateMarker(false, null, at).WindowEmpty); + Assert.False(new AnalysisStateMarker(false, "", at).WindowEmpty); + Assert.False(new AnalysisStateMarker(true, "still collecting", at).WindowEmpty); + } + + [Fact] + public void FromFindings_BothMarkers_InsufficientWinsDefensively() + { + // The writer never sets both (the engine nulls both and sets at most one), but if a skewed + // store ever did, the span-gate miss is the more fundamental answer. + var vm = RecommendationsViewModel.FromFindings( + Array.Empty(), "SQL2022", utcOffsetMinutes: 0, + insufficientData: true, insufficientDataMessage: "collecting", + windowEmpty: true, windowEmptyMessage: "window empty"); + + Assert.Equal(RecommendationsState.InsufficientData, vm.State); + } + [Fact] public void FromFindings_GroupsByIncident_HeaderNamesPrimaryPlusCount() { diff --git a/Darling/Darling.Tests/ViewerSlicerMetricMapTests.cs b/Darling/Darling.Tests/ViewerSlicerMetricMapTests.cs new file mode 100644 index 000000000..27d8dcd3a --- /dev/null +++ b/Darling/Darling.Tests/ViewerSlicerMetricMapTests.cs @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System.Text.RegularExpressions; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3547/#3556's Darling half — the twin of Lite.Tests.QuerySlicerMetricMapTests. The viewer's grid +/// sorting handlers translate a sorted column name into a slicer metric key and a label, and the value +/// switch below each translates that key into a bucket field — three hops where a physical-reads sort +/// silently plotted the LOGICAL aggregate (both the procedures and Query Store grids here; Lite fixed its +/// copies in #3550/#3567 and this port stayed broken), and where slice TOTALS were labeled "Avg". The +/// handlers are private event handlers on a WPF UserControl, so this pins the source text rather than +/// instantiating the control; the value-level halves are pinned with distinct per-column fixture values by +/// the live slicer and dedup tests in ViewerQueriesLivePostgresTests, so between them a +/// logical/physical swap on either side of the seam goes red. +/// +public sealed class ViewerSlicerMetricMapTests +{ + private const string QueriesSource = "Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Queries.cs"; + + private static string HandlerBody(string handlerName) + { + var source = ParitySourceLocal.ReadFile(QueriesSource); + var match = Regex.Match( + source, + @"private void " + handlerName + @".*?(?=\n private )", + RegexOptions.Singleline); + Assert.True(match.Success, $"{QueriesSource}: {handlerName} not found — update this pin if the handler moved."); + return match.Value; + } + + [Fact] + public void QueryStore_PhysicalSort_MapsToThePhysicalSeries() + { + var body = HandlerBody("QueryStoreGrid_Sorting"); + + Assert.Matches(@"""AvgPhysicalReads""\s*=>\s*\(""TotalPhysReads"",\s*""Total Physical Reads""\)", body); + Assert.Matches(@"""TotalPhysReads""\s*=>\s*bucket\.TotalPhysicalReads", body); + } + + /// #3556's label half: the plotted bucket values are execution-weighted slice TOTALS, so an + /// "Avg" label under-claimed what the bars showed. + [Fact] + public void QueryStore_SliceTotalMetrics_CarryTotalLabels() + { + var body = HandlerBody("QueryStoreGrid_Sorting"); + + Assert.Matches(@"""AvgLogicalReads""\s*=>\s*\(""TotalReads"",\s*""Total Reads""\)", body); + Assert.Matches(@"""AvgLogicalWrites""\s*=>\s*\(""TotalWrites"",\s*""Total Writes""\)", body); + } + + [Fact] + public void Procedures_PhysicalSort_MapsToThePhysicalSeries() + { + var body = HandlerBody("ProcedureStatsGrid_Sorting"); + + Assert.Matches(@"""TotalPhysicalReads""\s*=>\s*\(""TotalPhysReads"",\s*""Total Physical Reads""\)", body); + Assert.Matches(@"""TotalPhysReads""\s*=>\s*bucket\.TotalPhysicalReads", body); + } + + /// + /// The viewer-specific half of the fix: Lite's sorting handlers re-run the SelectionChanged handler + /// after a metric change so a selected row's overlay is re-projected onto the new metric, and this port + /// shipped without that — so honest bars would have drawn under a stale-metric overlay, the exact + /// mismatch the #3547 fix's commit warned about. Pinned per handler, because the omission was per + /// handler. + /// + [Theory] + [InlineData("QueryStatsGrid_Sorting", "QueryStatsGrid_SelectionChanged(QueryStatsGrid, null!)")] + [InlineData("ProcedureStatsGrid_Sorting", "ProcedureStatsGrid_SelectionChanged(ProcedureStatsGrid, null!)")] + [InlineData("QueryStoreGrid_Sorting", "QueryStoreGrid_SelectionChanged(QueryStoreGrid, null!)")] + public void EverySortingHandler_RecomputesTheOverlayOnMetricChange(string handlerName, string retrigger) + { + var body = HandlerBody(handlerName); + + Assert.Contains(retrigger, body, System.StringComparison.Ordinal); + } +} diff --git a/Darling/Darling.Tests/ViewerThemeColorOverridesTests.cs b/Darling/Darling.Tests/ViewerThemeColorOverridesTests.cs new file mode 100644 index 000000000..613244c7a --- /dev/null +++ b/Darling/Darling.Tests/ViewerThemeColorOverridesTests.cs @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Runtime.CompilerServices; +using System.Text.RegularExpressions; +using PerformanceMonitor.Ui; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3577 (arm B) as it lands in the Darling viewer. The loader, the key list, the rewriter and the +/// regeneration are shared PerformanceMonitor.Ui code and are pinned once, in +/// Lite.Tests/ThemeColorOverrideTests, against the theme files of BOTH apps. What is the viewer's own +/// is where its file lives and where it does not: theme-overrides.json sits beside +/// viewer-settings.json under the viewer's per-user directory, and the preference is LOCAL to the +/// machine — it never goes to the store or the control plane, because a color is a preference of the +/// person at this screen and not a fact about the fleet. A viewer seat on another machine keeps its own. +/// +public sealed class ViewerThemeColorOverridesTests +{ + private static readonly string ViewerDir = Path.Combine("Darling", "PerformanceMonitor.Darling.Viewer"); + + [Fact] + public void TheOverridesFile_SitsBesideViewerSettings_UnderTheViewersOwnDirectory() + { + var app = File.ReadAllText(Path.Combine(RepoRoot(), ViewerDir, "App.xaml.cs")); + + Assert.Contains("Path.GetDirectoryName(ViewerAppSettingsStore.DefaultFilePath())", app, StringComparison.Ordinal); + Assert.Contains("ThemeColorOverrides.FileName", app, StringComparison.Ordinal); + Assert.Equal("theme-overrides.json", ThemeColorOverrides.FileName); + } + + /// + /// The preference is not a settings-model property, not a store column and not a control-plane knob. + /// ViewerAppSettings keeps ColorTheme (which theme) and nothing about the colors inside + /// it; the service and storage projects never see the word. + /// + [Fact] + public void TheColorOverrides_AreNotPersistedInTheSettingsModel_TheStore_OrTheControlPlane() + { + var root = RepoRoot(); + + var settingsModel = File.ReadAllText(Path.Combine(root, ViewerDir, "ViewerAppSettings.cs")); + Assert.DoesNotContain("ThemeColorOverride", settingsModel, StringComparison.Ordinal); + Assert.DoesNotContain("theme-overrides", settingsModel, StringComparison.Ordinal); + + foreach (var project in new[] { "PerformanceMonitor.Darling.Storage", "PerformanceMonitor.Darling.Service" }) + { + var directory = Path.Combine(root, "Darling", project); + foreach (var file in Directory.EnumerateFiles(directory, "*.cs", SearchOption.AllDirectories)) + { + if (file.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.Ordinal)) + { + continue; + } + + var text = File.ReadAllText(file); + Assert.False(text.Contains("ThemeColorOverride", StringComparison.Ordinal) || text.Contains("theme-overrides", StringComparison.Ordinal), + $"{Path.GetRelativePath(root, file)} mentions the viewer's color overrides; they are viewer-local and must stay out of the store and the service."); + } + } + } + + /// + /// The viewer's Settings window hosts the same shared panel Lite does, beneath its theme combo, and the + /// panel is the ONLY colors surface — no viewer-only copy of the rows that could drift from Lite's. + /// + [Fact] + public void TheViewerSettingsWindow_HostsTheSharedPanel_AndNoPrivateCopy() + { + var xaml = File.ReadAllText(Path.Combine(RepoRoot(), ViewerDir, "SettingsWindow.xaml")); + + Assert.Single(Regex.Matches(xaml, "The viewer's own three theme files are the ones the shared regeneration reads — embedded under the loader's names. + [Fact] + public void TheViewerProject_EmbedsItsThemeText() + { + var csproj = File.ReadAllText(Path.Combine(RepoRoot(), ViewerDir, "PerformanceMonitor.Darling.Viewer.csproj")); + + Assert.Contains("", csproj, StringComparison.Ordinal); + foreach (var theme in ThemeColorOverrides.ThemeNames) + { + Assert.True(File.Exists(Path.Combine(RepoRoot(), ViewerDir, "Themes", theme + "Theme.xaml")), + $"{ThemeManager.ThemeTextResourceName(theme)} names a file the viewer does not ship."); + } + } + + private static string RepoRoot([CallerFilePath] string thisFile = "") + => Path.GetFullPath(Path.Combine(Path.GetDirectoryName(thisFile)!, "..", "..")); +} diff --git a/Darling/Darling.Tests/ViewerTraySnoozeTests.cs b/Darling/Darling.Tests/ViewerTraySnoozeTests.cs new file mode 100644 index 000000000..e96a78de1 --- /dev/null +++ b/Darling/Darling.Tests/ViewerTraySnoozeTests.cs @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Text.RegularExpressions; +using PerformanceMonitor.Darling.Viewer; +using PerformanceMonitor.Notifications; +using Xunit; + +namespace Darling.Tests; + +/// +/// Pins for #3570 — "Snoozing an alert in the tray does nothing". The tray toast is the viewer's own alert +/// channel, and until this issue the viewer never applied a mute rule to it: it toasted every polled row the +/// SERVICE had not stamped muted, so a Snooze suppressed the next toast only after the service noticed +/// the reload beacon, re-read its whole config view, refreshed its rule cache, and re-fired the alert through +/// it. Nothing viewer-side observed that chain; when it was slow or broken the rule sat in +/// config_mute_rules while the toasts kept coming. +/// +/// The decision logic that closes it — judging polled +/// rows against the viewer's own rule set — is pinned in ViewerAlertToastCoordinatorTests. This file +/// pins the two ends of the loop that feed it: the rule the Snooze WRITES +/// () and the context a row is JUDGED as +/// (), and that the one matches the other by construction — the +/// spelling question. Plus the honesty of the status line's "within N s", which restates a service constant the +/// viewer cannot reference. The WPF surfaces themselves (the balloon, the status bar) are not unit-testable; +/// everything that decides what they say is. +/// +public sealed class ViewerTraySnoozeTests +{ + private static readonly DateTime T0 = new(2026, 9, 18, 7, 22, 53, DateTimeKind.Utc); + + private static ViewerAlertRow Row(string serverName, string metric, string? detail = null, bool muted = false) => new() + { + AlertTime = T0, + ServerId = 7, + ServerName = serverName, + MetricName = metric, + CurrentValue = 0, + ThresholdValue = 0, + AlertSent = false, + NotificationType = AlertDelivery.ChannelNoneConfigured, + Muted = muted, + DetailText = detail, + }; + + /* ---------------- the rule the Snooze writes ---------------- */ + + [Fact] + public void BuildTraySnoozeRule_ScopesToTheRowsServerAndMetric_ExpiringAfterTheDuration() + { + var rule = ViewerDataService.BuildTraySnoozeRule("sql-prod-01", "Agent Not Running", TimeSpan.FromHours(4), T0); + + Assert.Equal("sql-prod-01", rule.ServerName); + Assert.Equal("Agent Not Running", rule.MetricName); + Assert.True(rule.Enabled); + Assert.Equal(T0, rule.CreatedAtUtc); + Assert.Equal(T0.AddHours(4), rule.ExpiresAtUtc); + Assert.Equal("Snoozed from tray (4h)", rule.Reason); + + /* A snooze is "this alert on this server" — never narrower. */ + Assert.Null(rule.DatabasePattern); + Assert.Null(rule.QueryTextPattern); + Assert.Null(rule.WaitTypePattern); + Assert.Null(rule.JobNamePattern); + + /* And never broader: a snooze must not be the blanket rule the Stale Mute Rules self-alert flags CRITICAL. */ + Assert.False(rule.MatchesEveryAlert); + Assert.False(string.IsNullOrEmpty(rule.Id)); + } + + [Theory] + [InlineData(15, "15m")] + [InlineData(60, "1h")] + [InlineData(240, "4h")] + public void BuildTraySnoozeRule_ReasonNamesTheButtonThatWroteIt(int minutes, string label) + { + var rule = ViewerDataService.BuildTraySnoozeRule("s", "m", TimeSpan.FromMinutes(minutes), T0); + + Assert.Equal($"{ViewerDataService.TraySnoozeReasonPrefix} ({label})", rule.Reason); + Assert.Equal(label, ViewerDataService.FormatSnoozeDuration(TimeSpan.FromMinutes(minutes))); + } + + /// A row with no server name yields a rule for every server — the only honest scope for a row + /// that did not say — never an empty-string server that would match nothing at all. + [Theory] + [InlineData(null)] + [InlineData("")] + public void BuildTraySnoozeRule_EmptyServerName_BecomesEveryServer(string? serverName) + { + var rule = ViewerDataService.BuildTraySnoozeRule(serverName, "Agent Not Running", TimeSpan.FromHours(1), T0); + + Assert.Null(rule.ServerName); + Assert.Equal("Agent Not Running", rule.MetricName); + Assert.False(rule.MatchesEveryAlert); /* still metric-scoped */ + } + + /* ---------------- the closed loop: the rule the toast writes matches the row the toast came from ---------------- */ + + /// + /// The spelling question, answered by construction. Alert rows on a Darling store spell server_name + /// two ways — the self-alert family writes the server's display name, the shared engine writes the name its + /// snapshot carries — and a mute rule must match the row's OWN spelling to suppress that row's producer. The + /// toast captures and off the + /// row, the rule is built from exactly those, and the viewer's filter judges the same row through + /// : whatever the spelling, the loop closes. Pinned over both + /// shapes so a future "normalize the server name" on one side but not the other fails here. + /// + [Theory] + [InlineData("SQL01", "Agent Not Running")] /* self-alert family: display name */ + [InlineData("sql01.corp.example.internal", "Failed Agent Job")] /* engine family: the snapshot's name */ + [InlineData("sql01,1433", "High CPU")] /* a display name with a port */ + public void TheRuleASnoozeWrites_MatchesTheRowItWasSnoozedFrom(string serverName, string metric) + { + var row = Row(serverName, metric, detail: " Job Name: Nightly ETL\n"); + var rule = ViewerDataService.BuildTraySnoozeRule(row.ServerName, row.MetricName, TimeSpan.FromHours(4), T0); + + Assert.True(rule.MatchesAt(row.ToMuteContext(), T0.AddMinutes(5))); + Assert.True(AlertToastCoordinator.IsMutedByViewerRules(row, new[] { rule }, T0.AddMinutes(5))); + + /* And the SAME rule, judged the way the service judges it — a bare server+metric context, no detail + text — also matches, so the tray and the service's channels agree about this snooze. */ + Assert.True(rule.MatchesAt(new AlertMuteContext { ServerName = serverName, MetricName = metric }, T0.AddMinutes(5))); + } + + /// The rule stops matching the instant it expires, on the clock it is judged with — no ambient UtcNow. + [Fact] + public void TheRuleASnoozeWrites_LapsesExactlyAtItsExpiry() + { + var row = Row("SQL01", "Agent Not Running"); + var rule = ViewerDataService.BuildTraySnoozeRule(row.ServerName, row.MetricName, TimeSpan.FromMinutes(15), T0); + + Assert.True(rule.MatchesAt(row.ToMuteContext(), T0.AddMinutes(15).AddTicks(-1))); + Assert.False(rule.MatchesAt(row.ToMuteContext(), T0.AddMinutes(15))); + } + + /* ---------------- the context a row is judged as ---------------- */ + + [Fact] + public void ToMuteContext_CarriesTheRowsServerAndMetricVerbatim() + { + var context = Row("SQL01", "Agent Not Running").ToMuteContext(); + + Assert.Equal("SQL01", context.ServerName); + Assert.Equal("Agent Not Running", context.MetricName); + Assert.Null(context.DatabaseName); + Assert.Null(context.WaitType); + Assert.Null(context.JobName); + Assert.Null(context.QueryText); + } + + /// The pattern dimensions come from the stored detail text, the same way the "Mute This Alert" + /// pre-fill has always read them — so a rule authored from a row covers that row's toast. + [Fact] + public void ToMuteContext_ParsesThePatternDimensionsOutOfDetailText() + { + var detail = "2 job failure(s)\n Job Name: Nightly ETL\n Database: SalesDb\n Wait Type: LCK_M_X\n Query: SELECT 1"; + var context = Row("SQL01", "Failed Agent Job", detail).ToMuteContext(); + + Assert.Equal("Nightly ETL", context.JobName); + Assert.Equal("SalesDb", context.DatabaseName); + Assert.Equal("LCK_M_X", context.WaitType); + Assert.Equal("SELECT 1", context.QueryText); + + var jobRule = new MuteRule { MetricName = "Failed Agent Job", JobNamePattern = "Nightly" }; + Assert.True(jobRule.MatchesAt(context, T0)); + } + + /// #3309 holds here too: a custom alert's detail text is a user-authored name, never parsed for + /// labels, so a crafted "Database: master" line cannot pre-fill or match a pattern dimension. + [Fact] + public void ToMuteContext_CustomAlert_DoesNotParseDetailText() + { + var context = Row("SQL01", "Custom:42", detail: "Database: master").ToMuteContext(); + + Assert.Equal("Custom:42", context.MetricName); + Assert.Null(context.DatabaseName); + } + + /* ---------------- the status line's "within N s" is the service's real cadence ---------------- */ + + /// + /// The snooze status line tells the operator the service's channels stop "within N s (the service's next + /// sweep)". N is , a restatement of + /// DarlingWorker.s_sweepInterval — the tick at whose top the service polls the reload beacon — which + /// the viewer cannot reference. Read out of the service's source (the field is private) so the sentence + /// cannot outlive the cadence it describes. + /// + [Fact] + public void ServiceReloadTickSeconds_IsTheServicesSweepTick() + { + var sweepTick = CommandPlaneCommandTimeoutTests.SecondsOfPrivateTimeSpan("DarlingWorker.cs", "s_sweepInterval"); + + Assert.Equal(ViewerDataService.ServiceReloadTickSeconds, sweepTick); + } + + /* ---------------- the wiring that makes the coordinator's new argument reach it ---------------- */ + + /// + /// 's rule-set parameter is OPTIONAL (null = the pre-#3570 + /// behavior), so a refactor that drops the argument at the one production call site compiles clean and + /// silently reverts the fix. This reads MainWindow.xaml.cs and asserts the call passes the viewer's + /// rule set, and that the rule-set refresh (UpdateServerSilencedAsync, which also drives the sidebar + /// bell) is awaited BEFORE it in the same poll — the ordering the rules-then-toasts contract rests on. + /// + [Fact] + public void PollAlertsAsync_PassesTheViewersRuleSet_AfterRefreshingIt() + { + var body = MemberBody(ViewerSource("MainWindow.xaml.cs"), "PollAlertsAsync"); + + var refresh = body.IndexOf("await UpdateServerSilencedAsync()", StringComparison.Ordinal); + var select = Regex.Match(body, @"SelectToasts\s*\([^;]*?_viewerMuteRules\s*\)", RegexOptions.Singleline); + + Assert.True(refresh >= 0, "PollAlertsAsync no longer awaits UpdateServerSilencedAsync — the toast filter's rule set is never refreshed"); + Assert.True(select.Success, "PollAlertsAsync's SelectToasts call no longer passes _viewerMuteRules — the tray has stopped honoring mute rules (#3570 regressed)"); + Assert.True(refresh < select.Index, "PollAlertsAsync selects toasts BEFORE refreshing the rule set — a rule read this poll reaches the tray a poll late"); + } + + /// + /// The two local writers (tray Snooze, server Silence) add their rule to the viewer's set only AFTER the + /// store write succeeds — persist-then-cache, 's ordering — so a + /// snooze that did not persist never suppresses toasts on this seat while every other surface says no such + /// rule exists. + /// + [Fact] + public void LocalRuleWriters_AddToTheViewersSet_OnlyAfterTheStoreWrite() + { + var snooze = MemberBody(ViewerSource("MainWindow.xaml.cs"), "SnoozeAlertAsync"); + AssertPersistThenCache(snooze, "SnoozeAlertAsync"); + + var silence = MemberBody(ViewerSource("MainWindow.ServerManagement.cs"), "ServerContextMenu_Silence_Click"); + AssertPersistThenCache(silence, "ServerContextMenu_Silence_Click"); + } + + private static void AssertPersistThenCache(string body, string member) + { + var insert = body.IndexOf("InsertMuteRuleAsync(", StringComparison.Ordinal); + var cache = body.IndexOf("_viewerMuteRules.Add(", StringComparison.Ordinal); + + Assert.True(insert >= 0, $"{member} no longer writes the rule to the store"); + Assert.True(cache >= 0, $"{member} no longer hands its rule to the toast filter (#3570 regressed for this writer)"); + Assert.True(insert < cache, $"{member} caches the rule before persisting it — a failed write would suppress toasts for a rule that does not exist"); + } + + /* ---------------- helpers ---------------- */ + + /// The text of one viewer source file, through the shared root resolver (worktree-safe). + private static string ViewerSource(string file) => + RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", file); + + /// + /// The CODE of one method, from its declaration to the brace that closes it, over the comment/string-stripped + /// source ( preserves offsets) so a brace or a name + /// inside a comment or a status-line string can neither close the body early nor satisfy an assertion meant + /// for code. Anchored on the DECLARATION (Task NAME( / void NAME() rather than the bare name: + /// both methods this file reads are also CALLED earlier in their files, and the first bare match would hand + /// back whatever method happens to follow that call. + /// + private static string MemberBody(string source, string member) + { + var stripped = CSharpSourceWalker.StripCommentsAndStrings(source); + var signature = Regex.Match( + stripped, @"\b(?:Task|void)\s+" + Regex.Escape(member) + @"\s*\(", RegexOptions.CultureInvariant); + Assert.True(signature.Success, $"could not find the declaration of {member}( in the viewer source"); + + var open = stripped.IndexOf('{', signature.Index); + Assert.True(open >= 0, $"could not find the opening brace of {member}"); + + var depth = 0; + for (var i = open; i < stripped.Length; i++) + { + if (stripped[i] == '{') + { + depth++; + } + else if (stripped[i] == '}' && --depth == 0) + { + return stripped.Substring(signature.Index, i - signature.Index + 1); + } + } + + Assert.Fail($"unbalanced braces while reading {member}"); + return ""; + } +} diff --git a/Darling/Darling.Tests/ViewerW2aTests.cs b/Darling/Darling.Tests/ViewerW2aTests.cs index 42250d3de..75ea36574 100644 --- a/Darling/Darling.Tests/ViewerW2aTests.cs +++ b/Darling/Darling.Tests/ViewerW2aTests.cs @@ -111,6 +111,25 @@ public void SummaryDeadlockSql_CountsOverTheWindow_AndNewestEver() Assert.Contains("MAX(deadlock_time)", sql, StringComparison.Ordinal); } + /// + /// #3539: the PostgreSQL arm of the same card read — the server's own pg_stat_database.deadlocks + /// counter differenced per database over the window, clamped at zero across a reset, summed, with the + /// sample that first showed the newest step as "last". Never SUM(deadlocks): the column is a + /// lifetime counter repeated in every sample. Per-server, so partitioned by database only. + /// + [Fact] + public void SummaryPgDeadlockSql_DifferencesTheCounterPerDatabase_OverTheWindow() + { + var sql = ViewerDataService.ServerSummaryPgDeadlockSql; + Assert.Contains("FROM pg_database_stats", sql, StringComparison.Ordinal); + Assert.Contains("WHERE server_id = $1", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $2", sql, StringComparison.Ordinal); + Assert.Contains("deadlocks - LAG(deadlocks) OVER (PARTITION BY database_name ORDER BY collection_time)", sql, StringComparison.Ordinal); + Assert.Contains("SUM(GREATEST(sampled.raw_delta, 0))", sql, StringComparison.Ordinal); + Assert.Contains("MAX(sampled.collection_time) FILTER (WHERE sampled.raw_delta > 0)", sql, StringComparison.Ordinal); + Assert.DoesNotContain("SUM(deadlocks)", sql, StringComparison.Ordinal); + } + [Fact] public void SummaryLastCollectionSql_TakesTheNewestCollectionTime() { @@ -131,6 +150,7 @@ public void SummaryReads_ArePgDialect_PositionalParams_NoBareNow_NoNLiterals() ViewerDataService.ServerSummaryThreadsSql, ViewerDataService.ServerSummaryBlockingSql, ViewerDataService.ServerSummaryDeadlockSql, + ViewerDataService.ServerSummaryPgDeadlockSql, ViewerDataService.ServerSummaryLastCollectionSql, }) { @@ -453,22 +473,54 @@ public void MemorySeverity_AnyPressure_IsCritical_DetailNamesIt(long waiters, lo Assert.Equal(expectedDetail, item.MemoryDetail); } + /// #3539 A3: the count arm is a RATE over the card's window, so the fixture declares the hour + /// the card reads — one to four reports is the measured quiet mode and Healthy by count, five the + /// Warning tier, twenty the Critical one; the wait arms band whatever the rate. [Theory] [InlineData(0, 0, HealthSeverity.Healthy)] - [InlineData(1, 0, HealthSeverity.Warning)] // any blocking at all → Warning - [InlineData(2, 0, HealthSeverity.Warning)] // still the "any blocking" arm; #3368 removed the indistinguishable >=2 one - [InlineData(5, 0, HealthSeverity.Critical)] // >=5 events → Critical - [InlineData(1, 10000, HealthSeverity.Warning)] // 10s max wait → Warning + [InlineData(1, 0, HealthSeverity.Healthy)] // 1/hr — the quiet mode, Healthy by count + [InlineData(4, 0, HealthSeverity.Healthy)] + [InlineData(5, 0, HealthSeverity.Warning)] // 5/hr → Warning + [InlineData(20, 0, HealthSeverity.Critical)] // 20/hr → Critical + [InlineData(1, 10000, HealthSeverity.Warning)] // 10s max wait → Warning, whatever the rate [InlineData(1, 59000, HealthSeverity.Warning)] - [InlineData(1, 60000, HealthSeverity.Critical)] // 60s max wait → Critical - public void BlockingSeverity_BandsOnCountAndDuration(int count, long maxWaitMs, HealthSeverity expected) + [InlineData(1, 60000, HealthSeverity.Critical)] // 60s max wait → Critical, whatever the rate + public void BlockingSeverity_BandsOnRateAndDuration(int count, long maxWaitMs, HealthSeverity expected) + { + Assert.Equal(expected, new ServerSummaryItem + { + BlockingCount = count, + MaxBlockingWaitMs = maxWaitMs, + BlockingWindow = TimeSpan.FromHours(1), + }.BlockingSeverity); + } + + /// A card built with no blocking window bands the count on the unrateable arm — a non-zero + /// count Warning, never Critical by count, a zero count Unknown — and publishes no rate; the same + /// count over a declared 24-hour window is a rate of its own. The pin that goes red on a revert to + /// counting, on the viewer's own card. + [Fact] + public void BlockingSeverity_NeedsTheWindow_ToBandTheCount() { - Assert.Equal(expected, new ServerSummaryItem { BlockingCount = count, MaxBlockingWaitMs = maxWaitMs }.BlockingSeverity); + var undeclared = new ServerSummaryItem { BlockingCount = 20 }; + Assert.Equal(HealthSeverity.Warning, undeclared.BlockingSeverity); + Assert.Null(undeclared.BlockingRatePerHour); + Assert.Equal(HealthSeverity.Unknown, new ServerSummaryItem { BlockingCount = 0 }.BlockingSeverity); + + var day = new ServerSummaryItem { BlockingCount = 20, BlockingWindow = TimeSpan.FromHours(24) }; + Assert.Equal(HealthSeverity.Healthy, day.BlockingSeverity); // 0.8/hr + Assert.Equal(20 / 24.0, day.BlockingRatePerHour!.Value, precision: 6); } [Fact] public void BlockingDetail_MaxWhenBlocked_LastAgoWhenClear_BlankWhenNever() { + /* #3539 A3: the banded RATE leads while blocking is present (the deadlock detail's rule), then the + worst wait; an undeclared window prints the wait alone. */ + Assert.Equal("3.0/hr, max: 42s", new ServerSummaryItem + { + BlockingCount = 3, MaxBlockingWaitMs = 42000, BlockingWindow = TimeSpan.FromHours(1), + }.BlockingDetail); Assert.Equal("max: 42s", new ServerSummaryItem { BlockingCount = 3, MaxBlockingWaitMs = 42000 }.BlockingDetail); /* Window clear but blocking happened earlier → the Dashboard's "Last: N ago". */ Assert.Equal("Last: 3h ago", new ServerSummaryItem { BlockingCount = 0, LastBlockingMinutesAgo = 180 }.BlockingDetail); @@ -527,12 +579,37 @@ the whole reason the rate is rendered at all. */ [Fact] public void CollectorSeverity_FailingIsWarning_HealthyOtherwise() { - Assert.Equal(HealthSeverity.Healthy, new ServerSummaryItem { HealthyCollectorCount = 30 }.CollectorSeverity); - var failing = new ServerSummaryItem { HealthyCollectorCount = 28, FailedCollectorCount = 2 }; + Assert.Equal(HealthSeverity.Healthy, new ServerSummaryItem { HealthyCollectorCount = 30, CollectorCount = 30 }.CollectorSeverity); + var failing = new ServerSummaryItem { HealthyCollectorCount = 28, FailedCollectorCount = 2, CollectorCount = 30 }; Assert.Equal(HealthSeverity.Warning, failing.CollectorSeverity); Assert.Equal("2 failed", failing.CollectorDisplay); Assert.Equal("Healthy: 28, Failing: 2", failing.CollectorDetail); - Assert.Equal("OK", new ServerSummaryItem { HealthyCollectorCount = 30 }.CollectorDisplay); + Assert.Equal("OK", new ServerSummaryItem { HealthyCollectorCount = 30, CollectorCount = 30 }.CollectorDisplay); + + /* #3539 A6: with NO collector banded the dot is Unknown and the word is the card's "no reading" + spelling, not a green "OK" — a reachable server nothing has classified yet is not a clean one. */ + var nothingBanded = new ServerSummaryItem { IsOnline = true }; + Assert.Equal(HealthSeverity.Unknown, nothingBanded.CollectorSeverity); + Assert.Equal("--", nothingBanded.CollectorDisplay); + Assert.Equal("Healthy: 0, Failing: 0", nothingBanded.CollectorDetail); + } + + /// #3539 A8d: the collector dot is graded on the FAILING share — one of forty is Warning, + /// nine of forty (past the collector-health classifier's 20% bar) is Critical, forty of forty is + /// Critical; a card with no denominator declared is Warning and never Critical. + [Fact] + public void CollectorSeverity_GradesOnTheFailingShare() + { + Assert.Equal(HealthSeverity.Warning, new ServerSummaryItem { FailedCollectorCount = 1, CollectorCount = 40 }.CollectorSeverity); + Assert.Equal(HealthSeverity.Warning, new ServerSummaryItem { FailedCollectorCount = 8, CollectorCount = 40 }.CollectorSeverity); + Assert.Equal(HealthSeverity.Critical, new ServerSummaryItem { FailedCollectorCount = 9, CollectorCount = 40 }.CollectorSeverity); + Assert.Equal(HealthSeverity.Critical, new ServerSummaryItem { FailedCollectorCount = 40, CollectorCount = 40 }.CollectorSeverity); + Assert.Equal(HealthSeverity.Warning, new ServerSummaryItem { FailedCollectorCount = 40 }.CollectorSeverity); + + /* The graded dot rides into the card's overall band through ToHealthMetrics — a server with half + its collection dark for a day is a Critical card, not an amber one. */ + var dark = new ServerSummaryItem { IsOnline = true, FailedCollectorCount = 20, CollectorCount = 40 }; + Assert.Equal(HealthSeverity.Critical, dark.OverallMetricSeverity); } [Fact] @@ -557,19 +634,19 @@ public void CollectorSeverity_OfflineServer_ReadsStaleNeutral_NotGreenOk() // A GENUINE collector failure on a reachable server still surfaces red / "N failed" — not swallowed // into Stale. - var failing = new ServerSummaryItem { IsOnline = true, HealthyCollectorCount = 28, FailedCollectorCount = 2 }; + var failing = new ServerSummaryItem { IsOnline = true, HealthyCollectorCount = 28, FailedCollectorCount = 2, CollectorCount = 30 }; Assert.Equal("2 failed", failing.CollectorDisplay); Assert.Equal(HealthSeverity.Warning, failing.CollectorSeverity); - // A healthy ONLINE server is unchanged — green "OK". - var healthy = new ServerSummaryItem { IsOnline = true, HealthyCollectorCount = 30, FailedCollectorCount = 0 }; + // A healthy ONLINE server is unchanged — green "OK" (its thirty banded collectors declared, #3539 A6). + var healthy = new ServerSummaryItem { IsOnline = true, HealthyCollectorCount = 30, FailedCollectorCount = 0, CollectorCount = 30 }; Assert.Equal("OK", healthy.CollectorDisplay); Assert.Equal("Healthy: 30, Failing: 0", healthy.CollectorDetail); Assert.Equal(HealthSeverity.Healthy, healthy.CollectorSeverity); // Not-yet-connection-classified (IsOnline null — awaiting first collection) keeps the normal reading: // "Stale" is for a KNOWN-offline server only, matching the web's strict `is_online === false`. - var notChecked = new ServerSummaryItem { HealthyCollectorCount = 30, FailedCollectorCount = 0 }; + var notChecked = new ServerSummaryItem { HealthyCollectorCount = 30, FailedCollectorCount = 0, CollectorCount = 30 }; Assert.False(notChecked.IsOffline); Assert.Equal("OK", notChecked.CollectorDisplay); Assert.Equal(HealthSeverity.Healthy, notChecked.CollectorSeverity); @@ -903,17 +980,23 @@ public async Task ServerSummary_ReadsEnrichedThreadsMemoryBlockingCollectors_Aga Assert.True(summary.HasMemoryPressure); Assert.Equal(HealthSeverity.Critical, summary.MemorySeverity); - /* Blocking — count + worst wait, both from the XE source; last-event read populated. */ + /* Blocking — count + worst wait, both from the XE source; last-event read populated. Two in + the card's hour is 2.0/hr (the quiet mode); the 42 s wait is the arm that bands Warning + (#3539 A3), and the detail names both. */ Assert.Equal(2, summary.BlockingCount); Assert.Equal(42000, summary.MaxBlockingWaitMs); - Assert.Equal("max: 42s", summary.BlockingDetail); + Assert.Equal(TimeSpan.FromHours(1), summary.BlockingWindow); + Assert.Equal("2.0/hr, max: 42s", summary.BlockingDetail); Assert.Equal(HealthSeverity.Warning, summary.BlockingSeverity); Assert.NotNull(summary.LastBlockingMinutesAgo); - /* Collectors — REUSE of the 7-day banding (one HEALTHY, one FAILING). */ + /* Collectors — REUSE of the 7-day banding (one HEALTHY, one FAILING). One of two banded + collectors failing is a 50% share, past the 20% bar, so the graded dot reads Critical + (#3539 A8d) — the presence-flat Warning it used to read is the defect. */ Assert.Equal(1, summary.HealthyCollectorCount); Assert.Equal(1, summary.FailedCollectorCount); - Assert.Equal(HealthSeverity.Warning, summary.CollectorSeverity); + Assert.Equal(2, summary.CollectorCount); + Assert.Equal(HealthSeverity.Critical, summary.CollectorSeverity); bodySucceeded = true; } diff --git a/Darling/Darling.Tests/ViewerWaitStatsTests.cs b/Darling/Darling.Tests/ViewerWaitStatsTests.cs index d9ef018c8..5d9ed3fb2 100644 --- a/Darling/Darling.Tests/ViewerWaitStatsTests.cs +++ b/Darling/Darling.Tests/ViewerWaitStatsTests.cs @@ -55,14 +55,16 @@ public void WaitTrendsSql_KeepsLitesPerTypeLagPerSecondMath() Assert.Contains("WITH raw AS", sql, StringComparison.Ordinal); Assert.Contains("FROM v_wait_stats", sql, StringComparison.Ordinal); - /* The LAG window is partitioned per wait_type so each type's per-second rate is independent. */ - Assert.Contains("LAG(collection_time) OVER (PARTITION BY wait_type ORDER BY collection_time)", sql, StringComparison.Ordinal); - /* The truncate-then-diff epoch idiom proven value-identical between DuckDB and Postgres. */ - Assert.Contains("extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time)", sql, StringComparison.Ordinal); - /* The three metric expressions: per-second wait, per-second signal, avg ms per wait. */ - Assert.Contains("CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds", sql, StringComparison.Ordinal); - Assert.Contains("CAST(delta_signal_wait_time_ms AS DOUBLE PRECISION) / interval_seconds", sql, StringComparison.Ordinal); - Assert.Contains("CAST(delta_wait_time_ms AS DOUBLE PRECISION) / delta_waiting_tasks", sql, StringComparison.Ordinal); + /* #3540: the STORED interval first — 0 (the calculator's unknowable marker) mapped to NULL — and the + per-type LAG window (the truncate-then-diff epoch idiom proven value-identical between DuckDB and + Postgres) only for pre-V127 rows that never recorded one. */ + ViewerLatchSpinlockSqlTests.AssertStoredIntervalIdiom(sql, "wait_type"); + /* The three metric expressions: per-second wait, per-second signal, avg ms per wait — none with an + ELSE 0, so an unknowable interval yields NULL and the reader drops the row. */ + Assert.Contains("CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds END AS wait_time_ms_per_second", sql, StringComparison.Ordinal); + Assert.Contains("CAST(delta_signal_wait_time_ms AS DOUBLE PRECISION) / interval_seconds END AS signal_wait_time_ms_per_second", sql, StringComparison.Ordinal); + Assert.Contains("interval_seconds > 0 AND delta_waiting_tasks > 0 THEN CAST(delta_wait_time_ms AS DOUBLE PRECISION) / delta_waiting_tasks", sql, StringComparison.Ordinal); + Assert.DoesNotContain("ELSE 0", sql, StringComparison.Ordinal); Assert.Contains("ORDER BY wait_type, collection_time", sql, StringComparison.Ordinal); } @@ -278,34 +280,43 @@ public async Task WaitTrends_ComputePerSecondAndAvgPerWait_ByType_AgainstDevPost var bodySucceeded = false; try { - var t1 = TruncateToSeconds(DateTime.UtcNow.AddMinutes(-10)); + var t1 = TruncateToSeconds(DateTime.UtcNow.AddMinutes(-20)); var t2 = t1.AddMinutes(5); // 300 seconds later + var t3 = t2.AddMinutes(5); + var t4 = t3.AddMinutes(5); - /* CXPACKET across two collections; WRITELOG is a second type that must NOT appear when - only CXPACKET is requested (the IN filter). */ + /* CXPACKET across four collections; WRITELOG is a second type that must NOT appear when + only CXPACKET is requested (the IN filter). t1/t2 are pre-V127 rows (NULL interval); t3 + stores interval 0 — the calculator's "no delta knowable" marker, a restart (#3540); t4 + stores a measured 120 s. */ await InsertWaitRowAsync(connection, 1, t1, "CXPACKET", deltaWait: 100, deltaSignal: 10, deltaTasks: 5); await InsertWaitRowAsync(connection, 2, t2, "CXPACKET", deltaWait: 600, deltaSignal: 60, deltaTasks: 3); await InsertWaitRowAsync(connection, 2, t2, "WRITELOG", deltaWait: 900, deltaSignal: 90, deltaTasks: 9); + await InsertWaitRowAsync(connection, 3, t3, "CXPACKET", deltaWait: 0, deltaSignal: 0, deltaTasks: 0, sampleIntervalSeconds: 0); + await InsertWaitRowAsync(connection, 4, t4, "CXPACKET", deltaWait: 1200, deltaSignal: 120, deltaTasks: 4, sampleIntervalSeconds: 120); var trends = await viewer.GetWaitStatsTrendsByTypesAsync( - WaitServerId, new List { "CXPACKET" }, t1.AddMinutes(-1), t2.AddMinutes(1)); + WaitServerId, new List { "CXPACKET" }, t1.AddMinutes(-1), t4.AddMinutes(1)); Assert.True(trends.ContainsKey("CXPACKET")); Assert.False(trends.ContainsKey("WRITELOG")); // the IN filter excluded it var cx = trends["CXPACKET"]; - Assert.Equal(2, cx.Count); - /* First point: no prior collection → per-second 0; avg = 100/5 = 20. */ - Assert.Equal(t1.Ticks, cx[0].CollectionTime.Ticks); - Assert.Equal(0.0, cx[0].WaitTimeMsPerSecond, precision: 3); - Assert.Equal(20.0, cx[0].AvgMsPerWait, precision: 3); + /* Two points, not four: t1 has no prior collection and no stored interval (it used to plot as + 0.00 ms/sec — the fabricated first point), and t3 is the unknowable marker, which must be + ABSENT rather than a confident 0.00 at exactly the moment (a restart) nothing is knowable. */ + Assert.Equal(new[] { t2.Ticks, t4.Ticks }, cx.Select(p => p.CollectionTime.Ticks).ToArray()); + + /* t2 (pre-V127): 600 ms over the LAG's 300 s = 2.0 ms/sec; signal 60/300 = 0.2; avg 600/3 = 200. */ + Assert.Equal(2.0, cx[0].WaitTimeMsPerSecond, precision: 3); + Assert.Equal(0.2, cx[0].SignalWaitTimeMsPerSecond, precision: 3); + Assert.Equal(200.0, cx[0].AvgMsPerWait, precision: 3); - /* Second point: 600 ms over 300 s = 2.0 ms/sec; signal 60/300 = 0.2; avg 600/3 = 200. */ - Assert.Equal(t2.Ticks, cx[1].CollectionTime.Ticks); - Assert.Equal(2.0, cx[1].WaitTimeMsPerSecond, precision: 3); - Assert.Equal(0.2, cx[1].SignalWaitTimeMsPerSecond, precision: 3); - Assert.Equal(200.0, cx[1].AvgMsPerWait, precision: 3); + /* t4: the STORED 120 s wins over the LAG's 300 s — 1200/120 = 10.0 ms/sec, signal 1.0, avg 300. */ + Assert.Equal(10.0, cx[1].WaitTimeMsPerSecond, precision: 3); + Assert.Equal(1.0, cx[1].SignalWaitTimeMsPerSecond, precision: 3); + Assert.Equal(300.0, cx[1].AvgMsPerWait, precision: 3); bodySucceeded = true; } @@ -318,14 +329,16 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup private static async Task InsertWaitRowAsync( NpgsqlConnection connection, long collectionId, DateTime collectionTimeUtc, - string waitType, long deltaWait, long deltaSignal, long deltaTasks) + string waitType, long deltaWait, long deltaSignal, long deltaTasks, int? sampleIntervalSeconds = null) { + /* sample_interval_seconds NULL by default — a pre-V127 row; pass 0 (unknowable) or a measured value + for the V127 contract (#3540). */ using var command = new NpgsqlCommand(@" INSERT INTO wait_stats (collection_id, collection_time, server_id, server_name, wait_type, waiting_tasks_count, wait_time_ms, signal_wait_time_ms, - delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms) -VALUES ($1, $2, $3, $4, $5, 0, 0, 0, $6, $7, $8)", connection); + delta_waiting_tasks, delta_wait_time_ms, delta_signal_wait_time_ms, sample_interval_seconds) +VALUES ($1, $2, $3, $4, $5, 0, 0, 0, $6, $7, $8, $9)", connection); command.Parameters.AddWithValue(collectionId); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(WaitServerId); @@ -334,6 +347,7 @@ INSERT INTO wait_stats command.Parameters.AddWithValue(deltaTasks); command.Parameters.AddWithValue(deltaWait); command.Parameters.AddWithValue(deltaSignal); + command.Parameters.Add(new NpgsqlParameter { Value = (object?)sampleIntervalSeconds ?? DBNull.Value, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Integer }); await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); } diff --git a/Darling/Darling.Tests/XminHorizonPersistenceReadTests.cs b/Darling/Darling.Tests/XminHorizonPersistenceReadTests.cs new file mode 100644 index 000000000..9faa683e4 --- /dev/null +++ b/Darling/Darling.Tests/XminHorizonPersistenceReadTests.cs @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Alerting; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3537: the xmin read's two window figures, against a real Postgres — the horizon-arm numerator (winning +/// age at/above the threshold, holder ignored) and its denominator (the collector's OWN successful runs, +/// off collection_log, so quiet captures count). +/// +/// Why live rather than a source pin. XminSql has no execution coverage anywhere else: +/// a pin can assert the captures CTE exists, but only a store can prove the filters count — that an +/// ERROR run, another collector's run and another server's run all stay out of the denominator while a +/// zero-row healthy run stays in, and that a below-threshold winner stays out of the numerator while still +/// counting as a holder-bearing observation. Each seeded row here is one of those predicates. +/// +[Collection("live-postgres")] +public sealed class XminHorizonPersistenceReadTests +{ + private const int ServerId = -353701; + private const int OtherServerId = -353702; + private const string ServerName = "xmin-persistence-read"; + private const string Collector = "pg_xmin_horizon"; + + private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + /// + /// One rotating-holder window, end to end: four held collections under three distinct winning pids + /// (one below the age bar), two quiet captures, and three log rows that must not count. The read's + /// figures are asserted exactly, then handed to the evaluator to prove the fire this issue exists for: + /// the horizon arm, under the stable rotating-holders subject, where the identity fraction (1 of 4) + /// never could. + /// + [Fact] + public async Task TheXminRead_CountsCapturesAndAboveThreshold_AgainstDevPostgres() + { + var cs = ConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(cs), + "Set DARLING_TEST_PG to a Postgres connection string to run the live xmin-persistence test."); + + var ct = TestContext.Current.CancellationToken; + using var connection = new NpgsqlConnection(cs); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, ct); + + var bodySucceeded = false; + try + { + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await using var postgres = NpgsqlDataSource.Create(cs!); + var adapter = new DarlingPostgresAlertReadAdapter(postgres); + + /* The empty store first: no holder rows means no row at all, the healthy null — and the one + execution of the full statement that cannot hide behind seeded data if the SQL stops + parsing. */ + Assert.Null(await adapter.GetXminHorizonAsync(ServerId, ct)); + + /* Four collections, three distinct winning pids — the parade. The 30M winner sits below the + 50M evaluator threshold, so it is a holder-bearing observation that must NOT count as an + above-threshold one. The t-10 loser row shares its winner's collection_time, pinning that + extra rows per collection inflate nothing. */ + await SeedHolderAsync(connection, ct, MinutesAgo(10), "session", 60_000_000, "101", "state=idle in transaction", isWinner: true); + await SeedHolderAsync(connection, ct, MinutesAgo(10), "replication_slot", 10_000_000, "slot_a", null, isWinner: false); + await SeedHolderAsync(connection, ct, MinutesAgo(8), "session", 70_000_000, "102", null, isWinner: true); + await SeedHolderAsync(connection, ct, MinutesAgo(6), "session", 30_000_000, "103", null, isWinner: true); + await SeedHolderAsync(connection, ct, MinutesAgo(4), "session", 80_000_000, "104", "state=idle in transaction", isWinner: true); + + /* The capture denominator: the four holder-bearing runs, two QUIET (zero-row, healthy) runs + the holder table cannot see — the whole reason the denominator lives in collection_log — + and three rows that must stay out of it: a run that failed, another collector's run, and + another server's. */ + await SeedLogAsync(connection, ct, MinutesAgo(10), ServerId, Collector, "SUCCESS", 2); + await SeedLogAsync(connection, ct, MinutesAgo(8), ServerId, Collector, "SUCCESS", 1); + await SeedLogAsync(connection, ct, MinutesAgo(6), ServerId, Collector, "SUCCESS", 1); + await SeedLogAsync(connection, ct, MinutesAgo(4), ServerId, Collector, "SUCCESS", 1); + await SeedLogAsync(connection, ct, MinutesAgo(2), ServerId, Collector, "SUCCESS", 0); + await SeedLogAsync(connection, ct, MinutesAgo(1), ServerId, Collector, "SUCCESS", 0); + await SeedLogAsync(connection, ct, MinutesAgo(3), ServerId, Collector, "ERROR", 0); + await SeedLogAsync(connection, ct, MinutesAgo(5), ServerId, "pg_database_stats", "SUCCESS", 4); + await SeedLogAsync(connection, ct, MinutesAgo(7), OtherServerId, Collector, "SUCCESS", 1); + + var info = await adapter.GetXminHorizonAsync(ServerId, ct); + + Assert.NotNull(info); + Assert.Equal("session", info!.Source); + Assert.Equal("104", info.Identifier); + Assert.Equal(80_000_000L, info.XminAge); + Assert.Equal(1, info.ObservationsHeld); + Assert.Equal(4, info.ObservationsTotal); + Assert.Equal(3, info.ObservationsAboveThreshold); + Assert.Equal(6, info.CapturesInWindow); + + /* And the consequence the figures exist for: 3 of 6 captures is the horizon arm's majority + where the identity fraction is 1 of 4 — the rotating-holder fire, under the subject the + host's per-subject cooldown can actually hold across rotations. */ + var finding = PostgresAlertEvaluator.EvaluateXmin(info); + Assert.NotNull(finding); + Assert.Equal(PostgresAlertEvaluator.XminRotatingHoldersSubject, finding!.Subject); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(cs!, bodySucceeded, DeleteRowsAsync); + } + } + + /* ── helpers ── */ + + private static DateTime MinutesAgo(int minutes) => + DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow.AddMinutes(-minutes)); + + private static Task SeedHolderAsync( + NpgsqlConnection connection, CancellationToken ct, + DateTime collectionTimeUtc, string source, long xminAge, string holder, string? detail, bool isWinner) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO pg_xmin_horizon + (collection_id, collection_time, server_id, server_name, source, xmin_age, holder, detail, is_winner) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + CollectionIdGenerator.Next(), collectionTimeUtc, ServerId, ServerName, + source, xminAge, holder, detail, isWinner); + + private static Task SeedLogAsync( + NpgsqlConnection connection, CancellationToken ct, + DateTime collectionTimeUtc, int serverId, string collectorName, string status, int rowsCollected) => + DarlingMcpTestData.ExecAsync(connection, ct, @" +INSERT INTO collection_log + (log_id, server_id, server_name, collector_name, collection_time, + duration_ms, status, error_message, rows_collected, sql_duration_ms, duckdb_duration_ms) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + CollectionIdGenerator.Next(), serverId, ServerName, collectorName, + collectionTimeUtc, 50, status, null, rowsCollected, 40, 10); + + private static async Task DeleteRowsAsync(NpgsqlConnection connection, CancellationToken ct) + { + await DarlingMcpTestData.ExecAsync(connection, ct, + "DELETE FROM pg_xmin_horizon WHERE server_id = $1", ServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, + "DELETE FROM collection_log WHERE server_id IN ($1, $2)", ServerId, OtherServerId); + await DarlingMcpTestData.ExecAsync(connection, ct, + "DELETE FROM servers WHERE server_id = $1", ServerId); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs b/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs index 714fd086e..7301e8920 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/DarlingAnalysisService.cs @@ -14,6 +14,7 @@ using Microsoft.Extensions.Logging; using Npgsql; using PerformanceMonitor.Analysis; +using PerformanceMonitor.Analysis.Baselines; using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Analysis; @@ -133,6 +134,25 @@ public sealed class DarlingAnalysisService /// public string? InsufficientDataMessage { get; private set; } + /// + /// Set after AnalyzeAsync when the server PASSED the data-span gate but the analysis window itself + /// produced zero facts (#3524). Null otherwise. The gate measures TOTAL history, so a server whose + /// collection died still sails through it and lands on an empty window — which is a dead collector + /// or an unreachable target, not a healthy server. Callers must not render an empty findings list + /// as an all-clear while this is set; nothing was measured. + /// + public string? WindowEmptyMessage { get; private set; } + + /// + /// How much of the last pass's window the collector actually observed (#3538 A2), stamped by the + /// fact collector and carried out here the way is, because the + /// findings list cannot say it: a pass over a window with a three-hour hole returns the SAME shape as + /// a pass over a fully collected one, and only this tells the caller that the rates were divided by + /// one hour rather than four and that the caveat is owed. Null when the pass never reached + /// collection (the data-span gate, or a fault before it). + /// + public WindowCoverage? LastWindowCoverage { get; private set; } + /// /// How the last pass ended EARLY, or null when it ran through (#2430). Set inside the pass's own /// catch, so here means a genuine fault: the pass reached the @@ -216,6 +236,8 @@ public async Task> AnalyzeAsync(AnalysisContext context) IsAnalyzing = true; InsufficientDataMessage = null; + WindowEmptyMessage = null; + LastWindowCoverage = null; EndedEarlyAs = null; try @@ -254,13 +276,54 @@ cannot unwind from inside it — these boundary checks are what turn the token i // 1. Collect facts from the Postgres store var facts = await _collector.CollectFactsAsync(context); + LastWindowCoverage = context.Coverage; - if (facts.Count == 0) + if (facts.Count == 0 || context.ObservedDurationMs <= 0) { + /* #3524: the span gate above passed on LIFETIME history, so an empty WINDOW here means + collection stopped producing rows for it — not that the server is healthy. Say so, + instead of returning a bare [] that reads exactly like "analyzed and found nothing". + + #3538 A2 widens the branch to a window with NO OBSERVED COLLECTION TIME even when some + facts exist. The facts that survive an unobserved window are the point-in-time ones + (server config, trace flags, hardware) — read from the latest row regardless of + window — and scoring those alone would produce a pass whose every windowed rate is + absent and whose all-clear (or config-only findings) still reads as "analyzed this + window". Nothing was measured over the window; the same envelope says so. */ + /* True when the WINDOW went unobserved but point-in-time facts (config, trace flags, + hardware) still read — the case that used to slip past the facts.Count == 0 check. */ + var hasPointInTimeFactsOnly = facts.Count > 0; + WindowEmptyMessage = hasPointInTimeFactsOnly + ? $"The collector observed none of the analysis window " + + $"({context.TimeRangeStart:yyyy-MM-dd HH:mm} to {context.TimeRangeEnd:yyyy-MM-dd HH:mm} UTC): " + + $"no collection interval landed inside it, even though this server has {dataSpanHours:F1} hours " + + $"of total collected history. The {facts.Count} fact(s) that could still be read are point-in-time " + + "configuration and state, not measurements of this window. Collection appears to have stopped or " + + "broken for this window, so nothing was measured — this is NOT an all-clear." + : $"No facts were collected in the analysis window " + + $"({context.TimeRangeStart:yyyy-MM-dd HH:mm} to {context.TimeRangeEnd:yyyy-MM-dd HH:mm} UTC) " + + $"even though this server has {dataSpanHours:F1} hours of total collected history. " + + "Collection appears to have stopped or broken for this window, so nothing was measured " + + "— this is NOT an all-clear."; + + _logger?.LogWarning( + "[DarlingAnalysisService] No observed collection in the analysis window for {Server} ({Start} to {End}) " + + "despite {Span:F1}h of total history — collection may be down ({FactCount} point-in-time fact(s) only)", + context.ServerName, context.TimeRangeStart, context.TimeRangeEnd, dataSpanHours, facts.Count); + LastAnalysisTime = DateTime.UtcNow; return []; } + if (context.Coverage is { IsPartial: true } partial) + { + /* Logged, not just carried: a scheduled pass has no payload to put the caveat in, and + the persisted findings from this pass were rated against the observed hours. */ + _logger?.LogWarning( + "[DarlingAnalysisService] Partial collection coverage for {Server}: {Coverage} — rates in this pass are per observed time, and the COLLECTION_GAP fact records the hole", + context.ServerName, partial.Describe()); + } + context.CancellationToken.ThrowIfCancellationRequested(); // 1.5. Detect anomalies (compare analysis window against baseline) @@ -405,13 +468,16 @@ letting the halves drift. */ /// /// Runs the collect + score pipeline without graph traversal. - /// Returns raw scored facts with amplifier details for direct inspection. + /// Returns raw scored facts with amplifier details for direct inspection, together with the + /// window's observed coverage (#3538 A2) — returned rather than parked on a property because this + /// path has no guard and two on-demand callers can overlap; a shared + /// property would let one read the other's window. Coverage is null only when collection threw. /// /// #2506: anchors the END of the window; null is "now", which is /// every caller but the anchored MCP tool. Nothing here persists, so the anchor carries no /// write-side question — this is a read that happens to score what it read. /// - public async Task> CollectAndScoreFactsAsync( + public async Task<(List Facts, WindowCoverage? Coverage)> CollectAndScoreFactsAsync( int serverId, string serverName, int hoursBack = 4, DateTime? asOfUtc = null) { var timeRangeEnd = asOfUtc ?? DateTime.UtcNow; @@ -429,22 +495,37 @@ public async Task> CollectAndScoreFactsAsync( try { var facts = await _collector.CollectFactsAsync(context); - if (facts.Count == 0) return facts; + if (facts.Count == 0) return (facts, context.Coverage); _scorer.ScoreAll(facts); - return facts; + return (facts, context.Coverage); } catch (Exception ex) { _logger?.LogError("[DarlingAnalysisService] Fact collection failed for {Server}: {Message}", serverName, ex.Message); - return []; + return ([], null); } } /// - /// Compares analysis of two time periods, returning facts from both for comparison. + /// Compares analysis of two time periods, returning facts from both for comparison, each with its + /// window's observed coverage (#3538 A2) so the caller can say when one side was only partly + /// collected — the case the empty-window caveats never reached, where a half-collected window + /// produces confident numbers with nothing to flag them. + /// + /// #3538 A3: also returns the stored per-server dispersion for the baselined metrics some + /// compared key is measured in (), keyed by metric + /// name, so compare_analysis can band a CPU or read-latency delta in the server's own robust + /// sigma instead of on a flat severity dead-band. The bucket is the comparison window's START hour + /// × day-of-week — the same coordinate the anomaly detectors read for a pass over that window + /// (PgAnomalyDetector passes context.TimeRangeStart), so the default same-hour-yesterday + /// call reuses the pass's cached buckets and an anchored one recomputes them the way an anchored + /// analyze_server does. The lookups are fenced separately from collection: a baseline read + /// that fails must not cost the caller the comparison it was only meant to refine, so it degrades + /// to an empty map and every key takes the absolute rule — the never-blind fallback the anomaly + /// gate follows. /// - public async Task<(List BaselineFacts, List ComparisonFacts)> ComparePeriodsAsync( + public async Task<(List BaselineFacts, List ComparisonFacts, WindowCoverage? BaselineCoverage, WindowCoverage? ComparisonCoverage, IReadOnlyDictionary Dispersion)> ComparePeriodsAsync( int serverId, string serverName, DateTime baselineStart, DateTime baselineEnd, DateTime comparisonStart, DateTime comparisonEnd) @@ -473,14 +554,39 @@ public async Task> CollectAndScoreFactsAsync( _scorer.ScoreAll(baselineFacts); _scorer.ScoreAll(comparisonFacts); - return (baselineFacts, comparisonFacts); + var dispersion = await LookUpDispersionAsync(serverId, serverName, baselineFacts, comparisonFacts, comparisonStart); + + return (baselineFacts, comparisonFacts, baselineContext.Coverage, comparisonContext.Coverage, dispersion); } catch (Exception ex) { _logger?.LogError("[DarlingAnalysisService] Period comparison failed for {Server}: {Message}", serverName, ex.Message); - return ([], []); + return ([], [], null, null, new Dictionary()); + } + } + + /// + /// The baseline buckets hands to the comparison, one per metric some + /// compared key is measured in. Its own try: see the summary above for why a failed baseline read + /// degrades to "no dispersion" rather than failing the comparison. + /// + private async Task> LookUpDispersionAsync( + int serverId, string serverName, List baselineFacts, List comparisonFacts, DateTime comparisonStart) + { + var dispersion = new Dictionary(StringComparer.Ordinal); + try + { + foreach (var metric in ComparisonBanding.DispersionMetricsFor(baselineFacts, comparisonFacts)) + dispersion[metric] = await _baselineProvider.GetBaselineAsync(serverId, metric, comparisonStart); + } + catch (Exception ex) + { + _logger?.LogWarning("[DarlingAnalysisService] Baseline dispersion lookup failed for {Server}; compare_analysis bands every key by the absolute rule: {Message}", + serverName, ex.Message); + dispersion.Clear(); } + return dispersion; } /// @@ -508,14 +614,27 @@ public async Task> GetRecentFindingsAsync( } /// - /// Mutes a finding pattern so it won't appear in future runs. + /// Mutes a finding pattern so it won't appear in future runs. Returns whether the registry row was written + /// ( logs and returns false on a store failure), so the + /// MCP mute verb reports what happened rather than what it asked for (#3541 A14). The Lite twin returns + /// Task because its store throws instead of swallowing; the caller-visible contract is the same — + /// a mute that did not land is never reported as one that did. /// - public async Task MuteFindingAsync(AnalysisFinding finding, string? reason = null) + public async Task MuteFindingAsync(AnalysisFinding finding, string? reason = null) { - await _findingStore.MuteStoryAsync( + return await _findingStore.MuteStoryAsync( finding.ServerId, finding.StoryPathHash, finding.StoryPath, reason); } + /// + /// How many stored findings carry for one server (or fleet-wide when + /// is null / the all-servers sentinel 0) — the mute verb's matched_now + /// disclosure (#3541 A14). A pass-through to ; see its + /// note for why this is reported beside the mute and not used to refuse it. + /// + public Task CountStoredFindingsAsync(int? serverId, string storyPathHash, CancellationToken cancellationToken = default) => + _findingStore.CountStoredFindingsAsync(serverId, storyPathHash, cancellationToken); + /// /// Cleans up old findings beyond the retention period. /// diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs index 776d66e4c..ecd62ed76 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs @@ -158,20 +158,29 @@ FROM v_cpu_utilization_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3"; - /* Wait-profile current window: all-types wait ms/sec per collection (interval via LAG — never an - assumed cadence, mirrors the WaitMsPerSec baseline), then PEAK across collections. Window bound - aligned to the baseline (>= $2 AND < $3). */ + /* Wait-profile current window: all-types wait ms/sec per collection (the collection's STORED interval + since V127, the LAG for pre-V127 collections — never an assumed cadence; mirrors the WaitMsPerSec + baseline), then PEAK across collections. A restart collection (every row's stored interval 0) has a + NULL interval, so it is neither a peak candidate nor counted as a sample — before #3540 it was a + sample worth 0.00 ms/sec. Window bound aligned to the baseline (>= $2 AND < $3). */ public const string WaitRateWindowSql = @" WITH per_collection AS ( SELECT collection_time, SUM(delta_wait_time_ms)::DOUBLE PRECISION AS total_wait_ms, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) AS interval_sec + /* #3540: the collection's STORED interval (MAX over its rows — a wait type first seen in an otherwise + steady pass carries 0 beside its siblings' real interval and adds 0 to the sum; MAX is 0 only when + EVERY row was unknowable, a restart) mapped through NULLIF so that collection is NOT a sample; a + pre-V127 collection (NULL) falls back to the LAG this read always used. */ + CASE WHEN MAX(sample_interval_seconds) IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) + ELSE NULLIF(MAX(sample_interval_seconds), 0) + END AS interval_sec FROM v_wait_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND delta_wait_time_ms >= 0 GROUP BY collection_time ) -SELECT MAX(CASE WHEN interval_sec > 0 THEN total_wait_ms / interval_sec ELSE 0 END) AS peak_ms_per_sec, +SELECT MAX(CASE WHEN interval_sec > 0 THEN total_wait_ms / interval_sec END) AS peak_ms_per_sec, SUM(total_wait_ms) AS total_wait_ms, COUNT(*) FILTER (WHERE interval_sec IS NOT NULL) AS sample_count FROM per_collection"; @@ -207,14 +216,20 @@ FROM v_file_io_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND (delta_reads > 0 OR delta_writes > 0)"; + /* Batch-request window: per-second rate per sample (#3527) — delta_cntr_value spans one collection + interval, so divide by the row's MEASURED sample_interval_seconds (#2234). Interval <= 0 marks an + unknowable delta (first sighting/reset/gap) and the row is skipped, never read as 0. Keeps the + window statistic in the same requests/sec unit as the baseline and the BatchRequestFloor/Fallback + thresholds. */ public const string BatchRequestWindowSql = @" -SELECT AVG(delta_cntr_value) AS avg_batch, - MAX(delta_cntr_value) AS peak_batch, +SELECT AVG(delta_cntr_value * 1.0 / NULLIF(sample_interval_seconds, 0)) AS avg_batch, + MAX(delta_cntr_value * 1.0 / NULLIF(sample_interval_seconds, 0)) AS peak_batch, COUNT(*) AS sample_count FROM v_perfmon_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND counter_name = 'Batch Requests/sec' -AND delta_cntr_value >= 0"; +AND delta_cntr_value >= 0 +AND sample_interval_seconds > 0"; public const string SessionWindowSql = @" WITH per_collection AS ( diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs index cc8e10ab5..06460b35e 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs @@ -366,23 +366,19 @@ FROM cpu_utilization_stats /* QUALIFY rewrite 1 of 4 — cumulative counter, restart exclusion. Excludes samples where the delta drops to 0 when the prior sample was > 1000 - (restart signature for cumulative counters). Lite's DuckDB original: - - SELECT EXTRACT(HOUR FROM collection_time)::INT AS hour_of_day, - EXTRACT(DOW FROM collection_time)::INT AS day_of_week, - AVG(delta_cntr_value) AS mean_val, - STDDEV_SAMP(delta_cntr_value) AS stddev_val, - COUNT(*) AS sample_count - FROM ( - SELECT collection_time, delta_cntr_value + (restart signature for cumulative counters). Lite's DuckDB original (#3527 unit: + v is delta / the row's measured sample_interval_seconds — a per-second rate): + + WITH clean AS ( + SELECT collection_time, delta_cntr_value * 1.0 / NULLIF(sample_interval_seconds, 0) AS v FROM v_perfmon_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND counter_name = 'Batch Requests/sec' AND delta_cntr_value >= 0 + AND sample_interval_seconds > 0 QUALIFY NOT (delta_cntr_value = 0 AND COALESCE(LAG(delta_cntr_value) OVER (ORDER BY collection_time), 0) > 1000) ) - GROUP BY hour_of_day, day_of_week QUALIFY evaluates AFTER window computation: LAG runs over every WHERE-surviving row (including rows QUALIFY itself is about to drop), THEN the predicate prunes. @@ -390,23 +386,40 @@ The rewrite computes the SAME LAG over the SAME WHERE-filtered rowset inside a CTE and applies the IDENTICAL predicate in the outer WHERE — window-before-filter is preserved, so only the FIRST zero after a >1000 sample is dropped, and a zero following another zero keeps LAG = 0 and SURVIVES (genuine idle, not a restart). - Row selection is exactly the original's. */ + Row selection is exactly the original's. + + #3527 divisor: this arm reads the perfmon_baseline supply (the #1757 CAGG / fallback + view), which materializes only (collection_time, delta_cntr_value) — it does NOT carry + sample_interval_seconds, and a continuous aggregate cannot grow a column without a + drop-and-rebuild that would forfeit the 31 days of baseline history the 4-day raw tier + can no longer refill. So the interval is DERIVED from the gap between consecutive + collections — LAG(collection_time) over the collapsed series, byte-for-byte the + WaitMsPerSec arm's idiom (see CreateWaitStatsBaselineSql's note: the provider computes + interval_sec, nothing extra is stored). Same requests/sec unit as Lite and as the + detector's window read; the first row of the window has no prior (interval NULL) and is + skipped, exactly like WaitMsPerSec. The ::DOUBLE PRECISION cast is the io-arm rule: + STDDEV_SAMP over numeric can overflow System.Decimal at materialization. */ MetricNames.BatchRequests => @" WITH windowed AS ( SELECT collection_time, delta_cntr_value, - COALESCE(LAG(delta_cntr_value) OVER (ORDER BY collection_time), 0) AS prior_delta + COALESCE(LAG(delta_cntr_value) OVER (ORDER BY collection_time), 0) AS prior_delta, + extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) AS interval_sec FROM perfmon_baseline WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 ), clean AS ( - SELECT collection_time, delta_cntr_value AS v + SELECT collection_time, delta_cntr_value::DOUBLE PRECISION / interval_sec AS v FROM windowed WHERE NOT (delta_cntr_value = 0 AND prior_delta > 1000) + AND interval_sec > 0 )," + RobustTierScaffold, /* QUALIFY rewrite 2 of 4 — cumulative counter, multiple rows per collection (per wait type): aggregate to total wait ms per collection FIRST, then restart - exclusion. Lite's DuckDB original applied QUALIFY inside the grouped CTE: + exclusion. Lite's DuckDB original applied QUALIFY inside the grouped CTE (since #3540 + Lite's WHERE also carries sample_interval_seconds IS DISTINCT FROM 0, dropping the + calculator's unknowable rows before the sum; this arm reads the wait_stats_baseline + aggregate, which cannot carry that filter without a rebuild — V127's rung note): WITH per_collection AS ( SELECT collection_time, @@ -542,7 +555,13 @@ FROM memory_baseline // ── Chart-unit baselines (for UI bands — units match what the chart displays) ── - /* QUALIFY rewrite 4 of 4 — wait ms per second (chart unit). Lite's DuckDB original: + /* QUALIFY rewrite 4 of 4 — wait ms per second (chart unit). Lite's DuckDB original (as it stood + when this rewrite was made; since #3540 Lite's per_collection takes the collection's STORED + sample_interval_seconds — MAX over its rows — and falls back to this LAG only for pre-v60 + rows, so a restart collection's 0 becomes NULL and is dropped by with_rate's WHERE. This arm + cannot follow yet: it reads wait_stats_baseline, which materializes only (collection_time, + total_wait_ms) and cannot grow the interval without the drop-and-rebuild the #3527 note below + declines — see V127's rung note for the new-aggregate follow-up): WITH per_collection AS ( SELECT collection_time, diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs index 60cbcf113..53c974887 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using Npgsql; using PerformanceMonitor.Analysis; +using PerformanceMonitor.Common; namespace PerformanceMonitor.Darling.Analysis; @@ -110,16 +111,56 @@ private async Task CollectQueriesAtSpike(AnalysisFinding finding, AnalysisContex } } + /// + /// #3648: max_dop here is sys.dm_exec_query_stats.max_dop — a PER-PLAN high-water mark since + /// the plan entered the cache, not a per-execution reading and not a per-statement one. This read groups + /// by (database, query_hash), so the old MAX(max_dop) folded every plan the statement text had + /// inside the window into one number and kept the largest: the highest DOP ANY plan for the hash ever ran + /// at, with nothing saying which plan, when, or whether that plan still exists. Live consequence: a High + /// CPU card read 16 for the #1 query on an instance whose MAXDOP had been 1 across its whole 14-day config + /// history and whose stored plan for that hash was serial (NonParallelPlanReason="MaxDOPSetToOne") + /// — a plan compiled before the pin, still cached with its old counter. A reader recommended a MAXDOP 1 + /// Query Store hint from the field and had to retract it after reading the plan. The same card's row #3 + /// said 1, honestly, for a hash with one serial plan — so the field was self-consistent and wrong. + /// + /// Now: max_dop is the NEWEST plan's reading (the row with the latest collection_time + /// among the rows that spent CPU in the window; ties broken by compile time, then by CPU spent), because + /// the card's question is what the query is doing to the CPU at this moment. The cross-plan maximum + /// survives as max_dop_any_plan with max_dop_any_plan_last_seen and plan_count + /// beside it — a history with provenance — and the reader coerces NULL to null, not 0: the DMV never + /// reports 0, so 0 was "no reading" rendered as a degree of parallelism. dop_note is the shared + /// sentence (PerformanceMonitor.Common.QueryDopProvenance) a renderer prints verbatim when the + /// history disagrees with the headline. New columns are appended after the old ones so the existing + /// ordinals are untouched; the SQL is byte-identical to Lite's CollectTopCpuQueries text. + /// public const string TopCpuQueriesSql = @" +WITH windowed AS +( + -- #3648: rank each hash's rows newest-first and carry the hash-wide maximum onto every row, so the + -- outer aggregate can name WHICH reading is current and WHEN the maximum was last seen. Explicit + -- NULLS LAST on the tie-breakers: DuckDB and Postgres default DESC null placement differently. + SELECT database_name, query_hash, query_plan_hash, collection_time, max_dop, + delta_worker_time, delta_execution_count, delta_spills, query_text, + ROW_NUMBER() OVER + ( + PARTITION BY database_name, query_hash + ORDER BY collection_time DESC, creation_time DESC NULLS LAST, delta_worker_time DESC NULLS LAST + ) AS newest_rn, + MAX(max_dop) OVER (PARTITION BY database_name, query_hash) AS max_dop_any_plan + FROM v_query_stats + WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 + AND delta_worker_time > 0 +) SELECT database_name, query_hash, SUM(delta_worker_time)::BIGINT AS total_cpu_us, SUM(delta_execution_count)::BIGINT AS exec_count, - MAX(max_dop) AS max_dop, + MAX(CASE WHEN newest_rn = 1 THEN max_dop END) AS max_dop, SUM(delta_spills)::BIGINT AS spills, - LEFT(MAX(query_text), 500) AS query_text -FROM v_query_stats -WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 -AND delta_worker_time > 0 + LEFT(MAX(query_text), 500) AS query_text, + COUNT(DISTINCT query_plan_hash) AS plan_count, + MAX(max_dop_any_plan) AS max_dop_any_plan, + MAX(CASE WHEN max_dop = max_dop_any_plan THEN collection_time END) AS max_dop_any_plan_last_seen +FROM windowed GROUP BY database_name, query_hash ORDER BY total_cpu_us DESC LIMIT 5"; @@ -138,15 +179,24 @@ private async Task CollectTopCpuQueries(AnalysisFinding finding, AnalysisContext using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); while (await reader.ReadAsync(context.CancellationToken)) { + var maxDop = reader.IsDBNull(4) ? (int?)null : Convert.ToInt32(reader.GetValue(4)); + var planCount = reader.IsDBNull(7) ? 0L : Convert.ToInt64(reader.GetValue(7)); + var maxDopAnyPlan = reader.IsDBNull(8) ? (int?)null : Convert.ToInt32(reader.GetValue(8)); + var maxDopAnyPlanLastSeen = reader.IsDBNull(9) ? (DateTime?)null : reader.GetDateTime(9); items.Add(new { database = reader.IsDBNull(0) ? "" : reader.GetString(0), query_hash = reader.IsDBNull(1) ? "" : reader.GetString(1), total_cpu_ms = reader.IsDBNull(2) ? 0.0 : Convert.ToDouble(reader.GetValue(2)) / 1000.0, execution_count = reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3)), - max_dop = reader.IsDBNull(4) ? 0 : Convert.ToInt32(reader.GetValue(4)), + /* #3648: newest plan's reading, null when unknown — never 0. History fields follow. */ + max_dop = maxDop, spills = reader.IsDBNull(5) ? 0L : Convert.ToInt64(reader.GetValue(5)), - query_text = reader.IsDBNull(6) ? "" : reader.GetString(6) + query_text = reader.IsDBNull(6) ? "" : reader.GetString(6), + plan_count = planCount, + max_dop_any_plan = maxDopAnyPlan, + max_dop_any_plan_last_seen = maxDopAnyPlanLastSeen?.ToString("o"), + dop_note = QueryDopProvenance.Note(maxDop, maxDopAnyPlan, maxDopAnyPlanLastSeen, planCount) }); } @@ -573,7 +623,32 @@ Appended after the older columns so the existing reader ordinals are untouched. finding.DrillDown!["regressed_queries"] = items; } + /// + /// #3648: the same per-plan max_dop provenance as — see the essay + /// there. This read is one hash, unfiltered by CPU spent, so the newest-plan tie-break (compile time, + /// then CPU spent) is what separates a stale parallel plan still sitting in the cache from the serial + /// one doing the work at the same collection_time. Byte-identical to Lite's + /// CollectBadActorDetail text. + /// public const string BadActorDetailSql = @" +WITH windowed AS +( + -- #3648: see CollectTopCpuQueries' windowed CTE; same ranking, same hash-wide maximum. + SELECT database_name, query_hash, query_plan_hash, collection_time, max_dop, + delta_worker_time, delta_execution_count, delta_elapsed_time, delta_logical_reads, delta_spills, + query_text, + ROW_NUMBER() OVER + ( + PARTITION BY database_name, query_hash + ORDER BY collection_time DESC, creation_time DESC NULLS LAST, delta_worker_time DESC NULLS LAST + ) AS newest_rn, + MAX(max_dop) OVER (PARTITION BY database_name, query_hash) AS max_dop_any_plan + FROM v_query_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + AND query_hash = $4 +) SELECT database_name, query_hash, LEFT(MAX(query_text), 500) AS query_text, SUM(delta_execution_count)::BIGINT AS exec_count, @@ -589,12 +664,11 @@ THEN SUM(delta_logical_reads)::DOUBLE PRECISION / SUM(delta_execution_count) SUM(delta_worker_time)::BIGINT AS total_cpu_us, SUM(delta_logical_reads)::BIGINT AS total_reads, SUM(delta_spills)::BIGINT AS total_spills, - MAX(max_dop) AS max_dop -FROM v_query_stats -WHERE server_id = $1 -AND collection_time >= $2 -AND collection_time <= $3 -AND query_hash = $4 + MAX(CASE WHEN newest_rn = 1 THEN max_dop END) AS max_dop, + COUNT(DISTINCT query_plan_hash) AS plan_count, + MAX(max_dop_any_plan) AS max_dop_any_plan, + MAX(CASE WHEN max_dop = max_dop_any_plan THEN collection_time END) AS max_dop_any_plan_last_seen +FROM windowed GROUP BY database_name, query_hash"; private async Task CollectBadActorDetail(AnalysisFinding finding, AnalysisContext context) @@ -615,6 +689,10 @@ private async Task CollectBadActorDetail(AnalysisFinding finding, AnalysisContex using var reader = await cmd.ExecuteReaderAsync(context.CancellationToken); if (await reader.ReadAsync(context.CancellationToken)) { + var maxDop = reader.IsDBNull(10) ? (int?)null : Convert.ToInt32(reader.GetValue(10)); + var planCount = reader.IsDBNull(11) ? 0L : Convert.ToInt64(reader.GetValue(11)); + var maxDopAnyPlan = reader.IsDBNull(12) ? (int?)null : Convert.ToInt32(reader.GetValue(12)); + var maxDopAnyPlanLastSeen = reader.IsDBNull(13) ? (DateTime?)null : reader.GetDateTime(13); finding.DrillDown!["bad_actor_query"] = new { database = reader.IsDBNull(0) ? "" : reader.GetString(0), @@ -627,7 +705,12 @@ private async Task CollectBadActorDetail(AnalysisFinding finding, AnalysisContex total_cpu_ms = reader.IsDBNull(7) ? 0.0 : Convert.ToDouble(reader.GetValue(7)) / 1000.0, total_reads = reader.IsDBNull(8) ? 0L : Convert.ToInt64(reader.GetValue(8)), total_spills = reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9)), - max_dop = reader.IsDBNull(10) ? 0 : Convert.ToInt32(reader.GetValue(10)) + /* #3648: newest plan's reading, null when unknown — never 0. History fields follow. */ + max_dop = maxDop, + plan_count = planCount, + max_dop_any_plan = maxDopAnyPlan, + max_dop_any_plan_last_seen = maxDopAnyPlanLastSeen?.ToString("o"), + dop_note = QueryDopProvenance.Note(maxDop, maxDopAnyPlan, maxDopAnyPlanLastSeen, planCount) }; } } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Resources.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Resources.cs index 8b60fdd25..8c126431b 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Resources.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Resources.cs @@ -331,22 +331,30 @@ private async Task CollectCpuUtilizationFactsAsync(AnalysisContext context, List } } + /* #3527: delta_cntr_value spans one COLLECTION INTERVAL, not one second — at a 60s cadence the raw + delta is 60x the true rate. The honest rate divides by the row's MEASURED sample_interval_seconds + (#2234); interval <= 0 marks an unknowable delta (first sighting, counter reset, gap past the + policy), so those rows are filtered rather than emitted as 0 — rn = 1 lands on the newest row a + rate can honestly be derived from. */ public const string PerfmonSql = @" WITH latest AS ( - SELECT counter_name, cntr_value, delta_cntr_value, + SELECT counter_name, cntr_value, delta_cntr_value, sample_interval_seconds, ROW_NUMBER() OVER (PARTITION BY counter_name ORDER BY collection_time DESC) AS rn FROM perfmon_stats WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND counter_name IN ('Batch Requests/sec', 'SQL Compilations/sec', 'SQL Re-Compilations/sec') + AND sample_interval_seconds > 0 ) -SELECT counter_name, cntr_value, delta_cntr_value +SELECT counter_name, cntr_value, delta_cntr_value, sample_interval_seconds FROM latest WHERE rn = 1"; /// /// Collects key perfmon throughput counters: Batch Requests/sec, compilations, recompilations. /// Unscored context that distinguishes a busy server from a sick one (used by the AI surfaces). + /// Fact values are per-second rates: the per-interval delta divided by the row's measured + /// sample_interval_seconds (#3527); the raw delta and the divisor ride the metadata. /// private async Task CollectPerfmonFactsAsync(AnalysisContext context, List facts) { @@ -365,6 +373,7 @@ private async Task CollectPerfmonFactsAsync(AnalysisContext context, List var counterName = reader.GetString(0); var cntrValue = reader.IsDBNull(1) ? 0L : ToInt64(reader.GetValue(1)); var deltaValue = reader.IsDBNull(2) ? 0L : ToInt64(reader.GetValue(2)); + var intervalSeconds = reader.IsDBNull(3) ? 0L : ToInt64(reader.GetValue(3)); var (factKey, source) = counterName switch { @@ -376,8 +385,11 @@ private async Task CollectPerfmonFactsAsync(AnalysisContext context, List if (factKey == null) continue; - // All remaining counters are per-second rates — use the delta. - var value = (double)deltaValue; + /* The delta spans one collection interval — divide by the measured interval for the + per-second rate (#3527). The SQL already filters interval <= 0 (unknowable delta); + this guard keeps a raw or zero value from ever escaping if that filter regresses. */ + if (intervalSeconds <= 0) continue; + var value = deltaValue / (double)intervalSeconds; facts.Add(new Fact { @@ -388,7 +400,8 @@ private async Task CollectPerfmonFactsAsync(AnalysisContext context, List Metadata = new Dictionary { ["cntr_value"] = cntrValue, - ["delta_cntr_value"] = deltaValue + ["delta_cntr_value"] = deltaValue, + ["sample_interval_seconds"] = intervalSeconds } }); } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs index 3b1ace451..812e1ccc9 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.Waits.cs @@ -12,11 +12,150 @@ using System.Threading.Tasks; using Npgsql; using PerformanceMonitor.Analysis; +using PerformanceMonitor.Collectors; namespace PerformanceMonitor.Darling.Analysis; public sealed partial class PgFactCollector { + /// + /// The coverage witness (#3538 A2): how much of the window the collector actually observed, from the + /// wait-stats collection series — the same rows sums, so numerator and + /// denominator agree about when the collector was up. carries the full + /// argument for the three rules; the parameters are what make them concrete: + /// + /// $1..$3 server and window, as every windowed fact query binds + /// them. + /// $4 = window start minus the gap policy: the scan reaches back ONE policy + /// so the first in-window row can find its predecessor (its interval is then clipped to the window + /// by the GREATEST). A predecessor further back than that would have exceeded the policy + /// anyway, so nothing honest is lost by not looking further. + /// $5 = , bound + /// rather than inlined so this query and the calculator that produced the deltas can never disagree + /// about where "observed" ends. An interval past it is credited as 0 (its delta was discarded) and is + /// reported separately as a discarded stretch for the largest-gap figure. + /// + /// DISTINCT collection_time because a collection writes one row per wait type; the interval + /// belongs to the collection, not to each of its hundred rows. Shared dialect, byte-identical to + /// Lite's: EXTRACT(EPOCH FROM …), LAG, GREATEST and COALESCE all run + /// unchanged on DuckDB and Postgres (the anomaly detector's WaitRateWindowSql already leans on + /// the first two). The lead-in and tail gaps are finished in C# from first_sample / + /// last_sample / orphan_count, because they are about the WINDOW's edges, which the + /// row set cannot see. + /// + public const string CoverageSql = @" +WITH samples AS ( + SELECT DISTINCT collection_time + FROM v_wait_stats + WHERE server_id = $1 + AND collection_time >= $4 + AND collection_time <= $3 +), +intervals AS ( + SELECT collection_time, + LAG(collection_time) OVER (ORDER BY collection_time) AS previous_time + FROM samples +) +SELECT + COUNT(*) AS sample_count, + COALESCE(SUM(CASE + WHEN previous_time IS NULL THEN 0 + WHEN EXTRACT(EPOCH FROM (collection_time - previous_time)) > $5 THEN 0 + ELSE EXTRACT(EPOCH FROM (collection_time - GREATEST(previous_time, $2))) + END), 0) AS observed_seconds, + COALESCE(MAX(CASE + WHEN previous_time IS NOT NULL AND EXTRACT(EPOCH FROM (collection_time - previous_time)) > $5 + THEN EXTRACT(EPOCH FROM (collection_time - GREATEST(previous_time, $2))) + ELSE 0 + END), 0) AS largest_discarded_seconds, + COALESCE(SUM(CASE WHEN previous_time IS NULL THEN 1 ELSE 0 END), 0) AS orphan_count, + MIN(collection_time) AS first_sample, + MAX(collection_time) AS last_sample +FROM intervals +WHERE collection_time >= $2"; + + /// + /// Step 0 of every pass: stamps from + /// and, when the window was only partly observed, emits the + /// context fact beside the facts it qualifies (#3538 A2). + /// + /// It lives in the collector rather than in the analysis service so that EVERY path that + /// collects facts — the scheduled pass, get_analysis_facts, both windows of + /// compare_analysis, and a test that hands the collector a bare context — gets the stamp + /// before the first division, with no caller able to forget it. It runs BEFORE the wait read and, + /// like that read, carries no catch: this is the canary series, and a store that cannot answer it + /// should fail the pass loudly (the service's catch classifies and logs it) rather than degrade into + /// "unobserved", which would report a dead collector for a store that merely timed out. + /// + /// A zero-length or reversed window skips the read and stamps unobserved: there is no time to + /// divide by and nothing the series could say about it. + /// + private async Task CollectObservedCoverageAsync(AnalysisContext context, List facts) + { + var nominalMs = context.PeriodDurationMs; + if (nominalMs <= 0) + { + context.Coverage = WindowCoverage.Unobserved(nominalMs); + return; + } + + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); + + using var command = new NpgsqlCommand(CoverageSql, connection) { CommandTimeout = FactCommandTimeoutSeconds }; + command.Parameters.AddWithValue(context.ServerId); + command.Parameters.AddWithValue(AsNaive(context.TimeRangeStart)); + command.Parameters.AddWithValue(AsNaive(context.TimeRangeEnd)); + command.Parameters.AddWithValue(AsNaive(context.TimeRangeStart.AddSeconds(-CollectorDeltaCalculator.DefaultMaxGapSeconds))); + command.Parameters.AddWithValue(CollectorDeltaCalculator.DefaultMaxGapSeconds); + + using var reader = await command.ExecuteReaderAsync(context.CancellationToken); + if (!await reader.ReadAsync(context.CancellationToken)) + { + context.Coverage = WindowCoverage.Unobserved(nominalMs); + return; + } + + var sampleCount = reader.IsDBNull(0) ? 0L : ToInt64(reader.GetValue(0)); + var observedSeconds = reader.IsDBNull(1) ? 0.0 : Convert.ToDouble(reader.GetValue(1)); + var largestDiscardedSeconds = reader.IsDBNull(2) ? 0.0 : Convert.ToDouble(reader.GetValue(2)); + var orphanCount = reader.IsDBNull(3) ? 0L : ToInt64(reader.GetValue(3)); + DateTime? firstSample = reader.IsDBNull(4) ? null : reader.GetDateTime(4); + DateTime? lastSample = reader.IsDBNull(5) ? null : reader.GetDateTime(5); + + context.Coverage = BuildCoverage( + context, nominalMs, sampleCount, observedSeconds, largestDiscardedSeconds, orphanCount, firstSample, lastSample); + + if (context.Coverage.IsPartial) + facts.Add(context.Coverage.ToGapFact(context.ServerId)); + } + + /// + /// Finishes the coverage stamp from the query's row: the lead-in gap (window start to a first row + /// that had no predecessor — the row itself is the earliest thing we know about) and the tail gap + /// (last row to window end — a collector that died mid-window leaves nothing after itself to LAG + /// from) are edge properties of the window and are computed here; the in-series discarded stretch + /// comes from the query. Largest gap is the longest of the three. + /// + private static WindowCoverage BuildCoverage( + AnalysisContext context, double nominalMs, long sampleCount, double observedSeconds, + double largestDiscardedSeconds, long orphanCount, DateTime? firstSample, DateTime? lastSample) + { + if (sampleCount == 0 || firstSample is null || lastSample is null) + return WindowCoverage.Unobserved(nominalMs); + + var leadInMs = orphanCount > 0 ? Math.Max(0, (firstSample.Value - context.TimeRangeStart).TotalMilliseconds) : 0; + var tailMs = Math.Max(0, (context.TimeRangeEnd - lastSample.Value).TotalMilliseconds); + var largestGapMs = Math.Max(largestDiscardedSeconds * 1000.0, Math.Max(leadInMs, tailMs)); + + return new WindowCoverage + { + NominalMs = nominalMs, + ObservedMs = Math.Min(nominalMs, Math.Max(0, observedSeconds * 1000.0)), + SampleCount = (int)Math.Min(int.MaxValue, sampleCount), + LargestGapMs = Math.Min(nominalMs, largestGapMs) + }; + } + public const string WaitStatsSql = @" SELECT wait_type, @@ -33,10 +172,27 @@ GROUP BY wait_type /// /// Collects wait stats facts — one Fact per significant wait type. - /// Value is wait_time_ms / period_duration_ms (fraction of examined period). + /// Value is wait_time_ms / the OBSERVED collection time in the window + /// (), i.e. the fraction of the time the collector + /// was actually up that this wait type was being waited on — not the fraction of the nominal window + /// (#3538 A2). The metadata keeps period_duration_ms (the nominal window) and adds + /// coverage_fraction, so the divisor is recoverable as their product. + /// + /// Read the Value with its known scale-dependence in mind: delta_wait_time_ms sums the + /// wait time of every CONCURRENT task, so a fraction above 1.0 is legal (many tasks waiting at once) + /// and the same 25% means different things on 4 schedulers and on 64 — a handful of parallel + /// queries on the big box, most of the machine on the small one. The thresholds that score it do not + /// yet normalise for that; documented here rather than corrected, because the correction belongs + /// with the threshold re-derivation (#3538 A5), not with the denominator fix. + /// + /// An unobserved window (coverage 0) emits NO wait facts: there is no time to divide by, and + /// a fabricated fraction of 0 would read as a quiet server. The service turns that into the + /// "unavailable" envelope; this method's job is only to not lie. /// private async Task CollectWaitStatsFactsAsync(AnalysisContext context, List facts) { + if (context.ObservedDurationMs <= 0) return; + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var command = new NpgsqlCommand(WaitStatsSql, connection) { CommandTimeout = FactCommandTimeoutSeconds }; @@ -54,24 +210,27 @@ private async Task CollectWaitStatsFactsAsync(AnalysisContext context, List 0 ? (double)waitTimeMs / waitingTasks : 0; + var metadata = new Dictionary + { + ["wait_time_ms"] = waitTimeMs, + ["waiting_tasks_count"] = waitingTasks, + ["signal_wait_time_ms"] = signalWaitTimeMs, + ["resource_wait_time_ms"] = waitTimeMs - signalWaitTimeMs, + ["avg_ms_per_wait"] = avgMsPerWait, + ["period_duration_ms"] = context.PeriodDurationMs + }; + FactCollectorHelpers.AddCoverageFraction(metadata, context); + facts.Add(new Fact { Source = "waits", Key = waitType, Value = fractionOfPeriod, ServerId = context.ServerId, - Metadata = new Dictionary - { - ["wait_time_ms"] = waitTimeMs, - ["waiting_tasks_count"] = waitingTasks, - ["signal_wait_time_ms"] = signalWaitTimeMs, - ["resource_wait_time_ms"] = waitTimeMs - signalWaitTimeMs, - ["avg_ms_per_wait"] = avgMsPerWait, - ["period_duration_ms"] = context.PeriodDurationMs - } + Metadata = metadata }); } } @@ -91,10 +250,16 @@ FROM blocked_process_reports /// /// Collects blocking facts from blocked_process_reports. /// Produces a single BLOCKING_EVENTS fact with event count, rate, and details. - /// Value is events per hour for threshold comparison. + /// Value is events per OBSERVED hour — the hours the collector was actually up inside the window + /// (), not the nominal window (#3538 A2): forty + /// events in the one hour the collector saw of a four-hour window is a 40/hr storm, not a 10/hr + /// murmur. period_hours stays the nominal window; observed_hours is the divisor. An + /// unobserved window emits no fact. /// private async Task CollectBlockingFactsAsync(AnalysisContext context, List facts) { + if (context.ObservedDurationMs <= 0) return; + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var command = new NpgsqlCommand(BlockingSql, connection) { CommandTimeout = FactCommandTimeoutSeconds }; @@ -114,7 +279,8 @@ private async Task CollectBlockingFactsAsync(AnalysisContext context, List var sleepingBlockerCount = reader.IsDBNull(4) ? 0L : ToInt64(reader.GetValue(4)); var periodHours = context.PeriodDurationMs / 3_600_000.0; - var eventsPerHour = periodHours > 0 ? eventCount / periodHours : 0; + var observedHours = context.ObservedDurationMs / 3_600_000.0; + var eventsPerHour = eventCount / observedHours; facts.Add(new Fact { @@ -130,7 +296,8 @@ private async Task CollectBlockingFactsAsync(AnalysisContext context, List ["max_wait_time_ms"] = maxWaitTimeMs, ["distinct_head_blockers"] = distinctHeadBlockers, ["sleeping_blocker_count"] = sleepingBlockerCount, - ["period_hours"] = periodHours + ["period_hours"] = periodHours, + ["observed_hours"] = observedHours } }); } @@ -145,10 +312,14 @@ FROM deadlocks /// /// Collects deadlock facts from the deadlocks table. /// Produces a single DEADLOCKS fact with count and rate. - /// Value is deadlocks per hour for threshold comparison. + /// Value is deadlocks per OBSERVED hour (see — same divisor, + /// same reason, #3538 A2). period_hours nominal, observed_hours the divisor; an + /// unobserved window emits no fact. /// private async Task CollectDeadlockFactsAsync(AnalysisContext context, List facts) { + if (context.ObservedDurationMs <= 0) return; + await using var connection = await _postgres.OpenConnectionAsync(context.CancellationToken); using var command = new NpgsqlCommand(DeadlocksSql, connection) { CommandTimeout = FactCommandTimeoutSeconds }; @@ -163,7 +334,8 @@ private async Task CollectDeadlockFactsAsync(AnalysisContext context, List if (deadlockCount <= 0) return; var periodHours = context.PeriodDurationMs / 3_600_000.0; - var deadlocksPerHour = periodHours > 0 ? deadlockCount / periodHours : 0; + var observedHours = context.ObservedDurationMs / 3_600_000.0; + var deadlocksPerHour = deadlockCount / observedHours; facts.Add(new Fact { @@ -175,7 +347,8 @@ private async Task CollectDeadlockFactsAsync(AnalysisContext context, List { ["deadlock_count"] = deadlockCount, ["deadlocks_per_hour"] = deadlocksPerHour, - ["period_hours"] = periodHours + ["period_hours"] = periodHours, + ["observed_hours"] = observedHours } }); } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.cs index bddf20660..417a749ed 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.cs @@ -19,7 +19,8 @@ namespace PerformanceMonitor.Darling.Analysis; /// /// Collects facts from Darling's Postgres store for the analysis engine — Lite's /// DuckDbFactCollector ported method-for-method (Phase-5 analysis slice AN2a): the same -/// 28 collect methods across the same seven partial files, the same fact keys / values / +/// collect methods across the same seven partial files (thirty-one fact readers plus the #3538 +/// coverage witness, censused by name in PgFactCollectorTests), the same fact keys / values / /// metadata shapes, the same emission order, and the same per-method degrade-to-no-facts error /// posture — a missing table or empty store yields "no facts", never an exception. Since #2826 /// that degradation is REPORTED rather than silent on both sides (see @@ -170,6 +171,10 @@ public async Task> CollectFactsAsync(AnalysisContext context) { var facts = new List(); + /* #3538 A2: the coverage stamp comes FIRST, because every rate and fraction fact below divides by + it. Nothing else may run ahead of it — a wait fact emitted before the stamp would have no + denominator, and the only "safe" fallback (the nominal window) is the defect being fixed. */ + await CollectObservedCoverageAsync(context, facts); await CollectWaitStatsFactsAsync(context, facts); FactCollectorHelpers.GroupGeneralLockWaits(facts, context); FactCollectorHelpers.GroupParallelismWaits(facts, context); @@ -215,6 +220,7 @@ public async Task> CollectFactsAsync(AnalysisContext context) /// public static IReadOnlyList AllSql { get; } = new[] { + CoverageSql, WaitStatsSql, BlockingSql, BlockingChainSql, diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs index f0039291e..3ee6cb5a3 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFindingStore.cs @@ -185,6 +185,18 @@ INSERT INTO analysis_muted (mute_id, server_id, story_path_hash, story_path, mut public const string UnmuteStorySql = "DELETE FROM analysis_muted WHERE mute_id = $1"; + /* #3541 A14: how many STORED findings a mute's story_path_hash matches right now, so the mute verb can + report what it matched rather than only that it wrote. Two statements rather than one with a nullable + parameter, so the all-servers form is a plain equality on the hash index (idx_analysis_findings_hash) + and the scoped form is that plus server_id, both sub-second by shape on the retained population. */ + public const string CountFindingsWithHashSql = @" +SELECT count(*) FROM analysis_findings +WHERE story_path_hash = $1"; + + public const string CountFindingsWithHashForServerSql = @" +SELECT count(*) FROM analysis_findings +WHERE story_path_hash = $1 AND server_id = $2"; + public const string CleanupOldFindingsSql = "DELETE FROM analysis_findings WHERE analysis_time < $1"; /// @@ -477,14 +489,20 @@ public async Task> GetLatestFindingsAsync(int serverId) } /// - /// Mutes a story pattern so it won't appear in future analysis runs. + /// Mutes a story pattern so it won't appear in future analysis runs. Returns true when the registry + /// row was written and false when the INSERT failed (logged, as every read-back surface here logs). + /// + /// The return value is #3541 A14: this method swallowed its failure and returned Task, so the + /// MCP mute verb above it reported status: "muted" whether or not a row exists — a write that + /// reported what it intended, not what happened. The swallow stays (the viewer's mute button has no better + /// answer than a logged line), but the caller now learns which of the two things occurred. /// /// #2443 exempt: off the analysis pass. This surface serves the viewer, the MCP and the /// retention sweep — lifetimes with no per-pass budget and no wedged analysis to abandon — so /// its store calls take no pass token. Threading one here would mean inventing a caller that /// does not exist. /// - public async Task MuteStoryAsync(int serverId, string storyPathHash, string storyPath, string? reason = null) + public async Task MuteStoryAsync(int serverId, string storyPathHash, string storyPath, string? reason = null) { try { @@ -500,13 +518,49 @@ public async Task MuteStoryAsync(int serverId, string storyPathHash, string stor command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = (object?)reason ?? DBNull.Value }); await command.ExecuteNonQueryAsync(); + return true; } catch (Exception ex) { _logger?.LogError("[PgFindingStore] MuteStoryAsync failed: {Message}", ex.Message); + return false; } } + /// + /// How many STORED findings currently carry — for one server, or across + /// the fleet when is null or the all-servers sentinel 0 — so a mute can say what + /// it matched at the moment it was registered (#3541 A14). + /// + /// This is a disclosure, not a gate. The mute registry is a PATTERN registry: nothing in it + /// references a finding row, and consults it by hash on every future + /// pass. A hash that matches nothing today is therefore a legitimate registration (the pattern may return + /// after the retention sweep has purged its history) AND the most likely shape of a mistyped hash, which is + /// why the count is reported beside the write rather than used to refuse it. Throws on a store failure — + /// the MCP caller owns the error envelope; a count that silently read as 0 would be the very + /// zero-vs-unknown confusion the payload contract forbids. + /// + public async Task CountStoredFindingsAsync(int? serverId, string storyPathHash, CancellationToken cancellationToken = default) + { + /* Scoped unless null or the all-servers sentinel 0 — and ONLY those two: a server_id is an FNV hash cast + to int, so roughly half of all real ids are negative and a `> 0` test here would silently count half the + fleet's scoped mutes fleet-wide. */ + var scoped = serverId is not (null or 0); + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + using var command = new NpgsqlCommand(scoped ? CountFindingsWithHashForServerSql : CountFindingsWithHashSql, connection) + { + CommandTimeout = DarlingAnalysisService.AnalysisCommandTimeoutSeconds, + }; + command.Parameters.AddWithValue(storyPathHash); + if (scoped) + { + command.Parameters.AddWithValue(serverId!.Value); + } + + var result = await command.ExecuteScalarAsync(cancellationToken); + return result is long count ? count : Convert.ToInt64(result, System.Globalization.CultureInfo.InvariantCulture); + } + /// /// Unmutes a story pattern (Dashboard-twin surface). /// diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs index af0dddb03..c01fff4d5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertDeliverer.cs @@ -74,7 +74,24 @@ public DarlingAlertDeliverer( _core = new EmailSendCore(settings, historyStore, webhookAlertService, s_branding, logger); } - public async Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) + public Task DeliverAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) => + DeliverAndReportAsync(outcome, cancellationToken); + + /// + /// The delivery, reporting its disposition (#3580). Every send goes through here — + /// is this with the answer discarded — so there is one delivery path and + /// not a reporting one beside a silent one. + /// + /// What comes back. On the combined send (Summary mode, or any alert without incidents, + /// which is every self-alert) the exact the history row was written with. + /// On a Per-event split there are N sends and N rows and no single disposition describes them, so this + /// returns null — "unreported" — rather than electing one; the two callers that read the + /// answer (the digest and the rollup) fire with Context: null and never take that path. The + /// belt-and-suspenders catch below also answers null: both TrySendAsync and + /// RecordAlertAsync are failure-isolated themselves, so a throw here is something outside the + /// channels and says nothing about whether they delivered. + /// + public async Task DeliverAndReportAsync(AlertOutcome outcome, CancellationToken cancellationToken = default) { if (outcome is null) { @@ -102,11 +119,11 @@ await SendAndRecordAsync( deliveryMode: mode); } - return; + return null; } /* Summary mode, or an alert with no incidents (CPU/low-disk/jobs): one combined send+row, unchanged. */ - await SendAndRecordAsync( + return await SendAndRecordAsync( outcome, outcome.CurrentValue, outcome.Context, outcome.DetailText, outcome.NumericCurrentValue, outcome.NumericThresholdValue, deliveryMode: mode); @@ -119,6 +136,7 @@ await SendAndRecordAsync( { _logger.LogError("Alert delivery failed for {Metric} on {Server}: {Message}", outcome.MetricName, outcome.ServerName, ex.Message); + return null; } } @@ -130,12 +148,14 @@ await SendAndRecordAsync( /// still record (flagged muted). /// /// - /// The mode resolved for this server, forwarded to the shared send core for - /// #3430's per-metric repeat ceiling. Passed rather than re-resolved so one alert's two channels and its - /// history row all describe the same decision, and passed FAITHFULLY on the Per-event split — those - /// messages must not be aggregated, which is that mode's own contract. + /// The mode resolved for this server, forwarded to the shared send + /// core for #3430's per-metric repeat ceiling. Passed rather than re-resolved so one alert's two channels + /// and its history row all describe the same decision, and passed FAITHFULLY on the Per-event split — + /// those messages must not be aggregated, which is that mode's own contract. /// - private async Task SendAndRecordAsync( + /// The disposition the history row was written with — the same value, so what the caller is + /// told and what the operator later reads in the alert log cannot disagree (#3580). + private async Task SendAndRecordAsync( AlertOutcome outcome, string currentValue, AlertContext? context, string? detailText, double? numericCurrentValue, double? numericThresholdValue, AlertNotificationMode deliveryMode) { @@ -143,8 +163,10 @@ private async Task SendAndRecordAsync( only Context.SeverityOverride — so every self-alert (fired with Context: null) rendered INFO-blue in Teams/Slack/PagerDuty/webhooks while its log line said Critical. Fold the outcome's severity into the context here, once, upstream of every channel; ??= so an - explicit override set by a context builder still wins. The context also serializes into - alert history, so replays keep the severity too. */ + explicit override set by a context builder still wins. The context serializes into alert + history below, and since #3539 A8e the serializer carries this property as the row's Severity + member — so the history grids and get_alert_history read the tier the alert fired at (before + that the projection dropped it, and this comment's "replays keep the severity" was not true). */ if (outcome.Severity is not null) { context ??= new AlertContext(); @@ -175,5 +197,7 @@ await _historyStore.RecordAlertAsync(new AlertHistoryRecord( numericCurrentValue, numericThresholdValue, delivery, outcome.Muted, detailText, contextJson)); + + return delivery; } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs index 3cea4b118..cead42bf8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs @@ -62,10 +62,21 @@ public sealed class DarlingAlertReadAdapter : IAlertReadAdapter /// one read — the forced-plan check at 1,744.9 ms cold, scanning ~6.0 GB of /// query_store_stats; every other read in the family lands under 3 ms. Ten seconds is 5.7x /// that worst case, so it absorbs a substantial stall rather than only the happy path. That margin - /// has been measured being eaten once: the collection-signals read's whole-history top-N sort grew - /// with the 90-day retention fill until its cold excursions clocked ~12 s against this deadline — - /// the first measured breach — and #3496 restored the margin by making that read chunk-orderable - /// rather than by moving this number. + /// has been measured being eaten twice, and both times restored by fixing the read rather than by + /// moving this number. First the collection-signals read's whole-history top-N sort grew with the + /// 90-day retention fill until its cold excursions clocked ~12 s — the first measured breach — and + /// #3496 made that read chunk-orderable. Then the 1,744.9 ms read itself: the ~6.0 GB it was measured + /// over became 23 GB, and its plan turned out to have been paying a fleet-width tax the whole time — + /// the planner walked the entire fleet's two-hour slice through the time index and filtered 95% of it + /// away per server (Rows Removed by Filter: 691,058, 57,307 buffers) rather than take the + /// (server_id, collection_time) composite that was already there — so the cold tail crossed + /// this deadline at 10.3 s while the fixed #3496 site sat at zero (#3573). The covering index in + /// PgTableTuning made that read an Index Only Scan over one server's rows (50 buffers against + /// 1,514 for the same statement on the rig; see ). The 1,744.9 ms + /// figure therefore stands as the measured floor this number was derived from and as the recorded + /// cost of the access path that has since been replaced, not as a current cost — and 10 s stays: the + /// cadence bound below has not moved, and a deadline re-fitted to a read that now costs milliseconds + /// would only mean the next drift is caught later. /// /// Bounded above by the cadence this pass runs on: s_alertSweepInterval is /// 30 s, so one stalled read must still leave the pass able to finish inside the interval that @@ -335,56 +346,88 @@ public async Task> GetRecentDeadlocksAsync( /* ---------------- poison waits ---------------- */ /// - /// Lite's poison-wait read verbatim (wait_stats table instead of the v_ view). $2 is the - /// parameterized naive-UTC "now minus 10 minutes" — the parameterization style the - /// long-running-query twin copies. + /// Accumulated poison wait per wait type over the alert's window (#3539 A4) — the SQL Server twin of + /// , over wait_stats. $1 server_id, + /// $2 the parameterized naive-UTC window floor (now minus ; + /// the parameterization style the long-running-query twin copies — a bare now() is timestamptz + /// and would compare in the server's time zone against these naive-UTC columns). + /// + /// What changed from the retired read and why. The old text selected the newest three rows + /// with delta_waiting_tasks > 0 and let the engine judge each row's avg-ms-per-wait; this one + /// SUMs every row in the window per wait type and applies no threshold. Three consequences, each + /// deliberate: there is no LIMIT, because a limit on a sum is an undercount; the + /// delta_waiting_tasks > 0 filter is gone, because the measured fleet holds THREADPOOL rows with + /// hundreds of ms of wait and ZERO completed tasks (a task still waiting across the interval boundary) + /// and that wait is evidence — the old filter dropped it; and every wait type with any row in the window + /// comes back whatever its sum, because the engine needs "observed and quiet" as an answer distinct + /// from "not observed" (an empty result holds a standing alert open; a sub-bar row clears it). + /// + /// (0, 0) rows. The delta calculator writes delta_wait_time_ms = 0, delta_waiting_tasks + /// = 0 both for a genuinely idle interval and for "no delta is knowable here" (first sighting, counter + /// reset, a gap past the policy) — the two are indistinguishable in this table today. A SUM treats them + /// identically and correctly: zero contributes nothing to the accumulated wait, and the evaluator's + /// denominator is the window's wall clock, not a row count, so a fabricated zero neither inflates nor + /// deflates the figure. It does count toward observed_intervals, which is honest for the one + /// question that column answers — did the collector deliver a row — and is why that column is not a + /// "known quiet" claim. Deltas are never negative (the calculator returns 0 on a reset), so no floor is + /// applied; a defensive one would only hide a calculator regression. + /// + /// Cost, under the alert pass's 10-second read deadline while the hourly CAGG refresh runs + /// (#3597: 4–7 minutes on the largest store). Cheap by SHAPE, not by any index the planner may or may not + /// pick: the WHERE is predicate-identical to the retired read's — one server_id, the three wait_type + /// literals, a collection_time floor ten minutes back — so it touches exactly the rows that read touched + /// and aggregates them instead of ordering them for a LIMIT 3. At the wait_stats collector's + /// one-minute cadence that is at most ~10 collections × 3 types ≈ 30 rows per server, all inside the + /// last ten minutes of the current one-day chunk (uncompressed head; chunk exclusion keeps compressed + /// history out of the plan). The PostgreSQL twin () + /// has run this exact aggregate shape over pg_wait_stats under the same deadline since #2711. + /// + /// The V128 keystone lane is adding sample_interval_seconds to this table and may later + /// guard readers on it; this text deliberately carries no interval handling so that lane can rebase + /// onto it cleanly. /// public const string PoisonWaitsSql = @" SELECT wait_type, - delta_wait_time_ms AS delta_ms, - delta_waiting_tasks AS delta_tasks, - CASE WHEN delta_waiting_tasks > 0 - THEN CAST(delta_wait_time_ms AS DOUBLE PRECISION) / delta_waiting_tasks - ELSE 0 END AS avg_ms_per_wait, - collection_time + SUM(delta_wait_time_ms)::bigint AS accumulated_wait_ms, + SUM(delta_waiting_tasks)::bigint AS accumulated_waits, + COUNT(*)::bigint AS observed_intervals, + MAX(collection_time) AS newest_collection_time FROM wait_stats WHERE server_id = $1 AND wait_type IN ('THREADPOOL', 'RESOURCE_SEMAPHORE', 'RESOURCE_SEMAPHORE_QUERY_COMPILE') -AND delta_waiting_tasks > 0 AND collection_time >= $2 -ORDER BY collection_time DESC -LIMIT 3"; +GROUP BY wait_type +ORDER BY accumulated_wait_ms DESC"; - public async Task> GetPoisonWaitDeltasAsync( - string serverKey, double thresholdMs, CancellationToken cancellationToken = default) + public async Task> GetPoisonWaitAccumulationAsync( + string serverKey, int windowMinutes, CancellationToken cancellationToken = default) { var serverId = ParseServerKey(serverKey); - var items = new List(); + var items = new List(); await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); using var command = new NpgsqlCommand(PoisonWaitsSql, connection) { CommandTimeout = AlertPassCommandTimeoutSeconds }; command.Parameters.AddWithValue(serverId); - command.Parameters.AddWithValue(NaiveUtcNow().AddMinutes(-10)); + /* The engine's window, not a local recency constant: the window IS the denominator the bars + normalize against, so read and evaluation must agree on it or the "average tasks stuck" + arithmetic silently means something else (the PostgreSQL twin's reasoning, verbatim). */ + command.Parameters.AddWithValue(NaiveUtcNow().AddMinutes(-windowMinutes)); using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) { - items.Add(new PoisonWaitDelta - { - WaitType = reader.GetString(0), - DeltaMs = reader.IsDBNull(1) ? 0 : reader.GetInt64(1), - DeltaTasks = reader.IsDBNull(2) ? 0 : reader.GetInt64(2), - AvgMsPerWait = reader.IsDBNull(3) ? 0 : reader.GetDouble(3), - CollectionTime = reader.GetDateTime(4) - }); + items.Add(new PoisonWaitAccumulation( + reader.GetString(0), + reader.IsDBNull(1) ? 0 : reader.GetInt64(1), + reader.IsDBNull(2) ? 0 : reader.GetInt64(2), + reader.IsDBNull(3) ? 0 : reader.GetInt64(3), + reader.IsDBNull(4) ? DateTime.MinValue : reader.GetDateTime(4))); } } - /* Fetch-then-filter, exactly like Lite's loop: the 3-row window is selected before - thresholding (see the IAlertReadAdapter contract). */ - return items.FindAll(w => w.AvgMsPerWait >= thresholdMs); + return items; } /* ---------------- long-running queries ---------------- */ @@ -514,6 +557,15 @@ public async Task> GetLongRunningQueriesAsync( /// chunks rather than scanning retention. The reported window width is measured rather than assumed, so a /// gap in collection cannot make a slow rise look fast. /// + /// The newest sample's collection_time travels with the row (#3636) as observed_at, + /// the last column. The growth is a fact about two COLLECTIONS and reads byte-identical on every alert pass + /// until the next collection lands — and this collector's cadence is an HOUR against a 5-minute cooldown, + /// so the engine needs the observation's identity to fire the rise gate once per collection instead of up to + /// twelve times. current_files already carried collection_time for the window-width arithmetic; + /// it was simply never projected. Appended rather than inserted so the fourteen ordinals the reader already + /// binds do not move. #3579's observed_at on the forced-plan read is the same column for the same + /// reason. + /// /// $1 server_id, $2 window start (naive UTC). /// public const string DatabaseFileGrowthSql = @" @@ -549,7 +601,8 @@ FROM database_size_stats c.auto_growth_mb, COALESCE(c.is_percent_growth, false) AS is_percent_growth, c.growth_pct, - c.max_size_mb + c.max_size_mb, + c.collection_time AS observed_at FROM current_files c LEFT JOIN baseline b ON b.database_name = c.database_name @@ -589,6 +642,11 @@ public async Task> GetDatabaseFileGrowthAsync( IsPercentGrowth = !reader.IsDBNull(11) && reader.GetBoolean(11), GrowthPct = reader.IsDBNull(12) ? null : Convert.ToDouble(reader.GetValue(12)), MaxSizeMb = reader.IsDBNull(13) ? null : Convert.ToDouble(reader.GetValue(13)), + /* #3636: the stored value is naive UTC (the $2 window bound above is built the same way), read + back with Kind Unspecified; stamped Utc because that is what it IS and what the property's name + says. The engine only ever compares one file's stamps with each other, so the Kind is honesty + rather than arithmetic — #3579's forced-plan read does exactly this. */ + ObservedAtUtc = reader.IsDBNull(14) ? null : DateTime.SpecifyKind(reader.GetDateTime(14), DateTimeKind.Utc), }); } @@ -1092,6 +1150,29 @@ one carried over from a restart (#2166). A database cleared here is one that is /// /// The > comparison is what makes this a delta read: equal counters are silence, and a /// LOWER counter (unforce/re-force reset) is silence too rather than a negative delta. + /// + /// The newer sighting's collection_time travels with the row (#3579) as + /// observed_at, the last column. The delta is a fact about two COLLECTIONS and stays byte-identical + /// on every alert pass until the next collection lands — "every row here is a live failure" is true at the + /// collection instant and stale for the rest of the interval — so the engine needs the observation's + /// identity to fire once per collection instead of once per cooldown. Appended rather than inserted so + /// the seven ordinals the reader already binds do not move. It is n.collection_time, already in + /// per_collection's GROUP BY and already carried by the covering index: no new qs. column, + /// so the access path below is untouched (the access-path pins re-derive the list from this text). + /// + /// The access path is a covering index, and the column list here is what it covers (#3573). + /// PgTableTuning.ForcePlanFailuresIndexName is (server_id, collection_time DESC) INCLUDE + /// every other column this statement touches, so it runs as an Index Only Scan over one server's two + /// hours. That is not a nicety. V1's plain (server_id, collection_time) composite was on the + /// production store the whole time and the planner refused it: server_id's physical correlation is + /// ~0.02 (forty-three servers interleaved by collection pass), so the cost model priced the composite's + /// heap fetches as one random page per row and preferred streaming the ENTIRE fleet's two-hour slice through + /// the time index — Rows Removed by Filter: 691,058 to keep 37,878, 57,307 buffers, 422 ms warm and + /// a 10.3 s cold tail against the 10 s deadline. Covering deletes the heap term the model got wrong. The + /// consequence for anyone editing this SQL: reference a column of qs that the index does not carry + /// and nothing fails — the plan silently reverts to the fleet-wide scan. ForcePlanFailuresAccessPathTests + /// pins the two lists against each other and EXPLAINs the shipped statement against a live store; add the + /// column to the index in the same change or that test tells you. /// public const string ForcePlanFailuresSql = @" WITH per_collection AS ( @@ -1122,7 +1203,8 @@ FROM per_collection AS pc n.forcing_type, n.reason, n.failures - p.failures AS failure_delta, - n.failures AS total_failures + n.failures AS total_failures, + n.collection_time AS observed_at FROM ranked AS n JOIN ranked AS p ON p.database_name = n.database_name @@ -1156,7 +1238,12 @@ public async Task> GetForcePlanFailuresAsync( ForcingType = reader.IsDBNull(3) ? "" : reader.GetString(3), FailureReason = reader.IsDBNull(4) ? "" : reader.GetString(4), FailureDelta = reader.IsDBNull(5) ? 0 : reader.GetInt64(5), - TotalFailures = reader.IsDBNull(6) ? 0 : reader.GetInt64(6) + TotalFailures = reader.IsDBNull(6) ? 0 : reader.GetInt64(6), + /* #3579: the stored value is naive UTC (see the window-bound remarks above), read back with + Kind Unspecified; stamped Utc because that is what it IS and what the property's name says. + The engine only ever compares one plan's stamps with each other, so the Kind is honesty + rather than arithmetic. */ + ObservedAtUtc = reader.IsDBNull(7) ? null : DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc) }); } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs index a422342a5..ada1a5a06 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertSettings.cs @@ -56,12 +56,27 @@ public DarlingAlertSettings(DarlingConfig config) public bool ForcePlanFailureEnabled => true; public int CpuThresholdPercent => _config.Alerts.CpuThresholdPercent; - public int BlockingCountThreshold => _config.Alerts.BlockingCountThreshold; + + /// #3528: floored at the same named constant as its PostgreSQL twin below, for the identical + /// reason — a store row hand-edited to 0 makes the gate's count >= threshold test true for a + /// count of zero and fires on a server with no blocking. The floor is also the + /// update_alert_settings lower write bound and the Settings window's save gate, so no value any + /// writer accepts is a value this clamp then rewrites. + public int BlockingCountThreshold => Math.Max( + PostgresAlertEvaluator.CountThresholdFloor, + _config.Alerts.BlockingCountThreshold); /* #1839: floored at 0 (= off) so a negative in darling.json or the store can't make the "is it above threshold" test true for every snapshot. */ public int BlockingWaitSecondsThreshold => Math.Max(0, _config.Alerts.BlockingWaitSecondsThreshold); - public int DeadlockCountThreshold => _config.Alerts.DeadlockCountThreshold; + + /// #3528: floored like above and + /// below — the constant lives on + /// only by birthplace; the failure it closes is engine-neutral. + public int DeadlockCountThreshold => Math.Max( + PostgresAlertEvaluator.CountThresholdFloor, + _config.Alerts.DeadlockCountThreshold); + /* #3539 A4: pass-through of a knob the engine no longer reads — see the interface member's doc. */ public int PoisonWaitThresholdMs => _config.Alerts.PoisonWaitThresholdMs; public int LongRunningQueryThresholdMinutes => _config.Alerts.LongRunningQueryThresholdMinutes; public int TempDbSpaceThresholdPercent => _config.Alerts.TempDbSpaceThresholdPercent; @@ -75,6 +90,12 @@ public DarlingAlertSettings(DarlingConfig config) public int DiskCriticalFreePercent => Math.Clamp(_config.Alerts.DiskCriticalFreePercent, 0, 100); public int DiskCriticalFreeGb => Math.Max(0, _config.Alerts.DiskCriticalFreeGb); public int SelfDiskFreeWarnPercent => Math.Clamp(_config.Alerts.SelfDiskFreeWarnPercent, 0, 100); + + /// #3528: the store warning's GB floor — Store Disk Pressure fires only when the percent above + /// is breached AND free space is below this many GB, so a large volume at a low percent (400 GB free on + /// a 4 TB store) stops paging CRITICAL. Zero removes the floor (the percent-only pre-#3528 condition); + /// the 0-floor GB shape is 's, whose AND-qualifier composition this mirrors. + public int SelfDiskFreeWarnGb => Math.Max(0, _config.Alerts.SelfDiskFreeWarnGb); public int CollectionStaleMinutes => Math.Clamp(_config.Alerts.CollectionStaleMinutes, 5, 1440); /// #2136: the Store Job Over Cadence warning percent. Clamped [5, 100]. @@ -122,16 +143,13 @@ public DarlingAlertSettings(DarlingConfig config) TimescaleSupport.RetentionHoldRatioFloor, TimescaleSupport.RetentionHoldRatioCeiling); - /// #3444 (V122): the PostgreSQL Deadlocks alert's rolling-window count threshold. - /// - /// Floored, where its SQL Server twin is not. passes - /// _config.Alerts through raw, so a store row hand-edited to 0 makes the gate's - /// count >= threshold test true for a count of zero and fires on a server with no deadlocks. - /// That is a pre-existing gap on the twin rather than a shape to copy: the floor here matches - /// two screens up, which floors for the identical reason. - /// The floor is also the update_alert_settings lower write bound, so no value the write path - /// accepts is a value this clamp then rewrites — the "setting did not stick" failure the write-bound - /// parity pins exist for. + /// #3444 (V122): the PostgreSQL Deadlocks alert's rolling-window count threshold, floored so a + /// store row hand-edited to 0 cannot make the gate's count >= threshold test true for a count + /// of zero and fire on a server with no deadlocks. Its SQL Server twin + /// () floors at the same constant since #3528, so the two engines' + /// gates are now the one shape. The floor is also the update_alert_settings lower write bound, + /// so no value the write path accepts is a value this clamp then rewrites — the "setting did not stick" + /// failure the write-bound parity pins exist for. public int PgDeadlockCountThreshold => Math.Max( PostgresAlertEvaluator.CountThresholdFloor, _config.Alerts.PgDeadlockCountThreshold); @@ -165,7 +183,8 @@ the percent it has no meaningful upper bound. */ /* #2349: the file-growth gates. Clamped the same way the neighbours are -- a negative threshold would make the comparison always true, which for a gate whose whole job is to be quiet until something moves is the worst possible default. A ZERO is meaningful here rather than nonsense: it disables that one - gate, so an operator can run rise-only or level-only without a second switch. */ + gate, so an operator can run rise-only or level-only without a second switch. The rise is MB per HOUR + averaged over the lookback (#3539 A8c); the clamp does not care about the unit, the builder does. */ public bool FileGrowthEnabled => _config.Alerts.FileGrowthEnabled; public int FileGrowthRiseMb => Math.Max(0, _config.Alerts.FileGrowthRiseMb); public int FileGrowthVolumePercent => Math.Clamp(_config.Alerts.FileGrowthVolumePercent, 0, 100); diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs index 60613fcd3..2ae956b4b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingConfig.cs @@ -569,6 +569,9 @@ public sealed class AlertsConfig [JsonPropertyName("poisonWaitEnabled")] public bool PoisonWaitEnabled { get; set; } = true; + /// Read and reported but no longer consulted by the alert engine since #3539 A4 — see + /// . Kept so an existing darling.json and the + /// store's control-plane row keep round-tripping. [JsonPropertyName("poisonWaitThresholdMs")] public int PoisonWaitThresholdMs { get; set; } = 500; @@ -610,15 +613,26 @@ public sealed class AlertsConfig /* #2349: the database file-growth alert. Ships OFF -- a new alert that starts firing on upgrade is a bad citizen, and the right thresholds are a property of the fleet rather than of the product. */ public bool FileGrowthEnabled { get; set; } + /* #3539 A8c: MB per HOUR, averaged over FileGrowthLookbackMinutes. Same column (file_growth_rise_mb), same + integer, one meaning on every lookback -- the engine scales it to the window at comparison time. */ public int FileGrowthRiseMb { get; set; } = 10240; public int FileGrowthVolumePercent { get; set; } = 60; + /* The window the rise RATE is averaged over (#3539 A8c) -- it does not rescale the threshold. */ public int FileGrowthLookbackMinutes { get; set; } = 60; /// #2107: the store volume's self-alert warning percent (was a compile-time 10.0; - /// 0 disables the check — percent is its only trigger). + /// 0 disables the check). [JsonPropertyName("selfDiskFreeWarnPercent")] public int SelfDiskFreeWarnPercent { get; set; } = 10; + /// #3528: the store warning's GB floor — the percent above additionally requires free space + /// below this many GB, an AND qualifier so a large volume at a low percent never pages (0 removes the + /// floor). The PVS-floor composition, not the target pair's OR. 50 puts the crossover at a 500 GB + /// store volume: below that the percent governs exactly as before; above it, 50 GB free is the line — + /// which is what stops 400 GB free on a 4 TB volume reading as "act now". + [JsonPropertyName("selfDiskFreeWarnGb")] + public int SelfDiskFreeWarnGb { get; set; } = 50; + /// #2107: how long collection may go quiet before Collection Stopped / Agent Not /// Running fire (was a compile-time 30 minutes). Defaults to the shared constant behind the /// display's Offline band, so the two definitions of "collection stopped" agree out of the @@ -685,10 +699,12 @@ public sealed class AlertsConfig /// retention and deadlock-band knobs above already follow. DarlingAlertSettings clamps it on /// read. /// - /// Separate from deliberately — see the V122 rung and - /// the default constant for why the SQL Server figure's justification does not travel to an engine with - /// no deadlock band. The enabled switch IS shared: governs both - /// engines. + /// Separate from deliberately — see the default + /// constant: the two engines count with different instruments (captured graphs vs deadlocks parsed + /// from the server log), and an operator tuning one should not silently move the other. The V122 rung's + /// second reason — that a PostgreSQL server had no deadlock band to agree with — ended with #3539, which + /// bands the PostgreSQL card's own counter difference through the shared tiers. The enabled + /// switch IS shared: governs both engines. [JsonPropertyName("pgDeadlockCountThreshold")] public int PgDeadlockCountThreshold { get; set; } = PostgresAlertEvaluator.DeadlockCountThresholdDefault; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingDeltaCalculator.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingDeltaCalculator.cs index 7f267c3ab..11e1fc888 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingDeltaCalculator.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingDeltaCalculator.cs @@ -7,6 +7,7 @@ */ using System; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -21,21 +22,60 @@ namespace PerformanceMonitor.Darling.Service; /// seeding that lets the first collection after a service restart produce accurate deltas instead /// of returning 0 for everything — the twin of Lite's DuckDB-seeding DeltaCalculator, with the /// seed queries, delta-group names, and key formats mirrored verbatim so the two hosts can never -/// restore different baselines. +/// restore different baselines. Seeds EVERY delta family the service monitors, keys and pass +/// window both (#3540 A4), so the restart contract in 's +/// remarks holds here; the two PostgreSQL families exist on this host only. /// public sealed class DarlingDeltaCalculator : CollectorDeltaCalculator { - /* The four seed queries are verbatim from Lite's DeltaCalculator — deliberately written in - the PG-shared dialect (the (server_id, collection_time) row-value latest-row form, and the - reused $1 positional placeholder, both run on either engine as-is). Pinned by - DarlingDeltaSeederTests, and mirrored in Lite by LiteDeltaSeederTests. + /* The seed queries are verbatim from Lite's DeltaCalculator — deliberately written in the + PG-shared dialect (the (server_id, collection_time) row-value latest-row form, DISTINCT ON for + the latest-row-per-key form, and the reused $1 positional placeholder, all of which run on + either engine as-is). Pinned by DarlingDeltaSeederTests, mirrored in Lite by + LiteDeltaSeederTests, and read from both hosts' sources by Lite.Tests' + DeltaFamilySeedingCensusTests, which proves every family has a seeder on each host. $1 is CollectorDeltaCalculator.SeedCutoff(), and it is bound on BOTH the outer read and the inner MAX() — either one left unbounded scans every chunk of the hypertable. That is what #1772 was: on a 276 GB field store the unbounded form could not finish inside the 30-second command timeout, so restart continuity silently degraded to first-cycle-zero deltas every time the service came up. The bound is free rather than a trade, because the delta gap policy - throws away anything older than it anyway — see CollectorDeltaCalculator.SeedLookback. */ + throws away anything older than it anyway — see CollectorDeltaCalculator.SeedLookback. + + Two shapes, chosen per family by how the collector WRITES (#3540 A4): + + - Latest collection per server (the original four, latch_stats, spinlock_stats): these + collectors write every key they read on every pass, so the newest collection holds every + key's current counter and the row-value probe is the cheapest exact read. + - Latest row per key (procedure_stats, query_stats, pg_wait_stats, pg_statement_stats): these + collectors do NOT write every key every pass — procedure_stats and query_stats are TOP (n) + reads that churn, and the two PostgreSQL collectors skip idle rows at the write — so the newest + collection is missing keys whose counters are nevertheless unchanged, and a key seeded from + nothing takes the first-sighting path. DISTINCT ON (server_id, key) ... ORDER BY + collection_time DESC over the + cutoff window returns each key's newest row instead. Its bound is the single + collection_time >= $1 on its only table read: there is no inner aggregate to bind a second + time. On the hypertable that bound is what keeps the read to the window's chunk(s) through + the (server_id, collection_time) index — the same chunk exclusion the #1772 pin proves for + the row-value shape — and the window's rows are the whole working set: one sort over fifteen + minutes of one family, which on the dogfood fleet's largest family is tens of thousands of + rows, not a hypertable. A key seeded from an older row inside the window carries that row's + timestamp, so its first delta spans a longer interval than the in-memory cache would have + measured; the value is exact (the counter was idle in between, which is why no newer row + exists) and the gap policy still bounds the span. + + query_stats joined the per-key shape with Darling V128 / Lite v61 (#3540). Its delta key is + sql_handle:statement_start_offset:statement_end_offset:plan_handle, and until V128 the store + persisted neither offset, so no row could reproduce the key the collector presents and only the + family's PASS WINDOW could be seeded (the #2235 series-age rescue's input). The offsets are + stored now, raw (-1 = "to the end of the batch", byte offsets into the nvarchar batch text) and + the seed rebuilds the key from them with the collector's own interpolation. Two rules, both in + the seeder rather than the SQL: a row whose offsets are NULL — every row written before V128 — + seeds NO key, because a key built from a fabricated 0/-1 would be one nothing ever presents and + the baseline under it would sit unread until it aged out; and EVERY row, NULL offsets or not, + still feeds the pass window, so the first restart after the upgrade (when the whole window is + pre-V128 rows) keeps the rescue armed exactly as #3614 left it. One read serves both, which is + why the offset filter is not in the WHERE. */ public const string WaitStatsSeedSql = @" SELECT server_id, wait_type, waiting_tasks_count, wait_time_ms, signal_wait_time_ms, collection_time @@ -78,11 +118,125 @@ FROM memory_grant_stats SELECT server_id, MAX(collection_time) FROM memory_grant_stats WHERE collection_time >= $1 GROUP BY server_id )"; + /* #3540 A4: the six families below were never seeded. Column lists and key formats mirror each + collector's own CalculateDelta* call exactly — a key that differs by one character seeds a + baseline nothing will ever read. LatchStatsCollector / SpinlockStatsCollector key on the class + / spinlock name and write every row every pass, so they take the latest-collection shape. */ + + public const string LatchStatsSeedSql = @" +SELECT server_id, latch_class, waiting_requests_count, wait_time_ms, max_wait_time_ms, collection_time +FROM latch_stats +WHERE collection_time >= $1 +AND (server_id, collection_time) IN ( + SELECT server_id, MAX(collection_time) FROM latch_stats WHERE collection_time >= $1 GROUP BY server_id +)"; + + public const string SpinlockStatsSeedSql = @" +SELECT server_id, spinlock_name, collisions, spins, sleep_time, backoffs, collection_time +FROM spinlock_stats +WHERE collection_time >= $1 +AND (server_id, collection_time) IN ( + SELECT server_id, MAX(collection_time) FROM spinlock_stats WHERE collection_time >= $1 GROUP BY server_id +)"; + + /* ProcedureStatsCollector keys on plan_handle, falling back to database.schema.object when the + handle is null; the SQL builds the SAME string (a null part formats as empty, as C# string + interpolation does) so DISTINCT ON partitions by the key the collector will present. Latest row + per key, because a TOP (150) drops and readmits plans between passes. */ + public const string ProcedureStatsSeedSql = @" +SELECT DISTINCT ON (server_id, delta_key) + server_id, delta_key, + execution_count, total_worker_time, total_elapsed_time, + total_logical_reads, total_logical_writes, total_physical_reads, total_spills, + collection_time +FROM ( + SELECT server_id, + COALESCE(plan_handle, COALESCE(database_name, '') || '.' || COALESCE(schema_name, '') || '.' || COALESCE(object_name, '')) AS delta_key, + execution_count, total_worker_time, total_elapsed_time, + total_logical_reads, total_logical_writes, total_physical_reads, total_spills, + collection_time + FROM procedure_stats + WHERE collection_time >= $1 +) AS recent +ORDER BY server_id, delta_key, collection_time DESC"; + + /* QueryStatsCollector keys on the dm_exec_query_stats row identity — sql_handle, both statement + offsets, plan_handle — and its TOP (n) churns like procedure_stats', so: latest row per that + identity. DISTINCT ON treats NULL offsets as one group, which is harmless: those are pre-V128 rows + the seeder reads for the pass window only (see the header). The eight counters are the ones the + collector's eight CalculateDeltaWithSeriesAge calls difference. */ + public const string QueryStatsSeedSql = @" +SELECT DISTINCT ON (server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle) + server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle, + execution_count, total_worker_time, total_elapsed_time, + total_logical_reads, total_logical_writes, total_physical_reads, total_rows, total_spills, + collection_time +FROM query_stats +WHERE collection_time >= $1 +ORDER BY server_id, sql_handle, statement_start_offset, statement_end_offset, plan_handle, collection_time DESC"; + + /* PgWaitStatsCollector keys on the numeric wait_event_id (never the name — it changes case across + Aurora majors) and skips idle rows at the write, so: latest row per (server, event id). */ + public const string PgWaitStatsSeedSql = @" +SELECT DISTINCT ON (server_id, wait_event_id) + server_id, wait_event_id, waits, wait_time_us, collection_time +FROM pg_wait_stats +WHERE collection_time >= $1 +ORDER BY server_id, wait_event_id, collection_time DESC"; + + /* PgStatementStatsCollector keys on the full pg_stat_statements identity + (queryid, database_id, user_id, toplevel) and skips idle rows at the write, so: latest row per + that identity. database_id is the datid OID the collector writes today (#3540 A11c names the + DROP/CREATE reuse trap in that choice); the seed reproduces the key as written, it does not + redesign it. total_exec_time_ms is a double in the store and the collector deltas it as a + truncated long — the reader below truncates the same way. */ + public const string PgStatementStatsSeedSql = @" +SELECT DISTINCT ON (server_id, queryid, database_id, user_id, toplevel) + server_id, queryid, database_id, user_id, toplevel, + calls, total_exec_time_ms, rows_returned, collection_time +FROM pg_statement_stats +WHERE collection_time >= $1 +ORDER BY server_id, queryid, database_id, user_id, toplevel, collection_time DESC"; + + /* The delta GROUP names each family's collector passes — the pass window is keyed by these, not + by the collector name, so a seeder that seeded "query_stats" would arm nothing. Spelled once per + family here and read by the seeders below; the census test compares them against the + collectors' own call sites. */ + internal static readonly string[] WaitStatsGroups = { "wait_stats_tasks", "wait_stats_time", "wait_stats_signal" }; + internal static readonly string[] FileIoStatsGroups = + { + "file_io_reads", "file_io_writes", "file_io_read_bytes", "file_io_write_bytes", + "file_io_stall_read", "file_io_stall_write", "file_io_stall_queued_read", "file_io_stall_queued_write", + }; + internal static readonly string[] PerfmonStatsGroups = { "perfmon" }; + internal static readonly string[] MemoryGrantStatsGroups = { "memory_grants_timeouts", "memory_grants_forced" }; + internal static readonly string[] LatchStatsGroups = { "latch_stats_waiting_requests", "latch_stats_wait_time", "latch_stats_max_wait" }; + internal static readonly string[] SpinlockStatsGroups = { "spinlock_stats_collisions", "spinlock_stats_spins", "spinlock_stats_sleep_time", "spinlock_stats_backoffs" }; + internal static readonly string[] ProcedureStatsGroups = + { + "proc_stats_exec", "proc_stats_worker", "proc_stats_elapsed", "proc_stats_reads", + "proc_stats_writes", "proc_stats_phys_reads", "proc_stats_spills", + }; + internal static readonly string[] QueryStatsGroups = + { + "query_stats_exec", "query_stats_worker", "query_stats_elapsed", "query_stats_reads", + "query_stats_writes", "query_stats_phys_reads", "query_stats_rows", "query_stats_spills", + }; + internal static readonly string[] PgWaitStatsGroups = { "pg_wait_stats_waits", "pg_wait_stats_time" }; + internal static readonly string[] PgStatementStatsGroups = { "pg_statement_stats_calls", "pg_statement_stats_time", "pg_statement_stats_rows" }; + /// /// Seeds the delta cache from the Postgres store so that the first collection after a service /// restart can produce accurate deltas instead of returning 0 for everything. A seed failure /// logs a warning and the service proceeds with first-cycle-zero deltas — restart continuity /// must never block collection. + /// + /// Each family seeds under its own guard. Before #3540 one connection-level try wrapped four + /// reads in series, so the first family to throw cost every later family its continuity, silently; + /// at eleven reads that posture is worse, and a family whose read is slow on one store is exactly + /// the case where the other ten still have their rows to give. A family that fails logs a warning + /// naming itself and the seed moves on. Every command carries the bootstrap deadline: these run + /// once, awaited, ahead of the collection loop, and the startup deadline census pins each one. /// public async Task SeedFromStoreAsync(NpgsqlDataSource postgres, ILogger? logger, CancellationToken cancellationToken) { @@ -90,13 +244,19 @@ public async Task SeedFromStoreAsync(NpgsqlDataSource postgres, ILogger? logger, { await using var connection = await postgres.OpenConnectionAsync(cancellationToken); - /* One cutoff for all four reads, so they describe the same instant. */ + /* One cutoff for all reads, so they describe the same instant. */ var cutoff = SeedCutoff(); - await SeedWaitStatsAsync(connection, cutoff, logger, cancellationToken); - await SeedFileIoStatsAsync(connection, cutoff, logger, cancellationToken); - await SeedPerfmonStatsAsync(connection, cutoff, logger, cancellationToken); - await SeedMemoryGrantStatsAsync(connection, cutoff, logger, cancellationToken); + await SeedFamilyAsync("wait_stats", () => SeedWaitStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("file_io_stats", () => SeedFileIoStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("perfmon_stats", () => SeedPerfmonStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("memory_grant_stats", () => SeedMemoryGrantStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("latch_stats", () => SeedLatchStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("spinlock_stats", () => SeedSpinlockStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("procedure_stats", () => SeedProcedureStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("query_stats", () => SeedQueryStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("pg_wait_stats", () => SeedPgWaitStatsAsync(connection, cutoff, logger, cancellationToken), logger); + await SeedFamilyAsync("pg_statement_stats", () => SeedPgStatementStatsAsync(connection, cutoff, logger, cancellationToken), logger); logger?.LogInformation( "Delta calculator seeded from Postgres store (baselines from the last {Minutes} minutes)", @@ -108,12 +268,25 @@ public async Task SeedFromStoreAsync(NpgsqlDataSource postgres, ILogger? logger, } } + private static async Task SeedFamilyAsync(string family, Func seed, ILogger? logger) + { + try + { + await seed(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning(ex, "Failed to seed {Family} delta baselines from Postgres store, its first collection will return 0 deltas", family); + } + } + private async Task SeedWaitStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) { using var cmd = new NpgsqlCommand(WaitStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; cmd.Parameters.AddWithValue(cutoff); using var reader = await cmd.ExecuteReaderAsync(cancellationToken); var count = 0; + var passes = new SeedPassTracker(); while (await reader.ReadAsync(cancellationToken)) { var serverId = reader.GetInt32(0); @@ -123,8 +296,10 @@ private async Task SeedWaitStatsAsync(NpgsqlConnection connection, DateTime cuto Seed(serverId, "wait_stats_tasks", waitType, reader.GetInt64(2), ts); Seed(serverId, "wait_stats_time", waitType, reader.GetInt64(3), ts); Seed(serverId, "wait_stats_signal", waitType, reader.GetInt64(4), ts); + passes.Observe(serverId, ts); count++; } + SeedPasses(passes, WaitStatsGroups); if (count > 0) logger?.LogDebug("Seeded {Count} wait_stats baseline rows", count); } @@ -134,6 +309,7 @@ private async Task SeedFileIoStatsAsync(NpgsqlConnection connection, DateTime cu cmd.Parameters.AddWithValue(cutoff); using var reader = await cmd.ExecuteReaderAsync(cancellationToken); var count = 0; + var passes = new SeedPassTracker(); while (await reader.ReadAsync(cancellationToken)) { var serverId = reader.GetInt32(0); @@ -149,8 +325,10 @@ private async Task SeedFileIoStatsAsync(NpgsqlConnection connection, DateTime cu Seed(serverId, "file_io_stall_write", deltaKey, reader.IsDBNull(8) ? 0 : reader.GetInt64(8), ts); Seed(serverId, "file_io_stall_queued_read", deltaKey, reader.IsDBNull(9) ? 0 : reader.GetInt64(9), ts); Seed(serverId, "file_io_stall_queued_write", deltaKey, reader.IsDBNull(10) ? 0 : reader.GetInt64(10), ts); + passes.Observe(serverId, ts); count++; } + SeedPasses(passes, FileIoStatsGroups); if (count > 0) logger?.LogDebug("Seeded {Count} file_io_stats baseline rows", count); } @@ -160,6 +338,7 @@ private async Task SeedPerfmonStatsAsync(NpgsqlConnection connection, DateTime c cmd.Parameters.AddWithValue(cutoff); using var reader = await cmd.ExecuteReaderAsync(cancellationToken); var count = 0; + var passes = new SeedPassTracker(); while (await reader.ReadAsync(cancellationToken)) { var serverId = reader.GetInt32(0); @@ -168,35 +347,199 @@ private async Task SeedPerfmonStatsAsync(NpgsqlConnection connection, DateTime c var instance = reader.IsDBNull(3) ? "" : reader.GetString(3); var ts = reader.IsDBNull(5) ? (DateTime?)null : reader.GetDateTime(5); Seed(serverId, "perfmon", $"{objectName}|{counter}|{instance}", reader.GetInt64(4), ts); + passes.Observe(serverId, ts); count++; } + SeedPasses(passes, PerfmonStatsGroups); if (count > 0) logger?.LogDebug("Seeded {Count} perfmon_stats baseline rows", count); } private async Task SeedMemoryGrantStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) { - try + using var cmd = new NpgsqlCommand(MemoryGrantStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) + { + var serverId = reader.GetInt32(0); + var poolId = reader.IsDBNull(1) ? 0 : reader.GetInt32(1); + var semaphoreId = reader.IsDBNull(2) ? (short)0 : reader.GetInt16(2); + var deltaKey = $"{poolId}_{semaphoreId}"; + var ts = reader.IsDBNull(5) ? (DateTime?)null : reader.GetDateTime(5); + Seed(serverId, "memory_grants_timeouts", deltaKey, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); + Seed(serverId, "memory_grants_forced", deltaKey, reader.IsDBNull(4) ? 0 : reader.GetInt64(4), ts); + passes.Observe(serverId, ts); + count++; + } + SeedPasses(passes, MemoryGrantStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} memory_grant_stats baseline rows", count); + } + + private async Task SeedLatchStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(LatchStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) + { + var serverId = reader.GetInt32(0); + var latchClass = reader.GetString(1); + var ts = reader.IsDBNull(5) ? (DateTime?)null : reader.GetDateTime(5); + Seed(serverId, "latch_stats_waiting_requests", latchClass, reader.IsDBNull(2) ? 0 : reader.GetInt64(2), ts); + Seed(serverId, "latch_stats_wait_time", latchClass, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); + Seed(serverId, "latch_stats_max_wait", latchClass, reader.IsDBNull(4) ? 0 : reader.GetInt64(4), ts); + passes.Observe(serverId, ts); + count++; + } + SeedPasses(passes, LatchStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} latch_stats baseline rows", count); + } + + private async Task SeedSpinlockStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(SpinlockStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) + { + var serverId = reader.GetInt32(0); + var spinlockName = reader.GetString(1); + var ts = reader.IsDBNull(6) ? (DateTime?)null : reader.GetDateTime(6); + Seed(serverId, "spinlock_stats_collisions", spinlockName, reader.IsDBNull(2) ? 0 : reader.GetInt64(2), ts); + Seed(serverId, "spinlock_stats_spins", spinlockName, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); + Seed(serverId, "spinlock_stats_sleep_time", spinlockName, reader.IsDBNull(4) ? 0 : reader.GetInt64(4), ts); + Seed(serverId, "spinlock_stats_backoffs", spinlockName, reader.IsDBNull(5) ? 0 : reader.GetInt64(5), ts); + passes.Observe(serverId, ts); + count++; + } + SeedPasses(passes, SpinlockStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} spinlock_stats baseline rows", count); + } + + private async Task SeedProcedureStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(ProcedureStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) + { + var serverId = reader.GetInt32(0); + /* The key exactly as ProcedureStatsCollector.WritePayload builds it — computed in the SQL so + DISTINCT ON partitions by it; read back verbatim. */ + var deltaKey = reader.IsDBNull(1) ? "" : reader.GetString(1); + var ts = reader.IsDBNull(9) ? (DateTime?)null : reader.GetDateTime(9); + Seed(serverId, "proc_stats_exec", deltaKey, reader.IsDBNull(2) ? 0 : reader.GetInt64(2), ts); + Seed(serverId, "proc_stats_worker", deltaKey, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); + Seed(serverId, "proc_stats_elapsed", deltaKey, reader.IsDBNull(4) ? 0 : reader.GetInt64(4), ts); + Seed(serverId, "proc_stats_reads", deltaKey, reader.IsDBNull(5) ? 0 : reader.GetInt64(5), ts); + Seed(serverId, "proc_stats_writes", deltaKey, reader.IsDBNull(6) ? 0 : reader.GetInt64(6), ts); + Seed(serverId, "proc_stats_phys_reads", deltaKey, reader.IsDBNull(7) ? 0 : reader.GetInt64(7), ts); + Seed(serverId, "proc_stats_spills", deltaKey, reader.IsDBNull(8) ? 0 : reader.GetInt64(8), ts); + passes.Observe(serverId, ts); + count++; + } + SeedPasses(passes, ProcedureStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} procedure_stats baseline rows", count); + } + + private async Task SeedQueryStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(QueryStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var preV128 = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) { - using var cmd = new NpgsqlCommand(MemoryGrantStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; - cmd.Parameters.AddWithValue(cutoff); - using var reader = await cmd.ExecuteReaderAsync(cancellationToken); - var count = 0; - while (await reader.ReadAsync(cancellationToken)) + var serverId = reader.GetInt32(0); + var ts = reader.IsDBNull(13) ? (DateTime?)null : reader.GetDateTime(13); + + /* The pass window takes EVERY row, offsets or not — see the header. */ + passes.Observe(serverId, ts); + + /* A pre-V128 row never recorded its offsets. Its key cannot be rebuilt, and a key built from a + guessed pair would be a baseline nothing ever reads — so it seeds nothing. */ + if (reader.IsDBNull(2) || reader.IsDBNull(3)) { - var serverId = reader.GetInt32(0); - var poolId = reader.IsDBNull(1) ? 0 : reader.GetInt32(1); - var semaphoreId = reader.IsDBNull(2) ? (short)0 : reader.GetInt16(2); - var deltaKey = $"{poolId}_{semaphoreId}"; - var ts = reader.IsDBNull(5) ? (DateTime?)null : reader.GetDateTime(5); - Seed(serverId, "memory_grants_timeouts", deltaKey, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); - Seed(serverId, "memory_grants_forced", deltaKey, reader.IsDBNull(4) ? 0 : reader.GetInt64(4), ts); - count++; + preV128++; + continue; } - if (count > 0) logger?.LogDebug("Seeded {Count} memory_grant_stats baseline rows", count); + + /* The key exactly as QueryStatsCollector.WritePayload spells it: the same interpolation over + the same raw parts, so a null handle formats as empty and the offsets — -1 included — are + spelled by the same int formatting on both sides. Never normalized. */ + var sqlHandle = reader.IsDBNull(1) ? null : reader.GetString(1); + var planHandle = reader.IsDBNull(4) ? null : reader.GetString(4); + var deltaKey = $"{sqlHandle}:{reader.GetInt32(2)}:{reader.GetInt32(3)}:{planHandle}"; + + Seed(serverId, "query_stats_exec", deltaKey, reader.IsDBNull(5) ? 0 : reader.GetInt64(5), ts); + Seed(serverId, "query_stats_worker", deltaKey, reader.IsDBNull(6) ? 0 : reader.GetInt64(6), ts); + Seed(serverId, "query_stats_elapsed", deltaKey, reader.IsDBNull(7) ? 0 : reader.GetInt64(7), ts); + Seed(serverId, "query_stats_reads", deltaKey, reader.IsDBNull(8) ? 0 : reader.GetInt64(8), ts); + Seed(serverId, "query_stats_writes", deltaKey, reader.IsDBNull(9) ? 0 : reader.GetInt64(9), ts); + Seed(serverId, "query_stats_phys_reads", deltaKey, reader.IsDBNull(10) ? 0 : reader.GetInt64(10), ts); + Seed(serverId, "query_stats_rows", deltaKey, reader.IsDBNull(11) ? 0 : reader.GetInt64(11), ts); + Seed(serverId, "query_stats_spills", deltaKey, reader.IsDBNull(12) ? 0 : reader.GetInt64(12), ts); + count++; } - catch (Exception ex) when (ex is not OperationCanceledException) + SeedPasses(passes, QueryStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} query_stats baseline rows", count); + if (preV128 > 0) logger?.LogDebug("Skipped {Count} query_stats rows with no stored statement offsets (pre-V128); their collection times still seeded the pass window", preV128); + } + + private async Task SeedPgWaitStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(PgWaitStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) + { + var serverId = reader.GetInt32(0); + /* The key exactly as PgWaitStatsCollector builds it: the event id, invariant-formatted. */ + var key = reader.GetInt64(1).ToString(CultureInfo.InvariantCulture); + var ts = reader.IsDBNull(4) ? (DateTime?)null : reader.GetDateTime(4); + Seed(serverId, "pg_wait_stats_waits", key, reader.IsDBNull(2) ? 0 : reader.GetInt64(2), ts); + Seed(serverId, "pg_wait_stats_time", key, reader.IsDBNull(3) ? 0 : reader.GetInt64(3), ts); + passes.Observe(serverId, ts); + count++; + } + SeedPasses(passes, PgWaitStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} pg_wait_stats baseline rows", count); + } + + private async Task SeedPgStatementStatsAsync(NpgsqlConnection connection, DateTime cutoff, ILogger? logger, CancellationToken cancellationToken) + { + using var cmd = new NpgsqlCommand(PgStatementStatsSeedSql, connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; + cmd.Parameters.AddWithValue(cutoff); + using var reader = await cmd.ExecuteReaderAsync(cancellationToken); + var count = 0; + var passes = new SeedPassTracker(); + while (await reader.ReadAsync(cancellationToken)) { - /* Table may not exist on first run after schema migration — mirrors Lite's tolerance. */ + var serverId = reader.GetInt32(0); + /* The key exactly as PgStatementStatsCollector.ReadAsync builds it. */ + var key = string.Create(CultureInfo.InvariantCulture, + $"{reader.GetInt64(1)}|{reader.GetInt64(2)}|{reader.GetInt64(3)}|{(!reader.IsDBNull(4) && reader.GetBoolean(4) ? 1 : 0)}"); + var ts = reader.IsDBNull(8) ? (DateTime?)null : reader.GetDateTime(8); + Seed(serverId, "pg_statement_stats_calls", key, reader.IsDBNull(5) ? 0 : reader.GetInt64(5), ts); + /* (long) of the stored double — the collector's own truncation. */ + Seed(serverId, "pg_statement_stats_time", key, reader.IsDBNull(6) ? 0 : (long)reader.GetDouble(6), ts); + Seed(serverId, "pg_statement_stats_rows", key, reader.IsDBNull(7) ? 0 : reader.GetInt64(7), ts); + passes.Observe(serverId, ts); + count++; } + SeedPasses(passes, PgStatementStatsGroups); + if (count > 0) logger?.LogDebug("Seeded {Count} pg_statement_stats baseline rows", count); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingObservability.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingObservability.cs index 86d961399..9b20be337 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingObservability.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingObservability.cs @@ -596,13 +596,16 @@ internal static (int Elapsed, int TargetMs, int StorageMs) SplitSweepPhases(long /// /// Upserts the per-server analysis-state marker (V19) after an analysis pass: insufficient_data = - /// true plus the engine's message when the pass hit the 24h data-span gate, or false + null - /// when a real pass completed on enough data. The engine ALREADY makes this determination - /// (DarlingAnalysisService.InsufficientDataMessage); this persists it so the Viewer's - /// Recommendations tab — which never calls the engine — shows "still collecting" instead of a false - /// all-clear on a young deployment's zero-finding read. One row per server, upserted on - /// server_id. Failure-isolated (Debug + no-op) like the other observability writes — an - /// analysis-state write must never break the collection loop. + /// true plus the engine's message when the pass hit the 24h data-span gate; false plus the + /// engine's message when the pass cleared the gate but the analysis window itself collected zero facts + /// (#3524/#3551 — false-with-a-message is written ONLY for that window-empty shape, which is how the + /// viewer tells it from a clean pass without a schema change); or false + null when a real pass + /// completed on measured facts. The engine ALREADY makes these determinations + /// (DarlingAnalysisService.InsufficientDataMessage / WindowEmptyMessage); this persists + /// them so the Viewer's Recommendations tab — which never calls the engine — shows "still collecting" + /// or "collection appears broken" instead of a false all-clear on a zero-finding read. One row per + /// server, upserted on server_id. Failure-isolated (Debug + no-op) like the other observability + /// writes — an analysis-state write must never break the collection loop. /// public static async Task WriteAnalysisStateAsync( NpgsqlDataSource postgres, diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs index 23ed7d0b7..84b22ea28 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingPostgresAlertReadAdapter.cs @@ -74,6 +74,25 @@ FROM pg_wraparound_stats /// times something held it, how often was it this one" — which is the question. Note this became reachable /// only once the collector stopped attributing its own backend: while Darling's own snapshot was always a /// session holder, every collection had a holder and the distinction was invisible. + /// #3537: two window figures the identity fraction cannot supply. + /// observations_above_threshold counts the collections whose WINNING age sat at or above the + /// evaluator's warning threshold, holder identity ignored — the alert's own claim is about the horizon, + /// and a horizon continuously pinned by a parade of DISTINCT holders never accumulates any single + /// holder's fraction. The threshold arrives as a bind from + /// so the condition counted here and the + /// one evaluated there cannot drift apart. + /// captures_in_window is the horizon arm's denominator, and it comes from + /// collection_log — this collector's own SUCCESS rows — rather than from this table, whose + /// distinct collection times count only holder-bearing collections (see above) and so read the first + /// holder after quiet hours as 1 of 1, 100%, "chronic". The log gets a row per run INCLUDING zero-row + /// (healthy, unheld) runs, sits behind its (server_id, collection_time) index, and counts the exact + /// collector whose captures are being fractioned — cheaper and more honest than inferring the cadence + /// from a cadence-mate table like pg_database_stats, which measures a different collector's + /// fate. Runs that stored nothing (ERROR / ABANDONED / PERMISSIONS / YIELDED) are excluded: a cycle + /// that could not look is not evidence the horizon was clear. The log write is failure-isolated and + /// can silently skip a row, so the count may UNDERCOUNT — which only inflates the fraction of a + /// horizon already measured above threshold, and the evaluator's minimum-captures floor keeps a + /// near-empty log from firing at all. /// internal const string XminSql = """ WITH latest AS ( @@ -92,10 +111,22 @@ window_stats AS ( WHERE is_winner AND source = (SELECT source FROM latest) AND holder IS NOT DISTINCT FROM (SELECT holder FROM latest) - ) AS observations_held + ) AS observations_held, + COUNT(DISTINCT collection_time) FILTER ( + WHERE is_winner + AND xmin_age >= $3 + ) AS observations_above_threshold FROM pg_xmin_horizon WHERE server_id = $1 AND collection_time >= $2 + ), + captures AS ( + SELECT COUNT(*) AS captures_in_window + FROM collection_log + WHERE server_id = $1 + AND collector_name = 'pg_xmin_horizon' + AND collection_time >= $2 + AND status = 'SUCCESS' ) SELECT l.source, @@ -103,9 +134,12 @@ FROM pg_xmin_horizon l.xmin_age, w.observations_held, w.observations_total, - l.detail + l.detail, + w.observations_above_threshold, + c.captures_in_window FROM latest AS l CROSS JOIN window_stats AS w + CROSS JOIN captures AS c """; /// @@ -241,6 +275,11 @@ answer consistently off one query. 0 reads as "no window data" -> FreezingIsKeep command.CommandTimeout = DarlingAlertReadAdapter.AlertPassCommandTimeoutSeconds; command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(NaiveUtcNow() - Freshness); + /* The evaluator's own age threshold, not a local copy: the SQL counts "collections above + threshold" and the evaluator fractions that count against the SAME bar, so read and evaluation + must agree on it or the horizon arm silently means something else — the PoisonWaitSql window + discipline, applied to a level. */ + command.Parameters.AddWithValue(PostgresAlertEvaluator.XminAgeWarningThreshold); await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (!await reader.ReadAsync(cancellationToken)) { @@ -253,7 +292,12 @@ answer consistently off one query. 0 reads as "no window data" -> FreezingIsKeep reader.IsDBNull(2) ? 0 : reader.GetInt64(2), reader.IsDBNull(3) ? 0 : (int)reader.GetInt64(3), reader.IsDBNull(4) ? 0 : (int)reader.GetInt64(4), - reader.IsDBNull(5) ? null : reader.GetString(5)); + reader.IsDBNull(5) ? null : reader.GetString(5), + /* ordinals 6/7: the #3537 horizon-arm figures. 0 reads as "no window data" in both — the + evaluator's floor keeps that from firing, the same conservative default the wraparound + window peaks take. */ + reader.IsDBNull(6) ? 0 : (int)reader.GetInt64(6), + reader.IsDBNull(7) ? 0 : (int)reader.GetInt64(7)); } public async Task> GetReplicationSlotRiskAsync( diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs index 62efe5959..7b2ecef93 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs @@ -94,15 +94,22 @@ faster than the staleness backstop when a connected server's collectors are erro cycle. 10 spans a couple of minutes of total failure across the frequently-scheduled collectors. */ internal const int ConsecutiveFailureThreshold = 10; - /* Store Disk Pressure fires when the store volume drops below this percent free — a percentage (not an - absolute floor) so it scales from a small managed box to a large fleet disk; 10% is the universal DBA - "act now" threshold for a database volume, and mirrors the shared engine's target-server low-disk - percent (LowDiskThresholdPercent). Percent-only by design (defaults over speculative config — a GB - floor is a trivial follow-up if an operator ever wants one). The condition no-ops when free space is - undeterminable (a remote BYO store), so it never false-alarms; the managed store's own volume is the - case it exists to protect. */ + /* Store Disk Pressure fires when the store volume drops below this percent free — a percentage so it + scales from a small managed box to a large fleet disk; 10% is the universal DBA "act now" threshold + for a database volume, and mirrors the shared engine's target-server low-disk percent + (LowDiskThresholdPercent). The condition no-ops when free space is undeterminable (a remote BYO + store), so it never false-alarms; the managed store's own volume is the case it exists to protect. */ internal const double DiskFreeWarnPercent = 10.0; + /* #3528: the percent's GB floor — pressure additionally requires free space BELOW this many GB, an AND + qualifier so a big volume at a low percent (400 GB free on a 4 TB store) never pages CRITICAL. This + was "percent-only by design (a GB floor is a trivial follow-up if an operator ever wants one)"; #3528 + is that want. The composition is PvsFloorGb's (percent triggers, the floor keeps it honest, 0 removes + the floor), deliberately NOT the target-volume pair's OR — there the GB dimension ADDS fires, which + would make this alert noisier, the opposite of the complaint. 50 puts the crossover at a 500 GB + volume: below that the percent governs exactly as before; above it, 50 GB free is the line. */ + internal const double DiskFreeWarnFloorGb = 50.0; + private readonly IAlertEngineSettings _settings; private readonly IAlertDeliverer _deliverer; private readonly IAlertHistoryStore _historyStore; @@ -328,11 +335,28 @@ reports so the threshold a reader falsifies against is the one that selected the /// and cannot be taken from a phone at 3am. A longer interval alone would have produced the same /// unactionable card less often. /// - /// In-memory, like every sibling's interval, and a restart costs one extra digest. That is - /// the failure class and #3430's RepeatDeliveryBudget both already - /// accept, and it is what keeps this change free of a migration rung. An extra copy of a report is the - /// cheapest possible failure; nothing is lost either way, because the digest is recomputed from the - /// store every time rather than accumulated in process. + /// Gated on DELIVERED-TODAY in the store, not fired-today in memory (#3580). This shipped + /// as "in-memory, like every sibling's interval, and a restart costs one extra digest" — the failure + /// class and #3430's RepeatDeliveryBudget both accept, on the + /// argument that an extra copy of a report is the cheapest possible failure. The v3.8.0 install night + /// priced it: three stores restarted once each, and the channel carried SIX re-announcements among ~23 + /// overnight posts — a quarter of the channel was this document and the rollup, twice — because a fresh + /// process has an empty and the gate was answering "has THIS PROCESS sent + /// one today" when the reader's question is "has one been DELIVERED today". The same night showed the + /// case the fix must keep: a pair whose delivery had FAILED (a transport fault, not a suppression) was + /// correctly re-attempted after the restart and landed; the restart was the recovery. + /// + /// So the gate now reads a delivery stamp from (the + /// store's existing key/value state table, no rung) and skips while now - stamp is inside this + /// interval, restart or not; and the stamp is written only when the deliverer reports a disposition + /// other than . A failed delivery writes nothing, so the next + /// tick — restart or not — retries; the in-memory dictionary remains as a same-process fast path (23 of + /// 24 ticks still cost one lookup and no store read) but is no longer the authority. A store fault on + /// the stamp falls back to that fast path and warns — fail-open toward delivering, the direction every + /// store-fault posture in this evaluator already takes, because the alternative (skip on an unreadable + /// stamp) would let a store hiccup silence a daily document, and the memory gate still bounds the + /// fallback at one copy per process. The nothing-is-lost half of the original argument still holds: + /// the digest is recomputed from the store every time rather than accumulated in process. /// internal static readonly TimeSpan CollectorCostDigestInterval = TimeSpan.FromDays(1); @@ -345,9 +369,11 @@ reports so the threshold a reader falsifies against is the one that selected the /// How many collectors the digest's heaviest-first census spells out. private const int MaxListedCostHeaviest = 10; - /// When the digest was last sent — the idiom, one fixed key. - /// A digest has no active flag and no resolution edge: it is a report of a measurement, not a condition - /// that can be entered and left, so there is nothing to clear. + /// When the digest was last known DELIVERED — the idiom, one + /// fixed key, but since #3580 a CACHE of the store's stamp rather than the authority: filled from the + /// stamp on the first tick that has to ask, and from the fire itself on a delivery the deliverer did + /// not report failed. A digest has no active flag and no resolution edge: it is a report of a + /// measurement, not a condition that can be entered and left, so there is nothing to clear. private readonly ConcurrentDictionary _lastCostDigest = new(); /// @@ -371,10 +397,15 @@ reports so the threshold a reader falsifies against is the one that selected the /// the trailing day and only the trailing day, so what a post claims to cover and how often one can /// arrive are the same number and cannot drift apart. /// - /// In-memory, like , and a restart costs one extra - /// rollup. The same accepted failure class: an extra copy of a report is the cheapest possible - /// failure, and the rollup is recomputed from the sweep store every time rather than accumulated in - /// process, so nothing is lost in either direction. + /// Gated on delivered-today in the store, like and + /// for the same night's reason (#3580). This shipped in-memory with "a restart costs one extra + /// rollup" accepted as the cheapest failure; the install night's census — three restarts, six + /// re-announcements, this document being three of them — is what that acceptance cost, and the digest's + /// remarks carry the arc. The one-post-per-day CEILING above is a ruling, and a gate that a restart + /// resets is a ceiling the deployment procedure breaches on every install. The rollup takes the same + /// stamp store, the same failed-writes-nothing rule and the same fail-open fallback, under its own key. + /// Still recomputed from the sweep store every time rather than accumulated in process, so nothing is + /// lost in either direction. /// internal static readonly TimeSpan FleetSweepRollupInterval = TimeSpan.FromDays(1); @@ -391,9 +422,9 @@ reports so the threshold a reader falsifies against is the one that selected the /// stay readable on the fleet-wide day it exists for. private const int MaxListedRollupLedgerServers = 10; - /// When the rollup was last sent — the idiom, one fixed key. A - /// rollup is a report of a period, not a condition: no active flag, no resolution edge, nothing to - /// clear. + /// When the rollup was last known DELIVERED — the idiom, one + /// fixed key, and since #3580 the same cache-of-the-stamp role rather than the authority. A rollup is a + /// report of a period, not a condition: no active flag, no resolution edge, nothing to clear. private readonly ConcurrentDictionary _lastSweepRollup = new(); /// The fixed key for the fleet-level Store Disk Pressure edge (not a real server). @@ -621,7 +652,10 @@ is. The rule set is handed in by the worker from the live MuteRuleService cache when the job stops being stuck. Keyed by the CompressionKeyPrefix + job_id so the alert serverKey never collides with a real server_id (an int hash) — the deliverer's #1236 int.TryParse override no-ops on it, exactly like the non-numeric DiskKey. */ - private enum CompressionJobHealth { ReArmed, Escalated } + /* AwaitingSchedulerRetry (#3591): a -infinity row on a TimescaleDB whose scheduler recovers it by itself + (StuckCompressionJob.SchedulerRetries) — seen once, not re-armed, not paged; a second consecutive + sighting escalates. The other two states are #1581's. */ + private enum CompressionJobHealth { ReArmed, Escalated, AwaitingSchedulerRetry } private readonly ConcurrentDictionary _compressionJobState = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _lastCompressionJobAlert = new(StringComparer.Ordinal); @@ -766,7 +800,8 @@ public DarlingSelfAlertEvaluator( Func? retentionHoldWarnRatio = null, Func? retentionHoldCriticalRatio = null, AlertReadFailureCounter? readFailures = null, - string? storeName = null) + string? storeName = null, + ISelfAlertDeliveryStampStore? deliveryStamps = null) { _settings = settings ?? throw new ArgumentNullException(nameof(settings)); _deliverer = deliverer ?? throw new ArgumentNullException(nameof(deliverer)); @@ -801,8 +836,18 @@ than restated (#3060): a literal here is a third copy of the same number that a seam behaves like a store that never opted in — the AG-seam discipline, and the byte-identical promise the opt-in stands on. */ _storeLabel = EffectiveStoreLabel(storeName); + /* #3580: unsupplied means the two daily documents gate on process memory alone — the pre-#3580 + behavior, and what every test harness that does not care about restarts gets. Production + passes the store-backed stamps. */ + _deliveryStamps = deliveryStamps; } + /// + /// Where the two daily documents' DELIVERED-TODAY stamps live across restarts (#3580), or null when the + /// process-memory gate is the only gate. See for the arc. + /// + private readonly ISelfAlertDeliveryStampStore? _deliveryStamps; + /// /// Where a SWALLOWED self-alert store read is counted (#3013), or null when nothing is counting. /// Only the conditions that READ the store here increment it; the fleet-scoped conditions are handed @@ -1185,11 +1230,13 @@ rather than guessing. Falling back to "route everything to the page" would reins var routing = RouteCostRegressions(regressions, census); await ApplyCostRegressionsAsync(routing.Paging, cancellationToken); - /* The digest's own interval, checked here so 23 of every 24 hourly ticks do no extra store work. - The master switch already returned above, before the first store read (#3464); AlertsEnabled is + /* The digest's own interval, checked here so 23 of every 24 hourly ticks do no extra store work — + the delivered-today gate (#3580): memory first, the stamp only when memory cannot answer. The + master switch already returned above, before the first store read (#3464); AlertsEnabled is ALSO checked inside the apply, like every sibling, so a direct caller cannot skip it. */ - if (_lastCostDigest.TryGetValue(CollectorCostDigestKey, out var lastDigest) - && _utcNow() - lastDigest < CollectorCostDigestInterval) + if (await DocumentDeliveredInsideIntervalAsync( + _lastCostDigest, CollectorCostDigestKey, PgSelfAlertDeliveryStampStore.CostDigestStateKey, + CollectorCostDigestInterval, _utcNow(), "collector-cost digest", cancellationToken)) { return; } @@ -1457,8 +1504,9 @@ internal async Task ApplyCollectorCostDigestAsync( } var now = _utcNow(); - if (_lastCostDigest.TryGetValue(CollectorCostDigestKey, out var lastSent) - && now - lastSent < CollectorCostDigestInterval) + if (await DocumentDeliveredInsideIntervalAsync( + _lastCostDigest, CollectorCostDigestKey, PgSelfAlertDeliveryStampStore.CostDigestStateKey, + CollectorCostDigestInterval, now, "collector-cost digest", cancellationToken)) { return; } @@ -1468,10 +1516,13 @@ internal async Task ApplyCollectorCostDigestAsync( return; } - _lastCostDigest[CollectorCostDigestKey] = now; var (shortMessage, detail) = RenderCollectorCostDigest(movers, census); - await FireAsync( + /* #3580: the stamp is written AFTER the fire and only on a delivery the deliverer did not report + failed — it used to be written before the fire, unconditionally, which is the "fired-today" gate + this issue retires. A fire that throws before the deliverer is reached (the mute seam) delivered + nothing either, and takes the same path: no stamp, the next tick retries. */ + var delivery = await FireAsync( StoreKey(CollectorCostDigestKey), _storeLabel, CollectorCostDigestMetric, currentValue: movers.Count.ToString(CultureInfo.InvariantCulture), /* There is no threshold. Saying so in the string is the point of the string: this surface @@ -1487,6 +1538,10 @@ numeric below is the 0 the NOT NULL column demands. */ numericCurrentValue: movers.Count, numericThresholdValue: 0, cancellationToken); + + await RecordDocumentDeliveredAsync( + _lastCostDigest, CollectorCostDigestKey, PgSelfAlertDeliveryStampStore.CostDigestStateKey, + delivery, now, "collector-cost digest", cancellationToken); } /// @@ -1626,8 +1681,10 @@ is spelled from the constant so the words cannot drift from the read. */ /// left to discover it. /// /// Called from the worker's hourly store-metrics tick beside the collector-cost evaluation; 23 of - /// every 24 ticks cost one dictionary lookup. Testable through - /// with a recording deliverer and a controllable clock. + /// every 24 ticks cost one dictionary lookup, and the first tick after a start costs one stamp read + /// instead of a re-announcement (#3580 — ). Testable + /// through with a recording deliverer, a controllable clock and + /// an in-memory stamp store. /// public async Task EvaluateFleetSweepRollupAsync(NpgsqlDataSource postgres, CancellationToken cancellationToken) { @@ -1639,8 +1696,10 @@ public async Task EvaluateFleetSweepRollupAsync(NpgsqlDataSource postgres, Cance } var now = _utcNow(); - if (_lastSweepRollup.TryGetValue(FleetSweepRollupKey, out var lastSent) - && now - lastSent < FleetSweepRollupInterval) + /* #3580: delivered-today, memory first and the stamp when memory cannot answer. */ + if (await DocumentDeliveredInsideIntervalAsync( + _lastSweepRollup, FleetSweepRollupKey, PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, + FleetSweepRollupInterval, now, "fleet-sweep rollup", cancellationToken)) { return; } @@ -1720,8 +1779,9 @@ internal async Task ApplyFleetSweepRollupAsync( } var now = _utcNow(); - if (_lastSweepRollup.TryGetValue(FleetSweepRollupKey, out var lastSent) - && now - lastSent < FleetSweepRollupInterval) + if (await DocumentDeliveredInsideIntervalAsync( + _lastSweepRollup, FleetSweepRollupKey, PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, + FleetSweepRollupInterval, now, "fleet-sweep rollup", cancellationToken)) { return; } @@ -1732,10 +1792,10 @@ internal async Task ApplyFleetSweepRollupAsync( return; } - _lastSweepRollup[FleetSweepRollupKey] = now; var (shortMessage, detail) = RenderFleetSweepRollup(facts, spanStartUtc, spanEndUtc); - await FireAsync( + /* #3580: stamped after the fire, and only on a delivery not reported failed — the digest's rule. */ + var delivery = await FireAsync( StoreKey(FleetSweepRollupKey), _storeLabel, FleetSweepRollupMetric, currentValue: facts.Sweeps.ToString(CultureInfo.InvariantCulture), /* There is no threshold - the digest's exact posture, stated in the string because the NOT NULL @@ -1750,6 +1810,179 @@ column demands a value and "report" is the honest one. */ numericCurrentValue: facts.Sweeps, numericThresholdValue: 0, cancellationToken); + + await RecordDocumentDeliveredAsync( + _lastSweepRollup, FleetSweepRollupKey, PgSelfAlertDeliveryStampStore.FleetSweepRollupStateKey, + delivery, now, "fleet-sweep rollup", cancellationToken); + } + + /* ------------------------- #3580: the daily documents' delivered-today gate ------------------------- */ + + /// + /// Whether a daily document was DELIVERED inside its interval as of — the gate + /// both documents' evaluate and apply halves consult (#3580). The store SEEDS process memory after a + /// start; memory serves the process; every delivery writes both — the shape #981 gave the email + /// cooldown (IAlertHistoryStore.GetLastEmailSentUtcAsync seeds it across restart), applied to + /// the one gate that was still memory-only. + /// + /// Memory answers whenever it holds anything. 23 of every 24 hourly ticks fall inside the + /// interval of a delivery this process already knows about, and those ticks cost one dictionary lookup + /// and no store round-trip — the cost promise the documents' callers make. Memory is also allowed to + /// answer "outside the interval, deliver" without re-asking the store, because there is ONE writer per + /// store and it is this process: once memory is seeded the store can never hold a newer stamp than + /// memory does, so a second read could only return what memory already knows. That is also why the + /// evaluate half's pre-check and the apply half's own check cost one store read between them and not + /// two — the first seeds, the second finds memory populated. + /// + /// The stamp store is asked ONCE per document per process — on the first tick, when memory is + /// empty — and whatever it answers is cached, including "nothing": a stamp inside the interval + /// gates; a stamp outside it lets the document proceed to its reads and stays cached, so a subsequent + /// failed delivery leaves memory pointing at the last REAL delivery rather than at nothing; no row, or + /// a read that failed, caches , which reads as "deliver" from then on. + /// Caching the empty answer is exact under the single-writer fact above — a store that had no row for + /// this process's first tick cannot gain one except through this process, which would populate memory + /// directly — and it is what keeps the cost model honest in the fault case as well as the happy one: the + /// evaluate half's pre-check asks, and the apply half's own check, seconds later on the same tick, + /// finds memory populated whether the store answered, was empty, or threw. Re-asking on a fault would + /// log the same failure twice and count it twice in the #3013 census on every tick the fault persisted, + /// for one logical failure. The retry the install night's recovery case needs is unaffected: a document + /// that has never delivered reads "deliver" from memory on every tick until a delivery lands. + /// + /// A stamp read that fails falls OPEN to memory, warns, and is counted. Fail-open toward + /// delivering is the direction every store-fault posture in this evaluator already takes for its + /// documents — the rollup's read fault "skips the tick WITHOUT consuming the interval" so an unreadable + /// store cannot become a permanently quiet channel; the digest's does the same — and it is bounded: the + /// memory gate still holds within the process once one delivery lands, so a store that cannot answer + /// costs at most one extra copy per process, which is exactly the pre-#3580 posture and not a spam + /// path. Failing CLOSED (skip when the stamp cannot be read) would let a store hiccup silence a daily + /// document, the worse failure. Counted into #3013's census because it is a store read the alert pass + /// performed, failed and swallowed, and the census exists so that population is not invisible; the + /// warning beside it names the document that could not ask. Counted ONCE per process, per the + /// paragraph above — the census measures faults the pass met, and this pass meets this one once. + /// + /// No store configured ( null) is the pre-#3580 gate exactly: memory + /// only. + /// + private async Task DocumentDeliveredInsideIntervalAsync( + ConcurrentDictionary lastDelivered, string memoryKey, string stampKey, + TimeSpan interval, DateTime now, string documentName, CancellationToken cancellationToken) + { + if (lastDelivered.TryGetValue(memoryKey, out var known)) + { + return known != NoDeliveryKnown && now - known < interval; + } + + if (_deliveryStamps is null) + { + return false; + } + + DateTime? stamped; + var stampClock = Stopwatch.StartNew(); + try + { + stamped = await _deliveryStamps.GetDeliveredAtUtcAsync(stampKey, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + /* One read name for both documents, because the #3013 census keys a counted site on a LITERAL + name with its own clock and this is one site serving two callers; the log line beside it + names the document, so the actionable half is not lost — it is a line away. The sentinel + is what makes this warning and this count fire once per process rather than once per check: + the apply half's own gate, seconds from now, finds memory populated and does not re-ask. */ + _logger?.LogWarning(ex, + "{Document} delivery stamp could not be read after {ElapsedMs} ms; gating on process memory from here, which re-announces once per restart until the store answers", + documentName, stampClock.ElapsedMilliseconds); + _readFailures?.RecordReadFailure(null, "daily-document delivery-stamp self-alert", stampClock.ElapsedMilliseconds); + lastDelivered[memoryKey] = NoDeliveryKnown; + return false; + } + + if (stamped is not DateTime deliveredAt) + { + lastDelivered[memoryKey] = NoDeliveryKnown; + return false; + } + + lastDelivered[memoryKey] = deliveredAt; + return now - deliveredAt < interval; + } + + /// + /// What caches when the store was asked and had no + /// answer — no row, or a read that threw — so the store is asked once per document per process and + /// never re-asked on the same tick by the apply half (#3580). Reads as "deliver": the gate compares it + /// by identity before the interval arithmetic, so it can never be mistaken for a real stamp, and the + /// first delivery that lands replaces it with a real one. rather than a + /// nullable value because the dictionaries are the pre-#3580 shape and every sibling gate in this file + /// keys on presence; a value that means "asked, nothing known" keeps presence meaning "asked". + /// + private static readonly DateTime NoDeliveryKnown = DateTime.MinValue; + + /// + /// Records that a daily document was DELIVERED at — into process memory and, + /// when a store is configured, into the delivery stamp — unless the deliverer reported the send + /// (#3580). + /// + /// "Failed" is the one disposition that writes nothing, and the rule is stated as that one + /// exclusion on purpose. It is the disposition the install night's recovery case wore: a channel + /// was attempted and came back unsuccessful, so nothing reached a reader, and the next tick — restart + /// or not — must retry. Every other answer is either a delivery () or + /// the product's own decision that nothing is owed: (a mute + /// rule chose the silence), (no channel exists to + /// deliver to — the history row IS the delivery, and retrying hourly would write 24 rows a day for + /// nothing), (a copy went out inside the cooldown, so this + /// one is not owed) and (reported on another delivery's + /// roster). A null report — a deliverer that does not report, or one whose outer isolation + /// caught something outside the channels — is "unreported", not "failed", and stamps: that is how + /// every fire before #3580 was treated, and a deliverer that KNOWS a send failed says so. Stating the + /// exclusion rather than an allow-list means a disposition added later defaults to the quiet side; one + /// that means "not delivered and owed" has to be added here by name. + /// + /// The stamp write is failure-isolated and NOT counted — a write, not a condition read, + /// the distinction. Memory is stamped first, so a store that will + /// not take the write still gates this process; the warning says the next restart will re-announce. + /// The instant recorded is the evaluator's , the controllable clock, so a test + /// can place the stamp and the interval compare is against the same clock it was written from. + /// + private async Task RecordDocumentDeliveredAsync( + ConcurrentDictionary lastDelivered, string memoryKey, string stampKey, + AlertDelivery? delivery, DateTime now, string documentName, CancellationToken cancellationToken) + { + if (delivery is { Channel: AlertDelivery.ChannelFailed }) + { + _logger?.LogWarning( + "{Document} delivery failed ({Error}); no delivery stamp written, so the next tick retries it", + documentName, delivery.SendError ?? "no error text"); + return; + } + + lastDelivered[memoryKey] = now; + + if (_deliveryStamps is null) + { + return; + } + + try + { + await _deliveryStamps.RecordDeliveredAtUtcAsync(stampKey, now, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + /* NOT counted by #3013's swallowed-read counter: a stamp WRITE, not a condition read. */ + _logger?.LogWarning(ex, + "{Document} was delivered but its delivery stamp could not be written; process memory gates it until the next restart, which will re-announce it", + documentName); + } } /// One band transition a covered sweep reported: the server by name (the documents carry names @@ -2697,10 +2930,11 @@ private static string DescribeAgDatabaseKey(string key) /* ---------------- store disk pressure (fleet-level, polled) ---------------- */ /// - /// Pure disk-pressure decision: the store volume is under pressure when its FREE space is below - /// of the volume total. No I/O, so it pins directly. A non-positive - /// total is treated as "can't tell" (false — the caller also guards this). The percentage scales across - /// disk sizes; see the constant for why it is percent-only. + /// Pure disk-pressure decision at the SHIPPED defaults: the store volume is under pressure when its + /// FREE space is below of the volume total AND below + /// absolute (#3528 — see the floor constant for the composition). + /// No I/O, so it pins directly. A non-positive total is treated as "can't tell" (false — the caller + /// also guards this). /// is the measurement the alert is ABOUT, handed back so the fire /// site can store it as a real numeric instead of leaving the history store to find it again by /// scanning for digits (#1881). It is computed whenever the total is usable, @@ -2709,12 +2943,16 @@ private static string DescribeAgDatabaseKey(string key) /// the one dangerous ambiguity this metric must never have back into the signature. /// internal static bool IsDiskPressure(long freeBytes, long totalBytes, out string reason, out double percentFree) - => IsDiskPressure(freeBytes, totalBytes, DiskFreeWarnPercent, out reason, out percentFree); + => IsDiskPressure(freeBytes, totalBytes, DiskFreeWarnPercent, DiskFreeWarnFloorGb, out reason, out percentFree); - /// #2107: the configurable form — the sweep passes the store-backed - /// SelfDiskFreeWarnPercent; the constant-threshold overload keeps the shipped default - /// for the tests pinning it. + /// #2107: the percent-only form — floor disabled, kept for the tests that pin the percent + /// edge on its own. The sweep calls the two-knob overload below. internal static bool IsDiskPressure(long freeBytes, long totalBytes, double warnPercent, out string reason, out double percentFree) + => IsDiskPressure(freeBytes, totalBytes, warnPercent, 0.0, out reason, out percentFree); + + /// #3528: the configurable form — the sweep passes the store-backed + /// SelfDiskFreeWarnPercent AND SelfDiskFreeWarnGb (0 = no floor). + internal static bool IsDiskPressure(long freeBytes, long totalBytes, double warnPercent, double floorGb, out string reason, out double percentFree) { if (totalBytes <= 0) { @@ -2724,7 +2962,8 @@ internal static bool IsDiskPressure(long freeBytes, long totalBytes, double warn } percentFree = (double)freeBytes / totalBytes * 100.0; - if (percentFree < warnPercent) + double freeGb = freeBytes / (1024.0 * 1024.0 * 1024.0); + if (percentFree < warnPercent && (floorGb <= 0 || freeGb < floorGb)) { reason = $"The monitor store's disk volume has only {percentFree.ToString("0.#", CultureInfo.InvariantCulture)}% free ({FormatGb(freeBytes)} of {FormatGb(totalBytes)})."; return true; @@ -2791,10 +3030,11 @@ internal async Task ApplyDiskPressureAsync( } var now = _utcNow(); - /* #2107: store-backed threshold (clamped on read); the constant remains only as the - shipped default. */ + /* #2107/#3528: store-backed thresholds (clamped on read); the constants remain only as the + shipped defaults. */ double warnPercent = _settings.SelfDiskFreeWarnPercent; - bool pressure = IsDiskPressure(free, total, warnPercent, out var reason, out var percentFree); + double floorGb = _settings.SelfDiskFreeWarnGb; + bool pressure = IsDiskPressure(free, total, warnPercent, floorGb, out var reason, out var percentFree); if (pressure) { @@ -2827,7 +3067,11 @@ shipped default. */ : ""; await FireAsync( StoreKey(DiskKey), _storeLabel, DiskPressureMetric, reason, - $"{warnPercent.ToString("0.#", CultureInfo.InvariantCulture)}% free", + /* #3528: the threshold string names BOTH gates when the floor is active, so the history + row's threshold column states the condition that actually fired. */ + floorGb > 0 + ? $"{warnPercent.ToString("0.#", CultureInfo.InvariantCulture)}% free and under {floorGb.ToString("0.#", CultureInfo.InvariantCulture)} GB" + : $"{warnPercent.ToString("0.#", CultureInfo.InvariantCulture)}% free", detail: reason + storeText + " When the store volume fills, collection and every write stop " + "for the WHOLE fleet, and a headless service has no dashboard to warn you. Free space on the " + "store volume, shorten retention (config_collector_schedules), enable TimescaleDB compression, " + @@ -3377,8 +3621,18 @@ await FireAsync( /// platform-neutral copy of the loaded certificate's facts so the alert path never touches an X.509 type. /// is false when there is no LAN TLS certificate to watch (loopback-only, no /// tls block, or an unusable one); the other fields are meaningful only when it is true. + /// is the host's own load-time verdict (#3517): it judged + /// still ahead of the clock, refused the certificate, and bound loopback-only + /// — a decision it does not revisit until its next start, which is why it travels as a flag and is never + /// re-derived here from the date. /// - internal sealed record WebTlsCertReport(bool Configured, DateTimeOffset NotAfterUtc, string Subject, string Thumbprint); + internal sealed record WebTlsCertReport( + bool Configured, + DateTimeOffset NotBeforeUtc, + DateTimeOffset NotAfterUtc, + string Subject, + string Thumbprint, + bool RefusedNotYetValid); /// /// The isolating entry point the worker's sweep calls for the web-dashboard TLS certificate expiry @@ -3422,10 +3676,24 @@ public async Task EvaluateWebTlsCertificateAsync(WebTlsCertReport report, Cancel /// certificate fails every TLS handshake, so the LAN dashboard is already unreachable and binds /// loopback-only on the next restart. /// + /// The not-yet-valid arm (#3517). A certificate whose NotBefore was still ahead when + /// the host loaded it — a skewed clock, or a certificate minted for a future rotation — is refused by the + /// host and the dashboard is loopback-only from the start. Its NotAfter is far out, so on the + /// expiry test alone it read as the healthiest certificate in the fleet and the degrade raised nothing + /// but a startup log line. This arm fires the SAME family at CRITICAL — the dashboard is exactly as + /// unreachable as it is when expired — under the same key and metric, so an operator's mute rule and the + /// deliverer's dedup treat it as the one condition it is: "the configured certificate is not being + /// served". It fires on the host's carried verdict, NOT on NotBefore against the clock, because + /// the host does not re-decide when the date passes: it stays loopback-only until it is restarted, and a + /// date-derived arm would have resolved the alert about a dashboard that was still down. + /// /// A STANDING condition like its siblings: fire on entry, re-state per /// while it holds, and ONE resolution when the served certificate is healthy again (renewed past the - /// window) or TLS is no longer configured. Gated on the master alerts switch. Internal so it pins directly - /// with a recording deliverer and a controllable clock. + /// window) or TLS is no longer configured — the not-yet-valid arm shares that resolution: the host + /// re-publishes a usable verdict on its next successful start (a fresh evaluator, so no resolution row is + /// written, the #3514 in-place-renewal finding), or clears the snapshot when the dashboard is stopped + /// (the Configured=false arm, which does resolve). Gated on the master alerts switch. Internal so it + /// pins directly with a recording deliverer and a controllable clock. /// internal async Task ApplyWebTlsCertificateAsync(WebTlsCertReport report, CancellationToken cancellationToken) { @@ -3436,22 +3704,30 @@ internal async Task ApplyWebTlsCertificateAsync(WebTlsCertReport report, Cancell var now = _utcNow(); - /* Healthy is either "no certificate to watch" or "more than the warning window still to run". The - subtraction is DateTime-on-DateTime so it is a pure TimeSpan and never trips the DateTimeOffset(...) - Kind guard on a test-injected clock. */ + /* The host's verdict, not the clock's: see the method summary. Meaningful only when configured. */ + var refusedNotYetValid = report.Configured && report.RefusedNotYetValid; + + /* Healthy is either "no certificate to watch" or "being served, with more than the warning window + still to run". The subtraction is DateTime-on-DateTime so it is a pure TimeSpan and never trips the + DateTimeOffset(...) Kind guard on a test-injected clock. */ var healthy = !report.Configured - || report.NotAfterUtc.UtcDateTime - now > WebTlsCertWarnWindow; + || (!refusedNotYetValid && report.NotAfterUtc.UtcDateTime - now > WebTlsCertWarnWindow); if (healthy) { if (_activeWebTlsCert.TryRemove(WebTlsCertKey, out var was) && was) { _lastWebTlsCertAlert.TryRemove(WebTlsCertKey, out _); + /* The configured-and-healthy line names BOTH facts the family alerts on — served, and outside + the window — because the active alert it clears may have been either arm (#3517): a + dashboard toggled off and on within one supervisor tick re-publishes a now-usable + certificate before the sweep ever sees the null, so this is the line a cured + not-yet-valid refusal resolves with too. */ await RecordResolutionAsync(new AlertResolution( StoreKey(WebTlsCertKey), _storeLabel, WebTlsCertExpiryMetric, WebTlsCertRenewedMetric, report.Configured - ? "The web dashboard's TLS certificate is no longer within the expiry window" + ? "The web dashboard's TLS certificate is being served and is outside the expiry window" : "The web dashboard is no longer serving a TLS certificate to watch"), cancellationToken); } @@ -3472,29 +3748,64 @@ await RecordResolutionAsync(new AlertResolution( _lastWebTlsCertAlert[WebTlsCertKey] = now; var expired = report.NotAfterUtc.UtcDateTime <= now; - var (shortMessage, detail, currentValue) = RenderWebTlsCert(report, now, expired); + var (shortMessage, detail, currentValue) = RenderWebTlsCert(report, now, expired, refusedNotYetValid); await FireAsync( StoreKey(WebTlsCertKey), _storeLabel, WebTlsCertExpiryMetric, currentValue: currentValue, - thresholdValue: $"{Hosting.DarlingWebTls.ExpiryWarningDays} days", + /* The not-yet-valid arm has no window to name — the bar it failed is "valid now". */ + thresholdValue: refusedNotYetValid && !expired + ? "valid at service start" + : $"{Hosting.DarlingWebTls.ExpiryWarningDays} days", detail: detail, - severity: expired ? AlertSeverityLevel.Critical : AlertSeverityLevel.Warning, + /* Critical for BOTH refusals: expired and not-yet-valid leave the LAN dashboard equally unreachable. */ + severity: expired || refusedNotYetValid ? AlertSeverityLevel.Critical : AlertSeverityLevel.Warning, shortMessage: shortMessage, /* State-only: an expiry is a date, not a quantity — see WebTlsCertExpiryMetric. */ numericCurrentValue: StateOnlyValue, numericThresholdValue: StateOnlyValue, cancellationToken); } - /// Renders the (shortMessage, detail, currentValue) for the web TLS certificate expiry alert. The + /// Renders the (shortMessage, detail, currentValue) for the web TLS certificate alert. The /// subject and thumbprint match the web host's own startup log line, so an operator can tie the alert to - /// the certificate it named. Pure but for the caller's clock; pinned by tests. + /// the certificate it named. Pure but for the caller's clock; pinned by tests. + /// + /// Expired outranks not-yet-valid when both hold (a refused-at-start certificate the process then + /// outlived): fixing the clock cannot bring an expired certificate back, so that is the fact to lead + /// with; the not-yet-valid text below is for the case a clock fix or the right certificate plus a restart + /// actually cures. private static (string ShortMessage, string Detail, string CurrentValue) RenderWebTlsCert( - WebTlsCertReport report, DateTime now, bool expired) + WebTlsCertReport report, DateTime now, bool expired, bool refusedNotYetValid) { var notAfter = report.NotAfterUtc.UtcDateTime; var certRef = $"Certificate: subject {report.Subject}, thumbprint {report.Thumbprint}."; + if (refusedNotYetValid && !expired) + { + var notBefore = report.NotBeforeUtc.UtcDateTime; + var currentValue = $"not valid until {notBefore:u}; not being served"; + var shortMessage = + $"web dashboard TLS certificate NOT YET VALID (valid from {notBefore:u}) — LAN dashboard is loopback-only"; + + /* Two tenses, because the operator reads this on the alert channel at some later hour: while the + window is still ahead, the clock is the likely culprit and the date is what to check it + against; once the window has opened, the only thing still wrong is that this process decided + before it did — and it will not re-decide without a restart, which is the one fact that would + otherwise surprise them. */ + var clockLine = now < notBefore + ? $"The window opens {notBefore:u}: if that is in the past by any wall clock you trust, this host's clock " + + "is behind; if it is genuinely ahead, the certificate installed is one issued for a future rotation." + : $"The window opened {notBefore:u}, after the service started — the host judged the certificate once, at load, " + + "and stays loopback-only on that verdict until it is restarted."; + var detail = + $"The web dashboard's configured TLS certificate was not yet valid when the service started (not valid " + + $"until {notBefore:u}), so the host refused to serve it and the LAN dashboard is bound LOOPBACK-ONLY — " + + $"unreachable from the network, and it will not fall back to plain HTTP. {clockLine} Correct the system " + + "clock or install the currently-valid certificate, then restart the service so the host loads it " + + $"again. {certRef}"; + return (shortMessage, detail, currentValue); + } + if (expired) { var agoDays = Math.Max(0, (int)Math.Floor((now - notAfter).TotalDays)); @@ -3854,6 +4165,47 @@ await RecordResolutionAsync(new AlertResolution( /// resolution row is written (the sibling conditions' edge shape). Gated on the master alerts switch. /// Re-arm happens at most ONCE per job per check (only on the first-detection transition). Internal so it /// pins directly with a recording deliverer, a controllable clock, and a fake re-arm delegate. + /// + /// What this machine trusts, and what it cost when the trust was misplaced (#3575). This takes + /// as settled fact: first sight re-arms and pages Critical, absence an hour + /// later posts Recovered. So one false row in the list is not one false message but three — the page, the + /// idempotent re-arm it narrates, and the recovery of a job that was never unwell — on the alert family + /// that reports the store's own health. A production store produced exactly that set from a healthy job: + /// the detector's -infinity arm already guarded on job_status, but TimescaleDB's + /// job_stats view assembles that status from pg_stat_activity and next_start from the + /// job-stat row, and for a few milliseconds at either edge of every run the two disagree in exactly the + /// dead-job shape; the check's sample landed 53 ms into a 63 ms run that succeeded. The fix is upstream + /// of here and deliberately so: TimescaleSupport.ReadStuckCompressionJobsAsync now confirms a + /// -infinity trip with a second read five seconds later before a job reaches this list, and the + /// worker pins its samples to :30 past the minute, off the policies' :MM:00 run instants. + /// This method keeps its single-sample semantics — first sight IS first sight — because the input is now + /// worth that trust, and adding hysteresis here instead would have bought the same protection for an + /// hour of detection latency on a genuinely dead job. + /// + /// Which TimescaleDB the dead-job arm's sentence and its re-arm are true on (#3591). "The + /// scheduler will never run it again" was true of every TimescaleDB below 2.26.4: a persisted + /// next_start = -infinity was returned to the scheduler as the due time and the job was never due + /// again — the state #1581 was built against, and the one the re-arm genuinely rescues. Upstream #9360 + /// ("Sanitize DT_NOBEGIN next_start to recover jobs stuck after primary failover", in 2.26.4 and + /// every 2.27+ release; TimescaleSupport.TimescaleNextStartSanitizedFrom) removed that state. On a + /// fixed store the only PERSISTENT -infinity is a crashed run — a worker killed between its start + /// and end marks by a SIGKILL, a crash-restart or a failover — which the scheduler holds in a CRASH + /// BACKOFF of at least five minutes and, for a compression policy with the default one-hour + /// retry_period, about an hour (±13 % jitter), and then re-runs by itself. Rows of that kind arrive + /// here with set, and this machine treats them + /// differently on the evidence of a 2.28.1 rig: re-arming a job in crash backoff does not shorten the + /// wait, it RESETS it — alter_job refreshes the scheduler's job list and every crash row's backoff + /// is recomputed from that instant (the un-re-armed sibling crash row moved too), and the re-arm overwrites + /// the -infinity so the next hourly read would call the job healthy before it had run. A re-arm on + /// such a row is therefore three untrue messages and a later retry, which is why the first sighting of + /// one is NOT re-armed and NOT paged: it is logged at Information and remembered as + /// AwaitingSchedulerRetry. If the same job is still on the arm an hour later — the scheduler's own + /// retry has not cleared it, because the jittered backoff fell just past the check cadence or because the + /// job crashed AGAIN and its backoff doubled — it escalates: one Critical page naming a crash the operator + /// should read the PostgreSQL log for, no re-arm, and the cooldown re-fires of the escalated state. A job + /// that clears while awaiting the retry is dropped silently, since nothing was paged for it to recover + /// from. Stores below 2.26.4, and stores whose version could not be read, keep every #1581 semantic + /// exactly — an unknown version is treated as old, because on an old store the re-arm is the rescue. /// internal async Task ApplyCompressionJobsStuckAsync( IReadOnlyList stuckJobs, @@ -3879,6 +4231,19 @@ internal async Task ApplyCompressionJobsStuckAsync( if (!_compressionJobState.TryGetValue(key, out var state)) { + if (job.SchedulerRetries) + { + /* #3591: a crash-backoff row on a TimescaleDB that re-runs it by itself. Re-arming would + reset that backoff and erase the evidence (see the method doc); paging would narrate a + self-recovering condition as a rescue. Remember it, say so in the log, and give the + scheduler one check cadence to do what it does. */ + _compressionJobState[key] = CompressionJobHealth.AwaitingSchedulerRetry; + _logger?.LogInformation( + "TimescaleDB {Label} read {Reason}; the scheduler re-runs a crashed job by itself after its crash backoff (at least five minutes, about an hour for a compression policy's default retry period), and a re-arm would reset that backoff rather than shorten it — not re-armed, not alerted; escalates if still there next check (#3591)", + label, job.Reason); + continue; + } + /* First detection this episode: re-arm ONCE, then alert on the outcome. */ bool rearmed = await rearmAsync(job.JobId); _lastCompressionJobAlert[key] = now; @@ -3892,7 +4257,10 @@ await FireAsync( "(alter_job next_start => now). A stuck compression policy halts the store's archival tier, so " + "uncompressed data grows without bound until the disk fills and collection stops for the WHOLE " + "fleet, and a headless service has no dashboard to warn you. If it re-hangs the service will " + - "escalate and stop auto-re-arming — investigate the TimescaleDB background-worker health.", + "escalate and stop auto-re-arming — investigate the TimescaleDB background-worker health. " + + "(A next_start of -infinity is permanent only below TimescaleDB 2.26.4; from 2.26.4 on, upstream " + + "#9360, the scheduler recovers it by itself and the service leaves such rows to it — if this store " + + "is on 2.26.4 or later, its extension version could not be read on this pass.)", severity: AlertSeverityLevel.Critical, shortMessage: $"{label} was stuck — auto-re-armed", /* #1881: job.Reason is elapsed minutes when a run HUNG and a scheduler state with no @@ -3920,6 +4288,29 @@ await FireAsync( cancellationToken); } } + else if (state == CompressionJobHealth.AwaitingSchedulerRetry) + { + /* #3591: an hour on and the scheduler's own retry has not cleared it. Either the jittered backoff + landed just past this check, or the job crashed AGAIN and its backoff doubled — both are worth a + human reading the PostgreSQL log, and neither is helped by alter_job (which would reset the + backoff once more). Escalate: page once now, re-fire on the cooldown, never re-arm. */ + _compressionJobState[key] = CompressionJobHealth.Escalated; + _lastCompressionJobAlert[key] = now; + await FireAsync( + StoreKey(CompressionKeyPrefix + key), _storeLabel, CompressionJobMetric, + job.Reason, "running on schedule", + detail: $"TimescaleDB {label} has sat in the scheduler's crash backoff ({job.Reason}) since at least the previous " + + "hourly check, and the scheduler's own retry has not cleared it. A crashed background worker means the " + + "PostgreSQL cluster crash-restarted, failed over, or the worker was killed mid-run — read the PostgreSQL log " + + "around the job's last_run_started_at for the cause, and check whether it has crashed more than once (each " + + "consecutive crash doubles the backoff). The service did NOT re-arm it: on TimescaleDB 2.26.4+ (upstream " + + "#9360) alter_job(next_start => now()) against a job in crash backoff resets the backoff instead of " + + "shortening it. Compression stays paused for this hypertable until the scheduler's retry succeeds.", + severity: AlertSeverityLevel.Critical, + shortMessage: $"{label} still in crash backoff an hour on — escalated", + numericCurrentValue: StateOnlyValue, numericThresholdValue: StateOnlyValue, + cancellationToken); + } else if (state == CompressionJobHealth.ReArmed) { /* Still stuck after last check's self-heal = a RE-HANG. Escalate and STOP re-arming (looping @@ -3967,8 +4358,19 @@ collection under enumeration. */ continue; } - _compressionJobState.TryRemove(key, out _); + _compressionJobState.TryRemove(key, out var was); _lastCompressionJobAlert.TryRemove(key, out _); + if (was == CompressionJobHealth.AwaitingSchedulerRetry) + { + /* #3591: the scheduler's own retry cleared it and nothing was paged, so there is nothing to + resolve — a lone "Recovered" with no preceding alert would be the very message shape #3575 + removed. The log carries the outcome instead. */ + _logger?.LogInformation( + "TimescaleDB compression job {JobId} is running on schedule again — the scheduler's own retry cleared its crash backoff; nothing was re-armed or alerted (#3591)", + key); + continue; + } + await RecordResolutionAsync(new AlertResolution( StoreKey(CompressionKeyPrefix + key), _storeLabel, CompressionJobMetric, "Compression Job Recovered", @@ -4073,33 +4475,69 @@ AND status IN ('SUCCESS', 'SKIPPED')) A return (lastSuccess, recentRuns, recentSuccess); } + /// The shipped capture-down statement, a constant so the #3597 pins and the gated plan test read + /// the text the method executes rather than a copy of it. + internal const string MissingCaptureSessionsSql = @" +SELECT x.collector_name +FROM +( + (SELECT cl.collector_name, cl.status + FROM collection_log AS cl + WHERE cl.server_id = $1 + AND cl.collector_name = 'deadlocks' + ORDER BY cl.collection_time DESC + LIMIT 1) + UNION ALL + (SELECT cl.collector_name, cl.status + FROM collection_log AS cl + WHERE cl.server_id = $1 + AND cl.collector_name = 'blocked_process_report' + ORDER BY cl.collection_time DESC + LIMIT 1) +) AS x +WHERE x.status = 'SESSION_MISSING' +ORDER BY x.collector_name"; + /// /// The blocking/deadlock XE collectors whose LATEST run logged SESSION_MISSING — the session is /// absent and couldn't be created, so capture is non-functional even though the tolerant reader "succeeds" /// with zero rows. The Darling twin of the Dashboard's GetMissingCaptureSessionsAsync, on Darling's /// collector names. Returns the friendly capture names ("Blocking" / "Deadlock"). + /// + /// #3597: one chunk-orderable LIMIT 1 per collector, not a window function over the + /// server's whole history. This read shipped as ROW_NUMBER() OVER (PARTITION BY collector_name + /// ORDER BY log_id DESC) over every collection_log row the server had for the two collectors — + /// the #3496 shape, which that fix named and deliberately left: a window function cannot early-stop by + /// reordering alone. collection_log is a hypertable partitioned on collection_time with no + /// index on log_id, so the window had exactly one legal plan: decompress EVERY chunk in the + /// retention horizon for the server, Merge Append them, and number 100 K rows to keep two. Measured on a + /// rig with 60 days of one server's log across 61 chunks (59 compressed): 9,573 buffers and every chunk + /// executed, per alert pass, every 30 seconds, per server — the largest read on the pass by two to three + /// orders of magnitude, and one of the issue's four sites that died at the 10 s deadline while the + /// interval-hourly refresh starved the store for I/O. The same question asked per collector as + /// ORDER BY collection_time DESC LIMIT 1 lets ChunkAppend order the chunks newest-first and stop + /// at the first row: 14 buffers, the newest chunk only, 120 of 122 chunk scans never executed — and the + /// property is horizon-independent, so a longer retention cannot regress it. Two literal arms rather than + /// a LATERAL over a VALUES list because the collector names are a closed set the alert owns, and a plan + /// with a literal predicate is the one the gated test can pin. + /// + /// Why no log_id tiebreak here, when #3496 kept one. The recent-N read orders across ALL + /// of a server's collectors, where many rows share a collection instant and the id decides among them. + /// Each arm here is ONE collector on ONE server, and a collector logs one row per run, stamped + /// DateTime.UtcNow at log time (DarlingObservability.LogCollectionAsync) — two runs of the + /// same collector on the same server cannot share a microsecond, so there is nothing for a tiebreak to + /// decide. What it would COST is measured: with , log_id DESC appended, the index on + /// (server_id, collection_time) cannot serve the second key, and each arm top-N-heapsorts the + /// server's whole newest chunk (1,158 buffers) instead of stopping at its first hit (5). /// + internal static async Task> ReadMissingCaptureSessionsAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken) { var missing = new List(); await using var connection = await postgres.OpenConnectionAsync(cancellationToken); - using var command = new NpgsqlCommand(@" -SELECT x.collector_name -FROM -( - SELECT - cl.collector_name, - cl.status, - ROW_NUMBER() OVER (PARTITION BY cl.collector_name ORDER BY cl.log_id DESC) AS n - FROM collection_log AS cl - WHERE cl.server_id = $1 - AND cl.collector_name IN ('deadlocks', 'blocked_process_report') -) AS x -WHERE x.n = 1 -AND x.status = 'SESSION_MISSING' -ORDER BY x.collector_name", connection) { CommandTimeout = DarlingAlertReadAdapter.AlertPassCommandTimeoutSeconds }; + using var command = new NpgsqlCommand(MissingCaptureSessionsSql, connection) { CommandTimeout = DarlingAlertReadAdapter.AlertPassCommandTimeoutSeconds }; command.Parameters.AddWithValue(serverId); await using var reader = await command.ExecuteReaderAsync(cancellationToken); @@ -4345,10 +4783,14 @@ FROM ag_database_replica_states /// own subject. That condition scans the rules it already holds for one that NAMES it and passes the /// verdict in here instead (#3348). Nothing else may pass this: a caller that hands in a decision it did /// not derive from an explicit naming has re-introduced the self-suppression the seam cannot see. + /// What the deliverer reported the channels did (#3580), or null when it reported + /// nothing — read by the two daily documents' stamps and ignored by every condition-class caller, + /// whose lifecycle is edge-driven and owes nothing to a failed send. See + /// for why the report rides a second method. /* The optional context TRAILS the cancellation token so the dozens of existing positional call sites stay untouched — only the callers that have discrete facts to carry (#2109: the AG database alerts) name it. Same for muted, which defaults to asking the seam like its siblings. */ - private async Task FireAsync( + private async Task FireAsync( string serverKey, string serverName, string metricName, string currentValue, string thresholdValue, string detail, AlertSeverityLevel? severity, string shortMessage, double? numericCurrentValue, double? numericThresholdValue, CancellationToken cancellationToken, @@ -4373,7 +4815,7 @@ so the service log showed "… Recovered" with nothing before it — which reads AlertFiringLog.Fired( serverName, metricName, severity?.ToString() ?? "Warning", shortMessage, isMuted)); - await _deliverer.DeliverAsync(new AlertOutcome( + return await _deliverer.DeliverAndReportAsync(new AlertOutcome( serverKey, serverName, metricName, currentValue, thresholdValue, Context: context, DetailText: detail, NumericCurrentValue: numericCurrentValue, NumericThresholdValue: numericThresholdValue, diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs index fdb6a6a85..c2fc154de 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingServerConnector.cs @@ -308,6 +308,40 @@ private static async Task ProbeAuroraAsync( } } + /// + /// Whether the pg_wait_sampling extension is created in the database this connection landed in + /// (#3604) — the connect-time fact that picks PgWaitSamplingCollector's arm on a stock target: + /// the extension's own 10 ms profile when present, the service-side pg_stat_activity sampler when + /// not. pg_extension is per database, and the collector reads pg_wait_sampling_profile from + /// exactly this database, so this is the fact that decides whether that read can succeed rather than a + /// proxy for it (pg_available_extensions would say "installable", which is a different question). + /// + public const string PostgresWaitSamplingProbeQueryText = + @"SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_wait_sampling')"; + + /// + /// Runs . Fails CLOSED to false on any error other than + /// cancellation, and for the same reason does: this probe decides which + /// arm an optional collector takes, and a target that answered every other connect question must still be + /// monitored. False routes the target to the sampler arm, which reads only pg_stat_activity — the + /// floor — so the failure direction produces a coarser wait history rather than none. + /// + private static async Task ProbeWaitSamplingExtensionAsync( + NpgsqlConnection connection, CancellationToken cancellationToken, ILogger? logger = null) + { + try + { + using var command = new NpgsqlCommand(PostgresWaitSamplingProbeQueryText, connection) { CommandTimeout = 15 }; + var present = await command.ExecuteScalarAsync(cancellationToken); + return present is bool b && b; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogDebug(ex, "pg_wait_sampling extension probe did not succeed; treating the extension as absent (#3604)"); + return false; + } + } + /// Connects, probes, and returns the runtime state for one configured server. public static async Task ConnectAsync(MonitoredServer config, ILogger? logger, CancellationToken cancellationToken) { @@ -406,9 +440,15 @@ private static async Task ConnectPostgresAsync( var isAurora = await ProbeAuroraAsync(connection, cancellationToken, logger); + /* #3604: the wait-tier decision's second fact. Probed AFTER Aurora-ness and independently of it — + an Aurora cluster cannot load the module, so the answer there is false by construction, but the + gate reads the two facts separately and the sweep varies them separately. */ + var hasWaitSampling = await ProbeWaitSamplingExtensionAsync(connection, cancellationToken, logger); + logger?.LogInformation( - "Connected to PostgreSQL target '{Server}': major {Major} (server_version_num {Num}), {Role}, Aurora: {Aurora} — {VersionText}", + "Connected to PostgreSQL target '{Server}': major {Major} (server_version_num {Num}), {Role}, Aurora: {Aurora}, pg_wait_sampling: {WaitSampling} — {VersionText}", config.DisplayName, majorVersion, versionNum, isInRecovery ? "reader (in recovery)" : "writer", isAurora, + hasWaitSampling ? "present (extension_sampled tier)" : isAurora ? "n/a (engine_cumulative tier)" : "absent (service_sampled tier)", versionText); /* A Postgres target reached through the SQL Server path would have failed on the detection @@ -434,6 +474,8 @@ read and the failure names a grant that would never have helped. Aurora was unaf IsAwsRds = RdsEndpoint.TryParse( new NpgsqlConnectionStringBuilder(connectionString).Host) is not null, IsInRecovery = isInRecovery, + /* #3604: which stock-PostgreSQL wait instrument this target gets, decided here once. */ + HasPgWaitSamplingExtension = hasWaitSampling, }, StorageName = storageName, ServerId = config.ServerId, @@ -478,7 +520,8 @@ succeeded against a different engine. */ PostgresMajorVersion: runtime.Target.PostgresMajorVersion, PostgresVersionNum: runtime.Target.PostgresVersionNum, IsAurora: runtime.Target.IsAurora, - IsInRecovery: runtime.Target.IsInRecovery); + IsInRecovery: runtime.Target.IsInRecovery, + HasPgWaitSamplingExtension: runtime.Target.HasPgWaitSamplingExtension); } catch (OperationCanceledException) { @@ -588,7 +631,10 @@ public sealed record ConnectionProbeResult( /* #2280: the database the connection ACTUALLY reached, so a registration-time collision check can compare what the SERVER says against what other registrations claim, rather than comparing two claims. Trailing and defaulted, so every existing construction of this record still compiles unchanged. */ - string? ConnectedDatabase = null) + string? ConnectedDatabase = null, + /* #3604: the wait-tier fact, so --test-connection's "which collectors would run" count is computed + against the same shape the gate reads. Trailing and defaulted for the #2280 reason. */ + bool HasPgWaitSamplingExtension = false) { /// /// Rebuilds the gate's-eye view of this target, so a caller can ask which collectors would actually @@ -607,5 +653,6 @@ public sealed record ConnectionProbeResult( PostgresVersionNum = PostgresVersionNum, IsAurora = IsAurora, IsInRecovery = IsInRecovery, + HasPgWaitSamplingExtension = HasPgWaitSamplingExtension, }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs index a80df9809..7da0ec2a5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs @@ -263,11 +263,12 @@ firing and the click is still visible rather than looking deleted. The alert his /* #3514: the web-dashboard TLS certificate expiry alert is config/host-shaped like the stale-mute one — get_store_metrics answers nothing about it, and renewing the certificate is an out-of-band - step on the service host. The actionable facts (subject, thumbprint, expiry) are in the alert - detail; the history is the firing trail, so the operator can see when the warning began. */ + step on the service host. The actionable facts (subject, thumbprint, expiry — or, for the #3517 + not-yet-valid arm of the same metric, the date the window opens) are in the alert detail; the + history is the firing trail, so the operator can see when the warning began. */ [DarlingSelfAlertEvaluator.WebTlsCertExpiryMetric] = new[] { - F("Recent alerts (the certificate's subject, thumbprint and expiry are in the alert detail)", "get_alert_history", ("hours", "168"), ("limit", "50")), + F("Recent alerts (the certificate's subject, thumbprint and validity dates are in the alert detail)", "get_alert_history", ("hours", "168"), ("limit", "50")), }, /* PostgreSQL alert family (PostgresAlertEvaluator). */ diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs index af9c8e635..28915ad3a 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs @@ -1869,7 +1869,7 @@ private static CatalogRead R(string category, string description, params Catalog ["get_waiting_tasks"] = R(CatSessions, "Tasks currently waiting, with wait type and duration.", PServer(), PHours(1), PLimit(30), PAsOf()), /* ── alerts / mute rules (DarlingMcpAlertTools) ── */ - ["get_alert_history"] = R(CatAlerts, "Recent fired-alert history for a server.", PServer(), PHours(24), PLimit(50), PAsOf()), + ["get_alert_history"] = R(CatAlerts, "Recent fired-alert history for a server, newest first and bounded by limit. Excludes operator-dismissed alerts unless include_dismissed is true (dismissed_excluded_count says how many the default hid).", PServer(), PHours(24), PLimit(50), PAsOf(), PBool("include_dismissed", false)), ["get_alert_settings"] = R(CatAlerts, "The current alert-settings configuration."), ["get_mute_rules"] = R(CatAlerts, "The alert mute rules (enabled-only by default).", PBool("enabled_only", true)), @@ -1916,6 +1916,7 @@ private static CatalogRead R(string category, string description, params Catalog ["get_pg_top_queries"] = R(CatData, "Top PostgreSQL query shapes by total execution time (Aurora targets).", PServer(), PHours(24), PLimit(20), PAsOf()), ["get_pg_plans"] = R(CatData, "Captured PostgreSQL execution plans, grouped by shape. Plans are redacted at collection.", PServer(), PHours(24), PLimit(10), PText("query_id"), PAsOf()), ["get_pg_plan_capture_readiness"] = R(CatData, "Whether a PostgreSQL target can capture execution plans at all, facet by facet, with the remedy for each step that is not in place. Read this when a plan or target-log read is empty.", PServer(), PHours(24), PLimit(25), PAsOf()), + ["get_pg_logging_audit"] = R(CatData, "Whether a PostgreSQL target's logging settings (log_min_duration_statement, log_lock_waits, log_temp_files, log_autovacuum_min_duration, log_checkpoints, log_connections, log_disconnections) are producing the lines they could, per setting: verdict, what it unlocks, the recommended value with its cost, and the remedy in the hosting flavour's syntax. Judged from the newest stored pg_server_config snapshot, not the live server; plan capture's own settings are listed and pointed at get_pg_plan_capture_readiness.", PServer()), ["get_pg_wraparound_risk"] = R(CatData, "PostgreSQL XID/MultiXact freeze headroom per database.", PServer(), PHours(24), PAsOf()), ["get_pg_xmin_horizon"] = R(CatData, "What is holding back the PostgreSQL xmin horizon, by cause.", PServer(), PHours(24), PAsOf()), ["get_pg_replication_slots"] = R(CatData, "PostgreSQL replication slot health, including whether retained WAL is still growing.", PServer(), PHours(24), PAsOf()), @@ -1990,7 +1991,7 @@ private static CatalogRead R(string category, string description, params Catalog ["get_table_index_sizes"] = R(CatObjects, "Per-table/index size breakdown.", PServer()), /* ── plan cache / scheduler (DarlingMcpPlanCacheSchedulerTools) ── */ - ["get_cpu_scheduler_pressure"] = R(CatPlanCache, "CPU scheduler pressure indicators.", PServer()), + ["get_cpu_scheduler_pressure"] = R(CatPlanCache, "CPU scheduler pressure indicators from the newest snapshot within the window.", PServer(), PHours(24), PAsOf()), ["get_plan_cache_bloat"] = R(CatPlanCache, "Plan-cache bloat / single-use plan indicators.", PServer(), PHours(24), PAsOf()), /* ── jobs (DarlingMcpJobTools) ── */ @@ -2551,7 +2552,7 @@ internal static IReadOnlyDictionary BuildReadDispatch(I ["get_waiting_tasks"] = (c, pg, an) => DarlingMcpSessionTools.GetWaitingTasks(pg, Server(c), Hours(c, 1), Rows(c, "limit", 30), as_of: AsOf(c)), /* ── alerts / mute rules ── */ - ["get_alert_history"] = (c, pg, an) => DarlingMcpAlertTools.GetAlertHistory(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), as_of: AsOf(c)), + ["get_alert_history"] = (c, pg, an) => DarlingMcpAlertTools.GetAlertHistory(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), as_of: AsOf(c), include_dismissed: QueryBool(c, "include_dismissed", false)), ["get_alert_settings"] = (c, pg, an) => DarlingMcpAlertTools.GetAlertSettings(pg), ["get_mute_rules"] = (c, pg, an) => DarlingMcpAlertTools.GetMuteRules(pg, QueryBool(c, "enabled_only", true)), @@ -2615,6 +2616,7 @@ logger is the tool's logger seat — the web host's SERVICE logger when MapAll b cannot parse exactly rather than silently matching nothing. */ ["get_pg_plans"] = (c, pg, an) => DarlingMcpPgPlanTools.GetPgPlans(pg, Server(c), Hours(c, 24), Rows(c, "limit", 10), Str(c, "query_id"), AsOf(c)), ["get_pg_plan_capture_readiness"] = (c, pg, an) => DarlingMcpPgPlanTools.GetPgPlanCaptureReadiness(pg, Server(c), Hours(c, 24), Rows(c, "limit", 25), as_of: AsOf(c)), + ["get_pg_logging_audit"] = (c, pg, an) => DarlingMcpPgLoggingAuditTools.GetPgLoggingAudit(pg, Server(c)), ["get_pg_wraparound_risk"] = (c, pg, an) => DarlingMcpPgWraparoundTools.GetPgWraparoundRisk(pg, Server(c), Hours(c, 24), as_of: AsOf(c)), ["get_pg_xmin_horizon"] = (c, pg, an) => DarlingMcpPgXminTools.GetPgXminHorizon(pg, Server(c), Hours(c, 24), as_of: AsOf(c)), ["get_pg_replication_slots"] = (c, pg, an) => DarlingMcpPgSlotTools.GetPgReplicationSlots(pg, Server(c), Hours(c, 24), as_of: AsOf(c)), @@ -2696,7 +2698,7 @@ cannot parse exactly rather than silently matching nothing. */ ["get_table_index_sizes"] = (c, pg, an) => DarlingMcpObjectStatsTools.GetTableIndexSizes(pg, Server(c)), /* ── plan cache / scheduler ── */ - ["get_cpu_scheduler_pressure"] = (c, pg, an) => DarlingMcpPlanCacheSchedulerTools.GetCpuSchedulerPressure(pg, Server(c)), + ["get_cpu_scheduler_pressure"] = (c, pg, an) => DarlingMcpPlanCacheSchedulerTools.GetCpuSchedulerPressure(pg, Server(c), Hours(c, 24), as_of: AsOf(c)), ["get_plan_cache_bloat"] = (c, pg, an) => DarlingMcpPlanCacheSchedulerTools.GetPlanCacheBloat(pg, Server(c), Hours(c, 24), as_of: AsOf(c)), /* ── jobs ── */ diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs index 8ae40f4fb..2c167f5c7 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs @@ -114,7 +114,14 @@ so matching its #3304 neighbour costs nothing. */ /* The compression-job self-heal check's cadence (fleet-level, #1581). Compression is a slow archival tier and a stuck policy job takes hours to matter, so hourly is ample and cheap (one job_stats read + at most - one alter_job per stuck job) — no need for the 15s sweep or the 30s alert cadence. */ + one alter_job per stuck job) — no need for the 15s sweep or the 30s alert cadence. + + The PHASE of that hour is not this constant's to choose and is not chosen by "UtcNow + interval" any + more (#3575): the compression policies this check watches fire at :MM:00 of the wall clock on a fixed + schedule (#3035), and a check scheduled from the instant of its previous fire slips a few seconds + every hour and eventually samples one of those :00 instants — which is where a production store's + false page came from. TimescaleSupport.NextCompressionCheckUtc snaps each due time to :30 past its + minute, so this interval sets how OFTEN and that phase sets WHEN in the minute. */ private static readonly TimeSpan s_compressionCheckInterval = TimeSpan.FromHours(1); /* The store self-metrics sweep's cadence (fleet-level, #2068). Store growth is a slow signal — the @@ -475,8 +482,12 @@ a best-effort errand and there is nothing in it that a later hour cannot do. */ private Task? _oversizedPlanSweep; /* MinValue = the first sweep after startup evaluates the compression-job self-heal check (#1581), then - every s_compressionCheckInterval. Fleet-level (one shared store), so it is a single field, not - per-server; only consulted when _timescaleAvailable. */ + every s_compressionCheckInterval, pinned to :30 past the minute by TimescaleSupport.NextCompressionCheckUtc + (#3575) so no steady-state sample lands on the :MM:00 instant the compression policies fire on. The + first sample is deliberately left unpinned — a restart is when an operator is reading the log and wants + the store's job health now — and the confirm-read inside ReadStuckCompressionJobsAsync covers it like + every other sample. Fleet-level (one shared store), so it is a single field, not per-server; only + consulted when _timescaleAvailable. */ private DateTime _nextCompressionCheckUtc = DateTime.MinValue; /* MinValue = the first loop pass after startup runs the fleet sweep (#3466 lane 2), then on the @@ -519,6 +530,11 @@ branches the retention purge onto drop_chunks. */ self-alerts inherit its delivery/cooldown/restart-replay. Held as a field because the connection edge fires from TryConnectAsync and the reconcile drops per-server state through it. */ private DarlingSelfAlertEvaluator? _selfAlerts; + /* The delta calculator, held for the same reason as _selfAlerts (#3540 A4): the reconcile-remove branch + drops a departing server's baselines and pass window through it, so a server removed and re-added + inside the gap policy's hour cannot subtract the new identity's counters from the old one's. Built + and seeded once in RunCollectionLoopAsync, ahead of the runner that shares it. */ + private CollectorDeltaCalculator? _deltas; /* Concrete rather than IAlertDeliverer: there is exactly one implementation here and it is constructed a few lines from where this is assigned, so the interface bought an indirection per delivered alert and no seam (CA1859). */ @@ -1203,18 +1219,21 @@ runnable commands so an elevated human can finish the job the service cannot — /// /// Maps the web host's published TLS-certificate snapshot to the report the evaluator consumes (#3514): /// a null snapshot — nothing served, or Clear()ed when the dashboard stopped — becomes - /// Configured=false (the evaluator's resolve arm), and a live snapshot carries its expiry and - /// identity through. Pure + static so the null-to-unconfigured seam pins in a unit test rather than only - /// through the sweep loop — the precedent, and the seam the #3514 - /// review flagged as previously tested only from the sides. + /// Configured=false (the evaluator's resolve arm), and a live snapshot carries its validity window, + /// identity and the host's not-yet-valid verdict (#3517) through unchanged — the verdict is the host's to + /// make and this mapping must not re-derive or drop it. Pure + static so the null-to-unconfigured seam + /// pins in a unit test rather than only through the sweep loop — the + /// precedent, and the seam the #3514 review flagged as previously tested only from the sides. /// internal static DarlingSelfAlertEvaluator.WebTlsCertReport BuildWebTlsCertReport( WebTlsCertificateState.Snapshot? snapshot) => new( Configured: snapshot is not null, + NotBeforeUtc: snapshot?.NotBeforeUtc ?? default, NotAfterUtc: snapshot?.NotAfterUtc ?? default, Subject: snapshot?.Subject ?? string.Empty, - Thumbprint: snapshot?.Thumbprint ?? string.Empty); + Thumbprint: snapshot?.Thumbprint ?? string.Empty, + RefusedNotYetValid: snapshot?.RefusedNotYetValid ?? false); /// /// Everything after the (optional) managed-Postgres bootstrap: store connection, migration, @@ -1392,6 +1411,26 @@ overlaps with an existing continuous aggregate policy". So on any store that eve await TimescaleSupport.EnsureContinuousAggregatesAsync(timescaleConnection, _logger, stoppingToken); + /* #3597: AFTER the aggregates exist and BEFORE compression, take the eleven per-column group + indexes off the interval-dedup materialization on any store whose aggregate predates + create_group_indexes = false on its CREATE. Nothing reads them, and every hourly refresh paid + twelve index inserts per re-materialized row for them — measured at 4.3x the WAL per bucket. + Before compression so the nightly pass compresses a relation already without them. Its own + catalog read, its own per-index isolation under a lock timeout that yields to a refresh in + flight, its own summary line; a no-op on every start after the first. */ + await TimescaleSupport.EnsureIntervalDedupMaterializationIndexesAsync(timescaleConnection, _logger, stoppingToken); + + /* #3581: AFTER the aggregates exist, put their materializations on the compression ladder the raw + tier has been on since the archival tier existed — none of the twenty ever was, and on the + largest store they were 235 GiB of a 415 GiB database, larger than the 9x-compressed raw they + roll up. Enables compression per materialization, attaches a once-a-day policy per aggregate + on a daily band off the hourly phase grid (one aggregate per hour at :35Z), and stages the + first runs one aggregate per night, largest first, so the backlog pass on an existing store + is one bounded relation a night. The raw compression converge above skips these jobs by name + or it would retune them to the hourly tick. Idempotent under the catalog, failure-isolated + per aggregate, and a no-op on every start after the first. */ + await TimescaleSupport.EnsureAggregateCompressionAsync(timescaleConnection, _logger, stoppingToken); + // AFTER the CAGGs exist: the tiered retention (raw 4d, hourly HISTORY CAGGs 90d per #1937, daily // history kept indefinitely; the interval-dedup and baseline tiers carry their own, #1958). await TimescaleSupport.EnsureRetentionPoliciesAsync(timescaleConnection, _logger, stoppingToken); @@ -1473,6 +1512,7 @@ baseline relations exist as CAGGs on TimescaleDB stores and as plain fallback vi of zeroes. A seed failure logs a warning and collection proceeds with first-cycle-zero. */ var deltas = new DarlingDeltaCalculator(); await deltas.SeedFromStoreAsync(postgres, _logger, stoppingToken); + _deltas = deltas; /* Control-plane Stage 1: SEED the config store from darling.json once (idempotent; only empty sections), then read the store view and make it authoritative — the held DarlingConfig is @@ -1673,7 +1713,11 @@ halves of a server's alert work. */ read would claim a hot-reload the config cannot deliver. Null/blank means the evaluator fires under the shipped "Monitor Store" constant, byte-identical to every release before the field existed. */ - storeName: config.Peers?.StoreName); + storeName: config.Peers?.StoreName, + /* #3580: the two daily documents' delivered-today stamps, in the store's own key/value state + table, so a restart of this process does not re-announce a digest or rollup the previous + process delivered an hour ago — and does re-attempt one whose delivery failed. */ + deliveryStamps: new PgSelfAlertDeliveryStampStore(postgres, _logger)); /* #1706: report this start's store runtime upgrade, now that there IS an alert engine to report it through. Fired once, here, and never re-evaluated — the store is down while an upgrade runs, so @@ -2142,7 +2186,9 @@ MuteRuleService cache the engine matches against — so it sees exactly what is /* #3514: the web-dashboard TLS certificate expiry self-alert. The web host loads the certificate once at start and publishes its served expiry to WebTlsCertificateState; a headless service can run for months without a restart, so the worker re-evaluates that fixed expiry against the clock - on its own slow cadence and the evaluator warns 30 days out / Critical once lapsed. A null + on its own slow cadence and the evaluator warns 30 days out / Critical once lapsed — and Critical + at once when the snapshot carries the host's not-yet-valid refusal (#3517: the dashboard is + loopback-only from the start, and the host does not re-decide when the date passes). A null snapshot means no LAN TLS certificate to watch. Fleet-level, and the Evaluate* wrapper is failure-isolated so a throw never stops the fleet loop. */ if (_selfAlerts is not null && DateTime.UtcNow >= _nextWebTlsCheckUtc) @@ -2155,10 +2201,21 @@ await _selfAlerts.EvaluateWebTlsCertificateAsync( /* #1581: the compression-job self-heal backstop. TimescaleDB compression policy jobs can silently die (next_start = -infinity) or hang, halting the store's archival tier so uncompressed data grows without bound until the disk fills and collection stops for the WHOLE fleet (the field incident). - Timescale-only; own hourly cadence; failure-isolated inside EvaluateCompressionJobHealthAsync. */ + Timescale-only; own hourly cadence; failure-isolated inside EvaluateCompressionJobHealthAsync. + + The next due time is SNAPPED to the wall clock rather than taken from this fire (#3575). The + dead-job arm the check judges reads next_start = -infinity, which is also what the scheduler + writes for the few milliseconds at either edge of every healthy run before the worker is, or + after it stops being, visible as Running — and the policies run at :MM:00 on a fixed schedule, + so a check that re-anchored itself as "UtcNow + 1 h" on every fire crept a few seconds per hour + across those instants until, on a production store, it sampled one 53 ms into a 63 ms run and + paged. NextCompressionCheckUtc puts every steady-state sample at :30 past its minute instead, + half the grid step from every policy's start in both directions. The read itself now confirms + a -infinity trip with a second read five seconds later (ReadStuckCompressionJobsAsync), so the + phase is hardening on top of the fix, not the fix. */ if (_timescaleAvailable && DateTime.UtcNow >= _nextCompressionCheckUtc) { - _nextCompressionCheckUtc = DateTime.UtcNow.Add(s_compressionCheckInterval); + _nextCompressionCheckUtc = TimescaleSupport.NextCompressionCheckUtc(DateTime.UtcNow, s_compressionCheckInterval); await EvaluateCompressionJobHealthAsync(stoppingToken); } @@ -2865,6 +2922,20 @@ internal static bool IsQueryStoreCollector(string collectorName) => internal static bool IsPlanCorrectionCollector(string collectorName) => string.Equals(collectorName, PlanCorrectionCollector.Instance.Name, StringComparison.OrdinalIgnoreCase); + /// + /// #3604: whether a dispatched collector name is pg_wait_sampling — the third collector detached + /// from the sequential body, and the first detached for a DELIBERATE run length rather than a bimodal + /// one. Its sampler arm holds its connection for thirty one-second snapshots per cycle by design; awaited + /// inline that would delay every other collector on a stock PostgreSQL target by half a minute every five, + /// which is the #2700 starvation with a known cause. Detached behind the generic per-(server, collector) + /// gate, a still-running window simply skips the tick — and it cannot still be running, because the run is + /// 30 s and the cadence is 300 s; the gate is there for the day someone lengthens the window. On the + /// extension arm the run is a 500-row read and the detach costs nothing. Compared against the collector's + /// OWN declared name, for the renaming-safety reason the two siblings state. + /// + internal static bool IsPgWaitSamplingCollector(string collectorName) => + string.Equals(collectorName, PgWaitSamplingCollector.Instance.Name, StringComparison.OrdinalIgnoreCase); + /// /// #2219: refreshes this PostgreSQL server's statement text if it is due, and swallows everything if not. /// @@ -3234,6 +3305,12 @@ private void ReconcileServers(List servers, IReadOnlyListNo query-text preview: pg_session_states deliberately stores none (see the collector's /// class remarks), so the message identifies the session by pid/database/command tag instead of the /// statement text SQL Server's equivalent shows. + /// + /// The noise opt-outs ride the SAME switches SQL Server's read takes (#3539): the shared + /// longRunningQueryExcludeBackups drops the dump/restore utilities' sessions, and the shared + /// excludedDatabases list is applied after the read exactly as DarlingAlertReadAdapter + /// applies it. The unconditional ones — non-client backends (autovacuum, walsender), the + /// VACUUM/ANALYZE/REINDEX/CLUSTER statements, idle-in-transaction — live in the read's SQL; see + /// for each one's SQL + /// Server sibling and for why the three remaining SQL Server switches have no honest reading here. What + /// is still reported carries its command_tag on the incident line, so a CREATE that is an + /// index build reads as what it is rather than being dropped on a guess — the annotate-never-suppress + /// posture SQL Server's Agent-job name (#3497) takes on its card. /// private async Task EvaluatePgLongRunningQueryAsync( ServerRuntime runtime, AlertServerSnapshot snapshot, DarlingConfig config, CancellationToken cancellationToken) @@ -4470,7 +4558,10 @@ private async Task EvaluatePgLongRunningQueryAsync( var rows = await DarlingPgSessionStatesReader.GetCurrentLongRunningSessionsAsync( _postgres, runtime.ServerId, thresholdMs: thresholdMinutes * 60_000L, now, - PgLongRunningQueryRecencyMinutes, limit: alertSettings.LongRunningQueryMaxResults, cancellationToken); + PgLongRunningQueryRecencyMinutes, limit: alertSettings.LongRunningQueryMaxResults, + excludeBackups: alertSettings.LongRunningQueryExcludeBackups, + excludedDatabases: alertSettings.ExcludedDatabases, + cancellationToken); readClock.Restart(); var cooldown = TimeSpan.FromMinutes(Math.Max(1, _alertCooldownMinutes)); @@ -5138,12 +5229,22 @@ evidence the alert is judged on - that is freeBytes/totalBytes above. Losing it /// /// The #1581 compression-job self-heal check (fleet-level, hourly, Timescale-only): read every stuck - /// COMPRESSION-policy job () and hand them to the - /// self-alert evaluator's re-arm-once/escalate machine, wired to - /// on the SAME open connection. One stuck job whose next_start went -infinity silently halts the - /// store's archival tier — the field incident — so this makes it visible AND self-heals it. Failure-isolated - /// at the worker level too (the connection open is OUTSIDE the evaluator's own isolation): a store hiccup logs - /// and skips this check, never aborting the sweep — mirroring the purge / disk-check isolation. + /// COMPRESSION-policy job () + /// and hand them to the self-alert evaluator's re-arm-once/escalate machine, wired to + /// on the SAME open connection. One stuck job whose + /// next_start went -infinity silently halts the store's archival tier — the field incident — so + /// this makes it visible AND self-heals it. Failure-isolated at the worker level too (the connection open is + /// OUTSIDE the evaluator's own isolation): a store hiccup logs and skips this check, never aborting the sweep — + /// mirroring the purge / disk-check isolation. + /// + /// The stuck-job read may hold this method for + /// (#3575), and only on a pass where a job's -infinity arm tripped: the read re-executes its query + /// after that delay and reports the job only if the arm still trips, because TimescaleDB's view assembles + /// next_start and job_status from independent sources and reads the dead-job shape for a few + /// milliseconds at either edge of every healthy run. This method is awaited on the serial sweep loop, so + /// that five seconds is a once-an-hour worst case paid only when there was something to confirm; the + /// budgeting argument is on the constant. The evaluator downstream receives a list that has already been + /// confirmed and does not second-guess it. /// private async Task EvaluateCompressionJobHealthAsync(CancellationToken cancellationToken) { @@ -5767,12 +5868,16 @@ this wrap keeps any residue from reclassifying a perfectly good analysis pass. * } } - /* Persist the pass's insufficient-data determination (V19 marker) so the Viewer's - Recommendations tab shows "still collecting" instead of a false all-clear on a young - deployment: true + the engine's message when the pass hit the 24h data-span gate, cleared - (false) when a real pass completed on enough data. Failure-isolated like the other - observability writes. Only the two REAL terminal states write it — a Skipped/TimedOut/Error - pass (handled above / in the catch) leaves the last known marker untouched. */ + /* Persist the pass's data-state determination (V19 marker) so the Viewer's Recommendations + tab shows a reason instead of a false all-clear on a zero-finding read: true + the + engine's message when the pass hit the 24h data-span gate ("still collecting"); false + + the engine's message when the pass cleared the gate but the window itself collected zero + facts (#3524/#3551 — a dead-collector shape, "collection appears broken"; false-with-a- + message is a shape only this arm writes, so the viewer distinguishes it without a schema + change); cleared (false + null) when a real pass completed on measured facts, which is + how both miss markers self-heal. Failure-isolated like the other observability writes. + Only the REAL terminal states write it — a Skipped/TimedOut/Error pass (handled above / + in the catch) leaves the last known marker untouched. */ if (analysisService.InsufficientDataMessage is string insufficient) { await DarlingObservability.WriteAnalysisStateAsync( @@ -5780,6 +5885,13 @@ await DarlingObservability.WriteAnalysisStateAsync( return new AnalysisPassResult(AnalysisPassStatus.InsufficientData, 0, insufficient); } + if (analysisService.WindowEmptyMessage is string windowEmpty) + { + await DarlingObservability.WriteAnalysisStateAsync( + _postgres!, serverId, insufficientData: false, windowEmpty, _logger, stoppingToken); + return new AnalysisPassResult(AnalysisPassStatus.Ran, 0, windowEmpty); + } + await DarlingObservability.WriteAnalysisStateAsync( _postgres!, serverId, insufficientData: false, null, _logger, stoppingToken); return new AnalysisPassResult(AnalysisPassStatus.Ran, findings.Count, null); @@ -6507,7 +6619,9 @@ sys.dm_db_tuning_recommendations is re-read whole on every successful pass regar at completion is correct only for the sequential arm; a detached run finishes 100-230s later, by which time the 15s sweep has reset and rebuilt the mark from unrelated ticks. */ var peerMaxAtDispatchMs = PeerMaxOrNull(server); - if (IsQueryStoreCollector(name) || IsPlanCorrectionCollector(name)) + /* #3604: pg_wait_sampling is the third, and the reason is different in kind — see + IsPgWaitSamplingCollector: a deliberate 30 s sampling window, not a bimodal tail. */ + if (IsQueryStoreCollector(name) || IsPlanCorrectionCollector(name) || IsPgWaitSamplingCollector(name)) { _ = RunDetachedAsync(server, runner, name, peerMaxAtDispatchMs, cancellationToken); } @@ -7516,7 +7630,7 @@ collector on THIS server has not finished — skip is safe because every collect NotGated (mirroring QueryStoreServerGate's) collapses this to a single null check below — a future third collector needs only its own IsXCollector check added to this one condition, never a second one to keep in sync. */ - using var detachedGate = IsPlanCorrectionCollector(collectorName) + using var detachedGate = IsPlanCorrectionCollector(collectorName) || IsPgWaitSamplingCollector(collectorName) ? _detachedCollectorGates.GetOrAdd((runtime.ServerId, collectorName), static _ => new DetachedCollectorGate()).TryAcquire() : DetachedCollectorGate.NotGated; diff --git a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs index 36a303a32..8aa44523d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs +++ b/Darling/PerformanceMonitor.Darling.Service/FleetSweepEngine.cs @@ -153,6 +153,11 @@ public static class FleetSweepEngine distinct from the alert engine's metric names: the ledger is derived from summary scoring, and borrowing the engine's spellings would claim a provenance the rows do not have. */ public const string FamilyDeadlocks = "deadlocks"; + + /// Retained as VOCABULARY for rows already in the ledger; no new row carries it. #3539 A2 made + /// collection errors a Warning-ceiling share of the span's runs (the collector-health surface's own bar + /// and tier), and the ledger is derived from CRITICAL triggers only — so the family has nothing left to + /// fire on. Stored verdicts are immutable and their readers key on this spelling. public const string FamilyCollectionErrors = "collection-errors"; public const string FamilyMemoryCritical = "memory-critical"; public const string FamilyHighCpu = "high-cpu"; @@ -175,13 +180,21 @@ public static FleetSweepComposition Compose( FleetSweepRun? previousRun, IReadOnlyList previousVerdicts, IReadOnlyList activeWatchItems, - FleetSweepInstrumentCounters instruments) + FleetSweepInstrumentCounters instruments, + DeadlockRateThresholds deadlockRateTiers) { ArgumentNullException.ThrowIfNull(readings); ArgumentNullException.ThrowIfNull(previousVerdicts); ArgumentNullException.ThrowIfNull(activeWatchItems); ArgumentNullException.ThrowIfNull(instruments); + /* #3525: the deadlock-rate tiers travel INTO the shared scorer, so the sweep's verdicts band on the + pair get_alert_settings reports — required rather than defaulted, the DeadlockSeverity discipline: + a caller that kept the old call would compile and silently band on the shipped pair while the + Overview card used the store's. Built once, because the banding thresholds must be one + configuration for the whole sweep. */ + var banding = new DailyHealthThresholds { DeadlockRates = deadlockRateTiers }; + var sweepId = nowUtc.Ticks; var inSettleWindow = nowUtc - instruments.ServiceStartedUtc < PostRestartSettleWindow; var previousByServer = previousVerdicts.ToDictionary(v => v.ServerId); @@ -209,7 +222,7 @@ header cannot read green over this card. */ } else { - var classified = DailyHealthBandCalculator.Classify(reading.Signals); + var classified = DailyHealthBandCalculator.Classify(reading.Signals, banding); band = DailyHealthBandCalculator.Label(classified); reason = classified switch { @@ -246,12 +259,12 @@ header cannot read green over this card. */ { foreach (var reading in readings.Where(r => r.ReadFault is null)) { - if (DailyHealthBandCalculator.Classify(reading.Signals) != DailyHealthBand.Critical) + if (DailyHealthBandCalculator.Classify(reading.Signals, banding) != DailyHealthBand.Critical) { continue; } - foreach (var (family, evidence) in DecomposeCriticalTriggers(reading)) + foreach (var (family, evidence) in DecomposeCriticalTriggers(reading, banding)) { wouldHavePaged.Add(new FleetSweepWouldHavePagedEntry(reading.ServerId, family, evidence)); } @@ -693,38 +706,90 @@ private static FleetSweepWatchItem BuildItemImage( /// decided. Each row carries the trigger, the measured figure and the threshold it crossed, so an /// operator auditing a mute reads evidence rather than an assertion. /// - private static IEnumerable<(string Family, string Evidence)> DecomposeCriticalTriggers(FleetSweepServerReading reading) + private static IEnumerable<(string Family, string Evidence)> DecomposeCriticalTriggers( + FleetSweepServerReading reading, DailyHealthThresholds thresholds) { - var thresholds = DailyHealthThresholds.Default; var signals = reading.Signals; - if (signals.Deadlocks > 0) - { - yield return (FamilyDeadlocks, Evidence("deadlocks in span", signals.Deadlocks, 1)); + /* #3525: the deadlock family fires on the RATE the scorer banded Critical with, never on a bare + count — the same DeadlockSeverity call Classify makes, so the ledger cannot page on a trigger the + verdict did not band. A sub-hour span's unrateable arm maxes out at Warning, so it can never + reach this. */ + if (ServerHealthClassifier.DeadlockSeverity(signals.Deadlocks, signals.Window, thresholds.DeadlockRates) + == HealthSeverity.Critical) + { + /* Critical implies a rateable window (the unrateable arm returns Warning or Unknown), so the + rate is present by construction. */ + var ratePerHour = ServerHealthClassifier.DeadlockRatePerHour(signals.Deadlocks, signals.Window)!.Value; + yield return (FamilyDeadlocks, DeadlockRateEvidence( + ratePerHour, thresholds.DeadlockRates.CriticalPerHour, signals.Deadlocks)); } - if (signals.CollectionErrors > 0) - { - yield return (FamilyCollectionErrors, Evidence("collector runs ending in ERROR", signals.CollectionErrors, 1)); - } + /* Collection errors are absent from this decomposition on purpose (#3539 A2): the arm is now a + share of the span's runs with a Warning ceiling — see DailyHealthBandCalculator.CollectionErrorSeverity + — so no Critical verdict can be attributed to it and FamilyCollectionErrors produces no new rows. */ if (signals.MemoryCriticalEvents > 0) { yield return (FamilyMemoryCritical, Evidence("severe memory-pressure events", signals.MemoryCriticalEvents, 1)); } - if (signals.HighCpuEvents >= thresholds.HighCpuCriticalSamples) + /* #3539 A2: the CPU family fires on the arm the verdict banded with — the hot-sample count against + the bar SCALED to this span (the greater of the excursion-scale minimum and the sustained-heat + rate), never against the fixed 6 the pre-#3539 constant applied to every span. */ + if (DailyHealthBandCalculator.HighCpuSeverity(signals.HighCpuEvents, signals.Window, thresholds) == HealthSeverity.Critical) { - yield return (FamilyHighCpu, Evidence("high-CPU samples (>= 80% total host)", signals.HighCpuEvents, thresholds.HighCpuCriticalSamples)); + yield return (FamilyHighCpu, Evidence( + "high-CPU samples (>= 80% total host) against the span-scaled bar", + signals.HighCpuEvents, + thresholds.HighCpuCriticalSamplesFor(signals.Window))); + } + + /* #3539 A2/A3: the blocking family fires on the same BlockingSeverity call Classify makes — the + rate over the span, or the 60 s wait arm — so the ledger cannot page on a count the verdict did + not band. Which arm decided is legible from the evidence: the rate row names the rate tier, the + wait row names the wait bar. */ + /* The SIGNALS' peak, not the reading's: Classify sees only the signals, and the two must agree. + The production read fills both from one MAX. */ + var peakBlockSeconds = signals.PeakBlockWaitMs / 1000.0; + if (ServerHealthClassifier.BlockingSeverity(signals.BlockingEvents, peakBlockSeconds, signals.Window) + == HealthSeverity.Critical) + { + yield return (FamilyBlocking, BlockingEvidence(signals.BlockingEvents, signals.Window, peakBlockSeconds)); } + } - if (signals.BlockingEvents >= thresholds.BlockingCriticalEvents) + /// The blocking family's evidence (#3539 A3): the arm that banded is the one named. A 60 s + /// block is the wait arm's Critical whatever the rate; otherwise the RATE is the value, with the raw + /// count as its own member for the operator reconciling against the blocking grid. + private static string BlockingEvidence(long count, TimeSpan window, double peakBlockSeconds) + { + if (peakBlockSeconds >= ServerHealthThresholds.BlockingCriticalWaitSeconds) { - yield return (FamilyBlocking, Evidence("blocking events", signals.BlockingEvents, thresholds.BlockingCriticalEvents)); + return JsonSerializer.Serialize(new + { + derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", + trigger = "longest single block in the sweep span, seconds", + value = peakBlockSeconds, + threshold = ServerHealthThresholds.BlockingCriticalWaitSeconds, + blocking_count = count, + }); } + + /* Critical without the wait arm implies a rateable window at or past the Critical tier, so the + rate is present by construction. */ + var ratePerHour = ServerHealthClassifier.BlockingRatePerHour(count, window)!.Value; + return JsonSerializer.Serialize(new + { + derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", + trigger = "blocking events per hour over the sweep span", + value = ratePerHour, + threshold = ServerHealthThresholds.BlockingCriticalPerHour, + blocking_count = count, + }); } - private static string Evidence(string what, long value, long threshold) => + private static string Evidence(string what, long value, double threshold) => JsonSerializer.Serialize(new { derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", @@ -733,13 +798,39 @@ private static string Evidence(string what, long value, long threshold) => threshold, }); + /// The deadlock family's evidence (#3525): the shared shape with the RATE as the value — + /// because the rate is what banded — plus the raw count as its own member, because the count is the + /// countable fact an operator reconciles against the deadlock grid. + private static string DeadlockRateEvidence(double ratePerHour, double criticalPerHour, long count) => + JsonSerializer.Serialize(new + { + derivation = "summary-scoring critical trigger under alerts_enabled: false — not an alert-engine replay", + trigger = "deadlocks per hour over the sweep span", + value = ratePerHour, + threshold = criticalPerHour, + deadlock_count = count, + }); + private static string SerializeSignals(FleetSweepServerReading reading) => JsonSerializer.Serialize(new { deadlocks = reading.Signals.Deadlocks, + /* #3525: the rate the deadlock signal banded on, beside the count it was derived from (null on + an unrateable span) — the card's own disclosure rule: evidence a reader can disagree with has + to include the figure the band read. Additive members on NEW rows only; stored verdicts are + immutable and their readers key on the members that were always here. */ + deadlock_rate_per_hour = ServerHealthClassifier.DeadlockRatePerHour( + reading.Signals.Deadlocks, reading.Signals.Window), + window_minutes = reading.Signals.Window.TotalMinutes, collection_errors = reading.Signals.CollectionErrors, + /* #3539 A2/A3, additive on NEW rows: the denominator the error share bands on, and the blocking + rate beside its count (null on an unrateable span) — the same disclosure rule as the deadlock + rate above. */ + collection_runs = reading.Signals.CollectionRuns, high_cpu_events = reading.Signals.HighCpuEvents, blocking_events = reading.Signals.BlockingEvents, + blocking_rate_per_hour = ServerHealthClassifier.BlockingRatePerHour( + reading.Signals.BlockingEvents, reading.Signals.Window), memory_pressure_events = reading.Signals.MemoryPressureEvents, memory_critical_events = reading.Signals.MemoryCriticalEvents, alert_count = reading.Signals.AlertCount, @@ -781,6 +872,11 @@ public static async Task RunAsync( : await FleetSweepStore.GetServerVerdictsForEngineAsync(postgres, previousRun.SweepId, cancellationToken).ConfigureAwait(false); var activeItems = await FleetSweepStore.GetActiveWatchItemsAsync(postgres, cancellationToken).ConfigureAwait(false); + /* #3525: the deadlock-rate tiers, read once per sweep off the fleet reader's own published SQL + — an engine-seam read, so a fault here loudly costs this sweep slot rather than quietly + banding the fleet on the shipped pair. */ + var deadlockRateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken).ConfigureAwait(false); + var nowUtc = DateTime.UtcNow; var spanStartUtc = ComputeSpanStart(nowUtc, interval, previousRun); @@ -796,7 +892,7 @@ at a time and a herd all at once — and the sweep is a background errand racing var composition = Compose( nowUtc, spanStartUtc, alertsEnabled, servers.Count, readings, - previousRun, previousVerdicts, activeItems, ReadInstrumentCounters()); + previousRun, previousVerdicts, activeItems, ReadInstrumentCounters(), deadlockRateTiers); await FleetSweepStore.RecordSweepAsync( postgres, composition.Run, composition.Verdicts, @@ -840,20 +936,30 @@ private static async Task ReadServerSignalsAsync( var rows = await DarlingHealthReader.GetWindowSignalsAsync( postgres, serverId, spanStartUtc, spanEndUtc, cancellationToken).ConfigureAwait(false); + var peakBlock = rows.Count == 0 ? 0L : rows.Max(r => r.MaxBlockDurationMs); + var signals = new DailyHealthSignals { HasData = rows.Count > 0, Deadlocks = rows.Sum(r => r.DeadlockCount), CollectionErrors = rows.Sum(r => r.CollectionErrors), + /* #3539 A2: runs sum exactly as the errors do (additive counts over one half-open window), + so the share the band reads is the span's, not the first day-bucket's. */ + CollectionRuns = rows.Sum(r => r.CollectionRuns), HighCpuEvents = rows.Sum(r => r.HighCpuEvents), BlockingEvents = rows.Sum(r => r.BlockingEvents), + /* #3539 A2: the longest block across the span's day buckets — a MAX, not a sum, because it + is a magnitude; the blocking band's wait arm reads it. */ + PeakBlockWaitMs = peakBlock, MemoryPressureEvents = rows.Sum(r => r.MemoryPressureEvents), MemoryCriticalEvents = rows.Sum(r => r.MemoryCriticalEvents), AlertCount = rows.Sum(r => r.AlertCount), + /* #3525: the sweep's own span, NOT a calendar day — the denominator the deadlock rate + bands on. At the floor cadence (15 min) this is sub-hour and the band's unrateable arm + applies: deadlocks read Warning, never a rate-multiplied Critical. */ + Window = spanEndUtc - spanStartUtc, }; - var peakBlock = rows.Count == 0 ? 0L : rows.Max(r => r.MaxBlockDurationMs); - return new FleetSweepServerReading(serverId, serverName, signals, peakBlock, ReadFault: null); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -866,6 +972,24 @@ private static async Task ReadServerSignalsAsync( } } + /// The deadlock band's tiers from the store's singleton settings row (#3368, V120) — the + /// fleet reader's read, off its own published SQL, hoisted here once per sweep (#3525). A store with + /// no row yet bands on the shipped pair, which is what such a store would seed anyway; values come + /// back RAW and clamps on read. + private static async Task ReadDeadlockRateThresholdsAsync( + NpgsqlDataSource postgres, CancellationToken cancellationToken) + { + await using var command = postgres.CreateCommand(DarlingFleetReader.FleetDeadlockRateThresholdSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + return new DeadlockRateThresholds(reader.GetDouble(0), reader.GetDouble(1)); + } + + return DeadlockRateThresholds.Default; + } + /// The in-process instrument counters, read at compose time: the process-global alert /// read-health counter's start instant (the restart detector), the pass total summed across every /// per-server bucket, and the instance-wide swallowed-read total. diff --git a/Darling/PerformanceMonitor.Darling.Service/Hosting/DarlingWebTls.cs b/Darling/PerformanceMonitor.Darling.Service/Hosting/DarlingWebTls.cs index b8a93fe28..ac5d4cb01 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Hosting/DarlingWebTls.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Hosting/DarlingWebTls.cs @@ -24,7 +24,7 @@ namespace PerformanceMonitor.Darling.Service.Hosting; /// well-understood story for an internal CA, and the operator here supplies a certificate rather than the /// product minting one. If MCP ever gets TLS it is a separate decision with a separate blast radius. /// -/// Split on purpose. , and +/// Split on purpose. , and /// are PURE — they decide shape and validity with no file, no clock, and no /// logger, so the whole matrix pins in a unit test with no certificate on disk. is the one /// effectful member. That is the same split the bind ladder uses in . @@ -157,30 +157,66 @@ wrote it believes TLS is on. Refusing is louder than ignoring it. */ return new TlsPlan(TlsShape.NotConfigured, null); } + /// The lifetime gate's three answers. Kept as a kind rather than only the rendered refusal + /// string because the web host has to CARRY the decision out to the worker's self-alert sweep (#3517): the + /// host decides once, at load, and stays loopback-only on that decision until the next start regardless of + /// what the clock does afterwards — so the consumer must be told what was decided, not left to re-derive + /// it from the dates and reach a different answer later. + internal enum LifetimeStatus + { + /// Inside its validity period — serve it. + Usable, + + /// NotAfter has passed. Refused. + Expired, + + /// NotBefore is still ahead — a clock skew, or a certificate minted for a future + /// rotation. Refused, and its own kind on purpose: "expired" would send the operator to the wrong + /// problem. + NotYetValid, + } + + /// The lifetime gate's verdict: the kind, and the operator-facing reason when it refuses + /// ( is non-null exactly when is not + /// ). + internal readonly record struct LifetimeVerdict(LifetimeStatus Status, string? Refusal); + /// - /// PURE lifetime gate: the reason this certificate cannot be served AT ALL, or null when it is usable. - /// Expired and not-yet-valid both refuse, because a listener that presents either one fails every - /// handshake — the dashboard is down whether we refuse here or the browser refuses there, and refusing - /// here says why in the service log instead of leaving it to a certificate warning nobody reads. + /// PURE lifetime gate: whether this certificate can be served AT ALL, and when it cannot, why. Expired and + /// not-yet-valid both refuse, because a listener that presents either one fails every handshake — the + /// dashboard is down whether we refuse here or the browser refuses there, and refusing here says why in + /// the service log instead of leaving it to a certificate warning nobody reads. /// /// Not-yet-valid is worth its own arm: it is the signature of a clock skew or a certificate issued - /// for a future rotation, and "expired" would be an actively misleading thing to log for it. + /// for a future rotation, and "expired" would be an actively misleading thing to log for it. Expired is + /// checked FIRST, so a certificate whose window has both not opened and already closed (a nonsense + /// NotBefore past its NotAfter) reads as expired — the fact a restart cannot fix. /// - internal static string? LifetimeRefusal(DateTimeOffset notBefore, DateTimeOffset notAfter, DateTimeOffset nowUtc) + internal static LifetimeVerdict CheckLifetime(DateTimeOffset notBefore, DateTimeOffset notAfter, DateTimeOffset nowUtc) { if (nowUtc >= notAfter) { - return $"the certificate expired on {notAfter.UtcDateTime:u} — TLS cannot be served with it"; + return new LifetimeVerdict( + LifetimeStatus.Expired, + $"the certificate expired on {notAfter.UtcDateTime:u} — TLS cannot be served with it"); } if (nowUtc < notBefore) { - return $"the certificate is not valid until {notBefore.UtcDateTime:u} (check the system clock) — TLS cannot be served with it yet"; + return new LifetimeVerdict( + LifetimeStatus.NotYetValid, + $"the certificate is not valid until {notBefore.UtcDateTime:u} (check the system clock) — TLS cannot be served with it yet"); } - return null; + return new LifetimeVerdict(LifetimeStatus.Usable, null); } + /// The refusal reason alone — 's string half, null when usable. The + /// gate's original shape, kept so the message matrix stays pinned in one place; the web host itself calls + /// , because it needs the kind as well as the line. + internal static string? LifetimeRefusal(DateTimeOffset notBefore, DateTimeOffset notAfter, DateTimeOffset nowUtc) + => CheckLifetime(notBefore, notAfter, nowUtc).Refusal; + /// /// PURE advance warning for a certificate that is usable today and expires within /// ; null otherwise. A certificate that expires takes the dashboard down diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs index ca09dc0f4..db44ddc28 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingAlertReader.cs @@ -39,10 +39,16 @@ internal static class DarlingAlertReader { /* ─────────────────────────── alert history ─────────────────────────── */ + /// is the operator's Viewer acknowledgement (#3541 A3): a row the + /// operator hid from the Alert History grid. Always false on the default read, which excludes those rows; + /// carried so a read that INCLUDES them can label each one. + /// (#3539 A8e) is the persisted context, read so the tool can report + /// the tier the alert FIRED at through AlertHistoryRowSeverity rather than the colour its name + /// implies; null on resolution rows and rows written with no context. public sealed record AlertHistoryReadRow( DateTime AlertTime, int ServerId, string ServerName, string MetricName, double CurrentValue, double ThresholdValue, bool AlertSent, string NotificationType, - string? SendError, bool Muted, string? DetailText); + string? SendError, bool Muted, string? DetailText, bool Dismissed, string? ContextJson = null); private const string AlertHistorySelectColumns = @" alert_time, @@ -55,42 +61,84 @@ public sealed record AlertHistoryReadRow( notification_type, send_error, muted, - detail_text"; + detail_text, + context_json, + dismissed"; /// Per-server alert history — the viewer's AlertHistorySql. $1 window start, $2 window - /// end, $3 server_id, $4 limit (naive UTC / naive UTC / int / int). + /// end, $3 server_id, $4 limit, $5 include-dismissed (naive UTC / naive UTC / int / int / bool). /// /// The upper edge is bounded rather than open (#2495): the row cap is applied by the database, so /// trimming after the read would spend the whole LIMIT on rows newer than the anchor and hand back an - /// empty window that looks like a quiet one. + /// empty window that looks like a quiet one. + /// + /// The dismissed = FALSE filter is now a caller's choice rather than a hidden one (#3541 + /// A3). Dismissal is the Viewer operator's acknowledgement — "I have seen this row, hide it from the + /// grid" — and hiding it from the grid is the right default for a person at the grid. It is NOT a fact + /// about whether the alert fired, and an agent reconstructing an incident from get_alert_history + /// was handed a window with its acknowledged criticals silently removed, under a field that called the + /// remainder total_alerts. The default stays the grid's (so a caller who never sends the flag reads + /// what they always read), the payload now SAYS the filter applied and how many rows it removed, and + /// $5 = TRUE switches it off. Spelled (dismissed = FALSE OR $5) rather than as two more + /// consts so the pinned exclusion literal stays one string in one place. public const string AlertHistorySql = @" SELECT" + AlertHistorySelectColumns + @" FROM config_alert_log WHERE alert_time >= $1 AND alert_time <= $2 AND server_id = $3 -AND dismissed = FALSE +AND (dismissed = FALSE OR $5) ORDER BY alert_time DESC LIMIT $4"; /// All-servers alert history (the fleet default) — the viewer's AlertHistoryAllServersSql. - /// $1 window start, $2 window end, $3 limit (naive UTC / naive UTC / int). + /// $1 window start, $2 window end, $3 limit, $4 include-dismissed (naive UTC / naive UTC / int / bool). public const string AlertHistoryAllServersSql = @" SELECT" + AlertHistorySelectColumns + @" FROM config_alert_log WHERE alert_time >= $1 AND alert_time <= $2 -AND dismissed = FALSE +AND (dismissed = FALSE OR $4) ORDER BY alert_time DESC LIMIT $3"; + /// How many rows in the window the default read's dismissed = FALSE filter removes, per + /// server. $1 window start, $2 window end, $3 server_id. The count is what turns "dismissed rows are + /// excluded" from a disclaimer into a measurement: zero means the filter hid nothing, and a caller can + /// decide whether the hidden rows matter before re-reading with them included. + public const string DismissedAlertCountSql = @" +SELECT COUNT(*) +FROM config_alert_log +WHERE alert_time >= $1 +AND alert_time <= $2 +AND server_id = $3 +AND dismissed = TRUE"; + + /// The fleet-wide twin of . $1 window start, $2 window end. + public const string DismissedAlertCountAllServersSql = @" +SELECT COUNT(*) +FROM config_alert_log +WHERE alert_time >= $1 +AND alert_time <= $2 +AND dismissed = TRUE"; + /// /// Recent alerts newest first, excluding dismissed rows — the Alert History read. With no /// it aggregates ALL servers (the fleet default); with one it scopes to that - /// server. Mirrors the viewer's optional-serverId GetAlertHistoryAsync. + /// server. Mirrors the viewer's optional-serverId GetAlertHistoryAsync. The grid's semantics, + /// kept for the callers that want the grid's answer (the triage endpoint); the MCP tool reads through + /// so it can also ask for the dismissed rows. + /// + public static Task> GetAlertHistoryAsync( + NpgsqlDataSource postgres, DateTime sinceUtc, DateTime untilUtc, int? serverId, int limit, CancellationToken cancellationToken = default) => + GetAlertHistoryPageAsync(postgres, sinceUtc, untilUtc, serverId, limit, includeDismissed: false, cancellationToken); + + /// + /// with the dismissed filter as a parameter. Callers detecting + /// truncation pass limit + 1 and read the extra row as the signal. /// - public static async Task> GetAlertHistoryAsync( - NpgsqlDataSource postgres, DateTime sinceUtc, DateTime untilUtc, int? serverId, int limit, CancellationToken cancellationToken = default) + public static async Task> GetAlertHistoryPageAsync( + NpgsqlDataSource postgres, DateTime sinceUtc, DateTime untilUtc, int? serverId, int limit, bool includeDismissed, CancellationToken cancellationToken = default) { var rows = new List(); @@ -103,6 +151,9 @@ public static async Task> GetAlertHistoryAsync( DarlingMcpReadParameters.AddInt(command, serverId.Value); } DarlingMcpReadParameters.AddInt(command, limit); + /* Typed bool so Npgsql binds a boolean rather than inferring from an object — the predicate is + `(dismissed = FALSE OR $N)` and an untyped parameter there is a runtime type error, not a compile one. */ + command.Parameters.Add(new NpgsqlParameter { TypedValue = includeDismissed }); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) @@ -118,12 +169,34 @@ public static async Task> GetAlertHistoryAsync( reader.IsDBNull(7) ? "" : reader.GetString(7), reader.IsDBNull(8) ? null : reader.GetString(8), !reader.IsDBNull(9) && reader.GetBoolean(9), - reader.IsDBNull(10) ? null : reader.GetString(10))); + reader.IsDBNull(10) ? null : reader.GetString(10), + /* context_json sits at ordinal 11 and dismissed stays the LAST column at 12 — the viewer's + own column order, and the "dismissed is selected" pin anchors on it closing the list. */ + !reader.IsDBNull(12) && reader.GetBoolean(12), + reader.IsDBNull(11) ? null : reader.GetString(11))); } return rows; } + /// How many dismissed rows the window (and server scope) holds — the rows the default read hides. + /// See . + public static async Task CountDismissedAlertsAsync( + NpgsqlDataSource postgres, DateTime sinceUtc, DateTime untilUtc, int? serverId, CancellationToken cancellationToken = default) + { + await using var command = postgres.CreateCommand(serverId.HasValue ? DismissedAlertCountSql : DismissedAlertCountAllServersSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + DarlingMcpReadParameters.AddTimestamp(command, sinceUtc); + DarlingMcpReadParameters.AddTimestamp(command, untilUtc); + if (serverId.HasValue) + { + DarlingMcpReadParameters.AddInt(command, serverId.Value); + } + + var count = await command.ExecuteScalarAsync(cancellationToken); + return count is long l ? l : Convert.ToInt64(count, System.Globalization.CultureInfo.InvariantCulture); + } + /* ─────────────────────────── alert settings ─────────────────────────── */ /// The single global alert-settings row the service hot-swaps into DarlingAlertSettings — @@ -178,7 +251,9 @@ and the two engines carry separate calibrations on purpose (see the V122 rung). /* #3466 (V124): the fleet sweep's cadence knobs. APPENDED, same reason. Darling-only: Lite has no fleet to sweep, so McpAlertSettingsKeyTests records the omitted group as a decision. */ bool FleetSweepEnabled, - int FleetSweepIntervalMinutes); + int FleetSweepIntervalMinutes, + /* #3528 (V126): the Store Disk Pressure warning's GB floor. APPENDED, same reason. */ + int SelfDiskFreeWarnGb); /// The single global alert-settings row (id=1) — the viewer's AlertSettingsSelectSql. The /// columns are read in the SAME order the service reads them (StoreConfigProvider), and @@ -209,7 +284,8 @@ and the two engines carry separate calibrations on purpose (see the V122 rung). retention_hold_warn_ratio, retention_hold_critical_ratio, deadlock_warn_per_hour, deadlock_critical_per_hour, pg_deadlock_count_threshold, pg_blocking_count_threshold, - fleet_sweep_enabled, fleet_sweep_interval_minutes + fleet_sweep_enabled, fleet_sweep_interval_minutes, + self_disk_free_warn_gb FROM config_alert_settings WHERE id = 1"; @@ -260,7 +336,9 @@ FROM config_alert_settings /* #3444: V122 PostgreSQL Deadlocks/Blocking count thresholds at 62–63. */ reader.GetInt32(62), reader.GetInt32(63), /* #3466: V124 fleet-sweep cadence knobs at 64–65. */ - reader.GetBoolean(64), reader.GetInt32(65)); + reader.GetBoolean(64), reader.GetInt32(65), + /* #3528: V126 store-disk-warn GB floor at 66. */ + reader.GetInt32(66)); } /* ─────────────────────── delivery cooldown (a SECOND config table) ─────────────────────── */ diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingReader.cs index 5de8472d1..7ade7a64f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingReader.cs @@ -67,6 +67,27 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// internal static class DarlingBlockingReader { + /// + /// How many rows the incident readers scan when a dedup_key is supplied (#3541 A3). + /// + /// #2159 promised that the fingerprint filter runs over the WHOLE window before limit is + /// applied, so the cap can never discard the incident the key names. The reads then quietly capped at + /// 200 rows newest-first, so the promise held for the newest 200 rows of the window and silently failed + /// for everything older — and a no-match answer said "examined 200 rows" as if that were the window. The + /// scan is now bounded by THIS constant, over-fetched by one so the tool can see the ceiling bite and say + /// so in the no-match message, rather than by a cap the caller cannot see. + /// + /// 2,000 and not unbounded, because a fingerprint scan has to carry the row's heavy columns: the + /// deadlock key is derived from the graph XML and the blocking key from both SQL texts, so a scan row is + /// several kilobytes on the production fleet (blocked-process-report XML runs 3-6 KB). 2,000 keeps a + /// worst-case scan in the low tens of megabytes on a path that is a targeted lookup from an alert, and + /// is ten times the old cap — a busy server's day of blocking events on the measured population. Not + /// 5,000, which the analysis pair-row readers use, because those project the pair columns without the + /// XML. A window whose rows exceed it is reported as scan_truncated, and the remedy — anchor + /// as_of at the alert time with a narrow hours_back — is named in the message. + /// + public const int FingerprintScanCeiling = 2000; + /* ─────────────────────────── result rows ─────────────────────────── */ /// One blocked-process event — the shared alert-row fields (used by the XE→DMV merge and the @@ -117,9 +138,40 @@ public sealed class DeadlockReadRow : DeadlockAlertRow /// to the columns Lite's get_blocked_process_reports surfaces. Reads the BASE table (the viewer reads /// base here too, for the V7 plan-column safety). The six transaction/batch stamps are de-skewed from the /// server's local clock to naive UTC; event_time is the XE @timestamp and is already UTC, so - /// it is deliberately left alone. $1 server_id, $2/$3 window (naive UTC). + /// it is deliberately left alone. $1 server_id, $2/$3 window (naive UTC), $4 row cap. + /// + /// The cap is a PARAMETER, not a literal (#3541 A3). It was LIMIT 200 while the tool advertised + /// a caller-supplied limit and applied it with Take(limit), so a window with 5,000 blocking + /// events answered a 24-hour question from its newest 200 and nothing in the payload said so; the tool + /// now passes limit + 1 and reads the extra row as the truncation signal. + /// + /// Two consts share one body: this one, which every blocked-process row satisfies, and + /// , which adds the report-XML predicate in SQL so + /// get_blocked_process_xml can page over exactly the rows it can return. Filtering for XML in C# + /// after a capped fetch was the shape of the defect: a page of graph-less rows read as "no XML in the + /// window" while older rows with reports sat behind the cap. /// - public const string BlockedProcessReportsSql = """ + public const string BlockedProcessReportsSql = BlockedProcessReportsBody + """ + + ORDER BY event_time DESC + LIMIT $4 + """; + + /// The same read restricted to rows that CARRY a report XML — the population + /// get_blocked_process_xml pages over. Same parameters as . + /// The predicate is on the base-table column, not on the projection, so the planner can apply it before + /// the ORDER BY / LIMIT rather than after materializing every row's XML. + public const string BlockedProcessReportsWithXmlSql = BlockedProcessReportsBody + """ + + AND blocked_process_report_xml IS NOT NULL + AND blocked_process_report_xml <> '' + ORDER BY event_time DESC + LIMIT $4 + """; + + /// The shared projection + window predicate behind the two XE consts above. Private so the + /// executable statements stay the two public consts the tests pin. + private const string BlockedProcessReportsBody = """ WITH svr AS ( SELECT COALESCE(( SELECT sp.utc_offset_minutes @@ -169,8 +221,6 @@ ORDER BY sp.collection_time DESC WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 - ORDER BY event_time DESC - LIMIT 200 """; /// @@ -180,7 +230,7 @@ LIMIT 200 /// (DmvBlockingSnapshotCollector stamps it from context.CollectionTime), while its two /// transaction stamps come straight off sys.dm_tran_active_transactions and are server-local — so /// the same de-skew applies here, on two columns instead of six. Same parameters as - /// . + /// , including the $4 row cap. /// public const string DmvBlockingSnapshotsSql = """ WITH svr AS ( @@ -218,74 +268,43 @@ ORDER BY sp.collection_time DESC AND collection_time >= $2 AND collection_time <= $3 ORDER BY event_time DESC - LIMIT 200 + LIMIT $4 """; /// /// The recent blocked-process reports over the window — the XE rows plus the DMV-fallback rows for any /// (blocked, blocker) SPID pair the XE session did not capture in the same minute, merged and re-capped - /// to the newest 200 via the shared (Lite's exact semantics). + /// to the newest via the shared (Lite's + /// exact semantics). + /// + /// The cap is the caller's, and the two arms are fetched so that a merged result LARGER than + /// the cap is observable whenever the window holds one. A caller wanting to detect truncation passes + /// limit + 1 and checks whether the extra row came back — the pattern every honest page in this + /// store uses (get_collection_log, get_query_heatmap). The XE arm fetches exactly + /// ; the DMV arm fetches PLUS the number of XE rows that came + /// back, because the merge drops a DMV row for every (pair, minute) an XE row already covers, and the DMV + /// collector samples once per cycle so at most one DMV row hides behind each XE row. Fetching the DMV arm + /// at the bare cap would let a surplus made entirely of XE-covered rows vanish in the merge and report a + /// full page as complete. The merge then re-caps to , so a result of exactly + /// rows means "at least this many" and a shorter one means "all of them". /// public static async Task> GetRecentBlockedProcessReportsAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) { var items = new List(); var dmvItems = new List(); await using var connection = await postgres.OpenConnectionAsync(cancellationToken); - await using (var command = new NpgsqlCommand(BlockedProcessReportsSql, connection)) - { - command.CommandTimeout = McpCommandDeadlines.ReadSeconds; - DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken)) - { - items.Add(new BlockedProcessReadRow - { - EventTime = reader.IsDBNull(0) ? null : reader.GetDateTime(0), - DatabaseName = reader.IsDBNull(1) ? "" : reader.GetString(1), - BlockedSpid = reader.IsDBNull(2) ? 0 : reader.GetInt32(2), - BlockedEcid = reader.IsDBNull(3) ? 0 : reader.GetInt32(3), - BlockingSpid = reader.IsDBNull(4) ? 0 : reader.GetInt32(4), - BlockingEcid = reader.IsDBNull(5) ? 0 : reader.GetInt32(5), - WaitTimeMs = reader.IsDBNull(6) ? 0 : reader.GetInt64(6), - WaitResource = reader.IsDBNull(7) ? null : reader.GetString(7), - LockMode = reader.IsDBNull(8) ? "" : reader.GetString(8), - BlockedStatus = reader.IsDBNull(9) ? null : reader.GetString(9), - BlockedIsolationLevel = reader.IsDBNull(10) ? null : reader.GetString(10), - BlockedLogUsed = reader.IsDBNull(11) ? 0 : reader.GetInt64(11), - BlockedTransactionCount = reader.IsDBNull(12) ? 0 : reader.GetInt32(12), - BlockedClientApp = reader.IsDBNull(13) ? null : reader.GetString(13), - BlockedHostName = reader.IsDBNull(14) ? null : reader.GetString(14), - BlockedLoginName = reader.IsDBNull(15) ? null : reader.GetString(15), - BlockedSqlText = reader.IsDBNull(16) ? "" : reader.GetString(16), - BlockingStatus = reader.IsDBNull(17) ? null : reader.GetString(17), - BlockingIsolationLevel = reader.IsDBNull(18) ? null : reader.GetString(18), - BlockingClientApp = reader.IsDBNull(19) ? null : reader.GetString(19), - BlockingHostName = reader.IsDBNull(20) ? null : reader.GetString(20), - BlockingLoginName = reader.IsDBNull(21) ? null : reader.GetString(21), - BlockingSqlText = reader.IsDBNull(22) ? "" : reader.GetString(22), - BlockedTransactionName = reader.IsDBNull(23) ? null : reader.GetString(23), - BlockingTransactionName = reader.IsDBNull(24) ? null : reader.GetString(24), - BlockedLastTranStartedUtc = reader.IsDBNull(25) ? null : reader.GetDateTime(25), - BlockingLastTranStartedUtc = reader.IsDBNull(26) ? null : reader.GetDateTime(26), - BlockedLastBatchStartedUtc = reader.IsDBNull(27) ? null : reader.GetDateTime(27), - BlockingLastBatchStartedUtc = reader.IsDBNull(28) ? null : reader.GetDateTime(28), - BlockedLastBatchCompletedUtc = reader.IsDBNull(29) ? null : reader.GetDateTime(29), - BlockingLastBatchCompletedUtc = reader.IsDBNull(30) ? null : reader.GetDateTime(30), - BlockedPriority = reader.IsDBNull(31) ? 0 : reader.GetInt32(31), - BlockingPriority = reader.IsDBNull(32) ? 0 : reader.GetInt32(32), - BlockedProcessReportXml = reader.IsDBNull(33) ? "" : reader.GetString(33), - ContentiousObject = reader.IsDBNull(34) ? "" : reader.GetString(34), - }); - } - } + await ReadXeRowsAsync(connection, BlockedProcessReportsSql, serverId, startUtc, endUtc, cap, items, cancellationToken); await using (var command = new NpgsqlCommand(DmvBlockingSnapshotsSql, connection)) { command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + /* cap + the XE rows in hand: one DMV row can hide behind each XE row in the merge (see the + method remarks), so this is what keeps a surplus observable after the dedupe. */ + DarlingMcpReadParameters.AddInt(command, cap + items.Count); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -317,21 +336,110 @@ public static async Task> GetRecentBlockedProcessRep } /* Lite's XE-preferred fallback, verbatim via the shared merge: keep all BPR rows; append a DMV row - only where no BPR covers the same SPID pair in the same minute; re-cap to the 200 newest. */ - BlockedProcessReportMerge.AppendDmvFallbackRows(items, dmvItems); + only where no BPR covers the same SPID pair in the same minute; re-cap to the caller's newest. */ + BlockedProcessReportMerge.AppendDmvFallbackRows(items, dmvItems, cap); return items; } + /// + /// The newest XE blocked-process reports that CARRY a report XML — the population + /// get_blocked_process_xml pages over. XE arm only: the DMV fallback never has a report, so merging + /// it in would only add rows the caller has to discard, and discarding after a cap is the defect this + /// exists to remove. Callers detecting truncation pass limit + 1. + /// + public static async Task> GetRecentBlockedProcessReportsWithXmlAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) + { + var items = new List(); + await using var connection = await postgres.OpenConnectionAsync(cancellationToken); + await ReadXeRowsAsync(connection, BlockedProcessReportsWithXmlSql, serverId, startUtc, endUtc, cap, items, cancellationToken); + return items; + } + + /// Runs one of the two XE consts (same projection, same parameters) and appends its rows. One + /// mapper for both so the with-XML variant cannot drift a column from the unfiltered one. + private static async Task ReadXeRowsAsync( + NpgsqlConnection connection, string sql, int serverId, DateTime startUtc, DateTime endUtc, int cap, + List items, CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand(sql, connection); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + DarlingMcpReadParameters.AddInt(command, cap); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + items.Add(new BlockedProcessReadRow + { + EventTime = reader.IsDBNull(0) ? null : reader.GetDateTime(0), + DatabaseName = reader.IsDBNull(1) ? "" : reader.GetString(1), + BlockedSpid = reader.IsDBNull(2) ? 0 : reader.GetInt32(2), + BlockedEcid = reader.IsDBNull(3) ? 0 : reader.GetInt32(3), + BlockingSpid = reader.IsDBNull(4) ? 0 : reader.GetInt32(4), + BlockingEcid = reader.IsDBNull(5) ? 0 : reader.GetInt32(5), + WaitTimeMs = reader.IsDBNull(6) ? 0 : reader.GetInt64(6), + WaitResource = reader.IsDBNull(7) ? null : reader.GetString(7), + LockMode = reader.IsDBNull(8) ? "" : reader.GetString(8), + BlockedStatus = reader.IsDBNull(9) ? null : reader.GetString(9), + BlockedIsolationLevel = reader.IsDBNull(10) ? null : reader.GetString(10), + BlockedLogUsed = reader.IsDBNull(11) ? 0 : reader.GetInt64(11), + BlockedTransactionCount = reader.IsDBNull(12) ? 0 : reader.GetInt32(12), + BlockedClientApp = reader.IsDBNull(13) ? null : reader.GetString(13), + BlockedHostName = reader.IsDBNull(14) ? null : reader.GetString(14), + BlockedLoginName = reader.IsDBNull(15) ? null : reader.GetString(15), + BlockedSqlText = reader.IsDBNull(16) ? "" : reader.GetString(16), + BlockingStatus = reader.IsDBNull(17) ? null : reader.GetString(17), + BlockingIsolationLevel = reader.IsDBNull(18) ? null : reader.GetString(18), + BlockingClientApp = reader.IsDBNull(19) ? null : reader.GetString(19), + BlockingHostName = reader.IsDBNull(20) ? null : reader.GetString(20), + BlockingLoginName = reader.IsDBNull(21) ? null : reader.GetString(21), + BlockingSqlText = reader.IsDBNull(22) ? "" : reader.GetString(22), + BlockedTransactionName = reader.IsDBNull(23) ? null : reader.GetString(23), + BlockingTransactionName = reader.IsDBNull(24) ? null : reader.GetString(24), + BlockedLastTranStartedUtc = reader.IsDBNull(25) ? null : reader.GetDateTime(25), + BlockingLastTranStartedUtc = reader.IsDBNull(26) ? null : reader.GetDateTime(26), + BlockedLastBatchStartedUtc = reader.IsDBNull(27) ? null : reader.GetDateTime(27), + BlockingLastBatchStartedUtc = reader.IsDBNull(28) ? null : reader.GetDateTime(28), + BlockedLastBatchCompletedUtc = reader.IsDBNull(29) ? null : reader.GetDateTime(29), + BlockingLastBatchCompletedUtc = reader.IsDBNull(30) ? null : reader.GetDateTime(30), + BlockedPriority = reader.IsDBNull(31) ? 0 : reader.GetInt32(31), + BlockingPriority = reader.IsDBNull(32) ? 0 : reader.GetInt32(32), + BlockedProcessReportXml = reader.IsDBNull(33) ? "" : reader.GetString(33), + ContentiousObject = reader.IsDBNull(34) ? "" : reader.GetString(34), + }); + } + } + /* ─────────────────────────── deadlocks ─────────────────────────── */ /// /// The recent deadlock events over the window — the viewer's RecentDeadlocksSql against the BASE /// deadlocks table (base, for the V7 victim-plan column). The parsed /// is computed on access from the graph XML. $1 server_id, - /// $2/$3 window (naive UTC). + /// $2/$3 window (naive UTC), $4 row cap — a parameter for the same reason as + /// 's: it was LIMIT 50 under a tool that advertised + /// limit, so a caller asking for 100 deadlocks silently got 50 and a payload that called them + /// total_deadlocks. /// - public const string RecentDeadlocksSql = """ + public const string RecentDeadlocksSql = RecentDeadlocksBody + """ + + ORDER BY deadlock_time DESC + LIMIT $4 + """; + + /// The same read restricted to rows that CARRY a graph — the population get_deadlock_detail + /// pages over, so its limit counts graphs rather than rows it will have to discard. Same parameters + /// as . + public const string RecentDeadlocksWithGraphSql = RecentDeadlocksBody + """ + + AND deadlock_graph_xml IS NOT NULL + AND deadlock_graph_xml <> '' + ORDER BY deadlock_time DESC + LIMIT $4 + """; + + private const string RecentDeadlocksBody = """ SELECT collection_time, deadlock_time, @@ -342,17 +450,19 @@ FROM deadlocks WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 - ORDER BY deadlock_time DESC - LIMIT 50 """; + /// The newest deadlocks over the window — every row, or with + /// only those carrying a graph. Callers detecting truncation pass + /// limit + 1 and read the extra row as the signal. public static async Task> GetRecentDeadlocksAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, bool graphOnly = false, CancellationToken cancellationToken = default) { var rows = new List(); - await using var command = postgres.CreateCommand(RecentDeadlocksSql); + await using var command = postgres.CreateCommand(graphOnly ? RecentDeadlocksWithGraphSql : RecentDeadlocksSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + DarlingMcpReadParameters.AddInt(command, cap); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs index 85f5a13d7..ee546f58b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs @@ -100,7 +100,12 @@ WITH raw AS collection_time, wait_type, delta_wait_time_ms, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY wait_type ORDER BY collection_time)))) AS interval_seconds + /* #3540: the STORED interval where the row has one; 0 (no delta knowable) becomes NULL through NULLIF + and the reader drops the row rather than reading 0.00. NULL (a pre-V127 row) falls back to the LAG. */ + CASE WHEN sample_interval_seconds IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY wait_type ORDER BY collection_time)))) + ELSE NULLIF(sample_interval_seconds, 0) + END AS interval_seconds FROM v_wait_stats WHERE server_id = $1 AND wait_type LIKE 'LCK%' @@ -110,7 +115,7 @@ AND wait_type LIKE 'LCK%' SELECT collection_time, wait_type, - CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS double precision) / interval_seconds ELSE 0 END AS wait_time_ms_per_second + CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS double precision) / interval_seconds END AS wait_time_ms_per_second FROM raw WHERE delta_wait_time_ms >= 0 ORDER BY collection_time, wait_type @@ -141,10 +146,16 @@ public static async Task> GetLockWaitTrendAsync( await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* A NULL rate is an unknowable interval (#3540): the row is dropped, not read as 0. */ + if (reader.IsDBNull(2)) + { + continue; + } + items.Add(new LockWaitTrendReadPoint( reader.GetDateTime(0), reader.IsDBNull(1) ? string.Empty : reader.GetString(1), - reader.IsDBNull(2) ? 0 : reader.GetDouble(2))); + reader.GetDouble(2))); } return items; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs index 3ddd5288d..c16183c7d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingConfigHistoryReader.cs @@ -200,19 +200,21 @@ FROM v_trace_flags /* ─────────────────────────── database scoped config (latest snapshot) ─────────────────────────── */ /// The latest sys.database_scoped_configurations snapshot — the viewer's - /// DatabaseScopedConfigSql. $1 server_id. + /// DatabaseScopedConfigSql plus the trailing capture_time (#3541 A10: captured on connect, so + /// the tool must be able to say how old "current" is). $1 server_id. public const string DatabaseScopedConfigSql = """ - SELECT database_name, configuration_name, value, value_for_secondary + SELECT database_name, configuration_name, value, value_for_secondary, capture_time FROM v_database_scoped_config WHERE server_id = $1 AND capture_time = (SELECT MAX(capture_time) FROM v_database_scoped_config WHERE server_id = $1) ORDER BY database_name, configuration_name """; - public static async Task> GetLatestDatabaseScopedConfigAsync( + public static async Task> GetLatestDatabaseScopedConfigAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(DatabaseScopedConfigSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); @@ -224,9 +226,10 @@ public static async Task> GetLatestDatabaseSco reader.IsDBNull(1) ? "" : reader.GetString(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.IsDBNull(3) ? null : reader.GetString(3))); + capturedAt ??= reader.GetDateTime(4); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── query store health (latest snapshot) ─────────────────────────── */ @@ -236,17 +239,18 @@ public static async Task> GetLatestDatabaseSco /// scoped-config read above). Unlike the config-family reads this table is HOURLY, not on-connect, /// so "latest" here is at most an hour old on a healthy schedule. $1 server_id. public const string QueryStoreHealthSql = """ - SELECT database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes + SELECT database_name, actual_state, desired_state, readonly_reason, current_storage_size_mb, max_storage_size_mb, size_based_cleanup_mode, stale_query_threshold_days, max_plans_per_query, interval_length_minutes, capture_time FROM v_query_store_health WHERE server_id = $1 AND capture_time = (SELECT MAX(capture_time) FROM v_query_store_health WHERE server_id = $1) ORDER BY database_name """; - public static async Task> GetLatestQueryStoreHealthAsync( + public static async Task> GetLatestQueryStoreHealthAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(QueryStoreHealthSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); @@ -264,8 +268,9 @@ public static async Task> GetLatestQueryStoreHealt reader.IsDBNull(7) ? 0L : reader.GetInt64(7), reader.IsDBNull(8) ? 0L : reader.GetInt64(8), reader.IsDBNull(9) ? 0L : reader.GetInt64(9))); + capturedAt ??= reader.GetDateTime(10); } - return rows; + return new LatestSnapshot(capturedAt, rows); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingCurrentConfigReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingCurrentConfigReader.cs index dc7a422de..4073e4af0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingCurrentConfigReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingCurrentConfigReader.cs @@ -37,20 +37,23 @@ public sealed record ServerConfigReadRow( public bool ValuesMatch => ValueConfigured == ValueInUse; } - /// Latest sys.configurations snapshot for one server — the viewer's ServerConfigSql. + /// Latest sys.configurations snapshot for one server — the viewer's ServerConfigSql plus + /// the trailing capture_time (#3541 A10): config is captured ON CONNECT, so the "current" value this + /// read serves can be as old as the last successful connect, and the tool must be able to say so. /// $1 server_id. public const string ServerConfigSql = """ - SELECT configuration_name, value_configured, value_in_use, is_dynamic, is_advanced + SELECT configuration_name, value_configured, value_in_use, is_dynamic, is_advanced, capture_time FROM v_server_config WHERE server_id = $1 AND capture_time = (SELECT MAX(capture_time) FROM v_server_config WHERE server_id = $1) ORDER BY configuration_name """; - public static async Task> GetLatestServerConfigAsync( + public static async Task> GetLatestServerConfigAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(ServerConfigSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); @@ -63,9 +66,10 @@ public static async Task> GetLatestServerConfigAsync( reader.IsDBNull(2) ? 0 : reader.GetInt64(2), !reader.IsDBNull(3) && reader.GetBoolean(3), !reader.IsDBNull(4) && reader.GetBoolean(4))); + capturedAt ??= reader.GetDateTime(5); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── database config (sys.databases) ─────────────────────────── */ @@ -80,7 +84,8 @@ public sealed record DatabaseConfigReadRow( bool IsMemoryOptimizedEnabled, bool IsOptimizedLockingOn); /* 28 columns in the viewer's / Lite's exact SELECT order — the reader below maps them by incrementing - ordinal, so this list's order is load-bearing and must stay byte-identical. */ + ordinal, so this list's order is load-bearing and must stay byte-identical. capture_time is APPENDED + as a 29th column (#3541 A10) and read by explicit ordinal 28, so the 28-column mapping is untouched. */ public const string DatabaseConfigSql = """ SELECT database_name, state_desc, compatibility_level, collation_name, recovery_model, is_read_only, is_auto_close_on, is_auto_shrink_on, @@ -89,17 +94,23 @@ public sealed record DatabaseConfigReadRow( is_query_store_on, is_encrypted, is_trustworthy_on, is_db_chaining_on, is_broker_enabled, is_cdc_enabled, is_mixed_page_allocation_on, log_reuse_wait_desc, page_verify_option, target_recovery_time_seconds, delayed_durability, - is_accelerated_database_recovery_on, is_memory_optimized_enabled, is_optimized_locking_on + is_accelerated_database_recovery_on, is_memory_optimized_enabled, is_optimized_locking_on, + capture_time FROM v_database_config WHERE server_id = $1 AND capture_time = (SELECT MAX(capture_time) FROM v_database_config WHERE server_id = $1) ORDER BY database_name """; - public static async Task> GetLatestDatabaseConfigAsync( + /// The ordinal of the appended capture_time column in — one + /// past the 28-column block the incrementing mapping consumes. + private const int DatabaseConfigCaptureTimeOrdinal = 28; + + public static async Task> GetLatestDatabaseConfigAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(DatabaseConfigSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); @@ -138,29 +149,32 @@ DatabaseConfigSql binds to the same fields — see the byte-identical note above !reader.IsDBNull(++ordinal) && reader.GetBoolean(ordinal), !reader.IsDBNull(++ordinal) && reader.GetBoolean(ordinal), !reader.IsDBNull(++ordinal) && reader.GetBoolean(ordinal))); + capturedAt ??= reader.GetDateTime(DatabaseConfigCaptureTimeOrdinal); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── trace flags (DBCC TRACESTATUS) ─────────────────────────── */ public sealed record TraceFlagReadRow(int TraceFlag, bool Status, bool IsGlobal, bool IsSession); - /// Latest trace-flags snapshot for one server — the viewer's TraceFlagsSql. $1 server_id. - /// A row exists only while a flag is enabled, so an empty result means no active flags at the last capture. + /// Latest trace-flags snapshot for one server — the viewer's TraceFlagsSql plus the trailing + /// capture_time (#3541 A10). $1 server_id. A row exists only while a flag is enabled, so an empty + /// result means no active flags at the last capture — and, having no row, no stamp either. public const string TraceFlagsSql = """ - SELECT trace_flag, status, is_global, is_session + SELECT trace_flag, status, is_global, is_session, capture_time FROM v_trace_flags WHERE server_id = $1 AND capture_time = (SELECT MAX(capture_time) FROM v_trace_flags WHERE server_id = $1) ORDER BY trace_flag """; - public static async Task> GetLatestTraceFlagsAsync( + public static async Task> GetLatestTraceFlagsAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(TraceFlagsSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); @@ -172,8 +186,9 @@ public static async Task> GetLatestTraceFlagsAsync( !reader.IsDBNull(1) && reader.GetBoolean(1), !reader.IsDBNull(2) && reader.GetBoolean(2), !reader.IsDBNull(3) && reader.GetBoolean(3))); + capturedAt ??= reader.GetDateTime(4); } - return rows; + return new LatestSnapshot(capturedAt, rows); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs index 85a523d02..ccb1d6a33 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs @@ -165,11 +165,21 @@ PlanFetchProbeMs is null && TextFetchProbeMs is null + (TextFetchProbeMs ?? 0) + (TextFetchWriteMs ?? 0); } - /// One database file's latest I/O snapshot; avg latency is computed by the tool. + /// One database file's latest I/O snapshot; avg latency is computed by the tool. + /// (#3540): the measured seconds the deltas accrued over; + /// 0 is the calculator's "no delta knowable" marker (first sighting, counter reset, gap past the policy) + /// and null is a pre-V127 row that never recorded one. The tool reports latency as null on a 0 rather than + /// the "0.00 ms" a restart used to render. public sealed record FileIoRow( string DatabaseName, string FileName, string FileType, string PhysicalName, double SizeMb, long DeltaReads, long DeltaWrites, long DeltaReadBytes, long DeltaWriteBytes, - long DeltaStallReadMs, long DeltaStallWriteMs); + long DeltaStallReadMs, long DeltaStallWriteMs, int? SampleIntervalSeconds) + { + /// True when the row's deltas are the calculator's unknowable marker — a stored interval of + /// exactly 0. NULL (pre-V127, interval never recorded) is NOT unknowable: those rows keep the + /// pre-#3540 reading, because nothing about them can say otherwise. + public bool IsUnknowable => SampleIntervalSeconds == 0; + } /// One tempdb space-usage sample over the window. public sealed record TempDbSample( @@ -335,7 +345,12 @@ public static async Task GetCpuWindowAggregateAsync( /// wait_type, heaviest first. The SUMs CAST to bigint for the typed GetInt64 reader (Postgres /// SUM(bigint) is numeric). Lite's per-user IgnoredWaitTypes exclusion is dropped (headless /// has no per-user ignore config — the viewer's wait reads drop it the same way). $1 server_id, $2/$3 - /// window (naive UTC). + /// window (naive UTC), $4 row cap. + /// + /// The cap is a PARAMETER, not a literal (#3541 A3). It was LIMIT 50 while the tool advertised + /// a limit up to 1,000 and applied it with Take(limit), so a caller asking for every wait + /// type on a server that had observed 80 silently got 50 — the same shape DarlingPgWaitReader fixed + /// for the PostgreSQL twin. The tool passes limit + 1 and reads the extra row as truncation. /// public const string WaitStatsSql = """ SELECT @@ -349,16 +364,19 @@ FROM v_wait_stats AND collection_time <= $3 GROUP BY wait_type ORDER BY SUM(delta_wait_time_ms) DESC - LIMIT 50 + LIMIT $4 """; + /// The heaviest wait types over the window. Callers detecting truncation + /// pass limit + 1 and read the extra row as the signal. public static async Task> GetWaitStatsAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) { var rows = new List(); await using var command = postgres.CreateCommand(WaitStatsSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; AddWindow(command, serverId, startUtc, endUtc); + AddInt(command, cap); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -429,9 +447,18 @@ public static async Task HasAnyWaitStatAsync( /// /// A single wait type's per-second trend — Lite's GetWaitStatsTrendAsync: the interval rate is - /// this collection's delta divided by the seconds since the previous collection (a LAG over the - /// truncate-then-diff epoch idiom proven value-identical DuckDB↔Postgres). $1 server_id, $2 wait_type, + /// this collection's delta divided by the seconds the delta accrued over. $1 server_id, $2 wait_type, /// $3/$4 window (naive UTC). + /// + /// The interval is the STORED one where the row has it (#3540). wait_stats carries + /// sample_interval_seconds since V127 — the calculator's measured seconds, 0 when no delta was + /// knowable (first sighting, counter reset, a gap past the policy). A 0 maps to NULL through + /// NULLIF, so the rate is NULL rather than the confident 0.00 ms/sec this read used to emit at + /// exactly the moments (restarts) it was unknowable; the reader drops the row (a missing sample, never a + /// fabricated idle one). A NULL interval is a pre-V127 row whose interval was never recorded, and for + /// those the LAG over collection_time (the truncate-then-diff epoch idiom proven value-identical + /// DuckDB↔Postgres) is what this read always did, so history keeps rendering. The first row of the + /// window has no prior and no stored interval either way — NULL, not 0. /// public const string WaitTrendSql = """ WITH raw AS @@ -440,7 +467,10 @@ WITH raw AS collection_time, delta_wait_time_ms, delta_signal_wait_time_ms, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) AS interval_seconds + CASE WHEN sample_interval_seconds IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) + ELSE NULLIF(sample_interval_seconds, 0) + END AS interval_seconds FROM v_wait_stats WHERE server_id = $1 AND wait_type = $2 @@ -449,8 +479,8 @@ FROM v_wait_stats ) SELECT collection_time, - CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS wait_time_ms_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(delta_signal_wait_time_ms AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS signal_wait_time_ms_per_second + CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS DOUBLE PRECISION) / interval_seconds END AS wait_time_ms_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(delta_signal_wait_time_ms AS DOUBLE PRECISION) / interval_seconds END AS signal_wait_time_ms_per_second FROM raw ORDER BY collection_time """; @@ -468,9 +498,16 @@ public static async Task> GetWaitTrendAsync( await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* A NULL rate is an unknowable interval (#3540) — the row is dropped rather than read as 0. Both + rates share one interval, so they are NULL together; the first is the test. */ + if (reader.IsDBNull(1)) + { + continue; + } + items.Add(new WaitTrendPoint( reader.GetDateTime(0), - reader.IsDBNull(1) ? 0 : reader.GetDouble(1), + reader.GetDouble(1), reader.IsDBNull(2) ? 0 : reader.GetDouble(2))); } @@ -531,19 +568,22 @@ LIMIT 1 /// /// The latest memory-clerk breakdown — Lite's GetLatestMemoryClerksAsync: every clerk at the /// newest collection, heaviest first. memory_mb is numeric(18,2) → double precision. $1 server_id. + /// collection_time rides along on every row (#3541 A10) so the tool can say WHEN the snapshot it + /// serves was taken — the same statement as the rows, never a second read that could stamp the next one. /// public const string LatestMemoryClerksSql = """ - SELECT clerk_type, CAST(memory_mb AS double precision) + SELECT clerk_type, CAST(memory_mb AS double precision), collection_time FROM v_memory_clerks WHERE server_id = $1 AND collection_time = (SELECT MAX(collection_time) FROM v_memory_clerks WHERE server_id = $1) ORDER BY memory_mb DESC """; - public static async Task> GetLatestMemoryClerksAsync( + public static async Task> GetLatestMemoryClerksAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(LatestMemoryClerksSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; AddInt(command, serverId); @@ -553,9 +593,10 @@ public static async Task> GetLatestMemoryClerksAsync( rows.Add(new MemoryClerkRow( reader.GetString(0), reader.IsDBNull(1) ? 0 : reader.GetDouble(1))); + capturedAt ??= reader.GetDateTime(2); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── file I/O ─────────────────────────── */ @@ -563,7 +604,8 @@ public static async Task> GetLatestMemoryClerksAsync( /// /// The latest file-I/O snapshot per database file — Lite's GetLatestFileIoStatsAsync, ordered /// by total stall descending; avg latency (stall/op) is computed by the tool. size_mb is - /// numeric → double precision; the delta columns are bigint. $1 server_id. + /// numeric → double precision; the delta columns are bigint. $1 server_id. collection_time + /// is the trailing column (#3541 A10): the snapshot's stamp, read once and published as captured_at. /// public const string LatestFileIoStatsSql = """ SELECT @@ -577,17 +619,20 @@ public static async Task> GetLatestMemoryClerksAsync( delta_read_bytes, delta_write_bytes, delta_stall_read_ms, - delta_stall_write_ms + delta_stall_write_ms, + sample_interval_seconds, + collection_time FROM v_file_io_stats WHERE server_id = $1 AND collection_time = (SELECT MAX(collection_time) FROM v_file_io_stats WHERE server_id = $1) ORDER BY (delta_stall_read_ms + delta_stall_write_ms) DESC """; - public static async Task> GetLatestFileIoStatsAsync( + public static async Task> GetLatestFileIoStatsAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(LatestFileIoStatsSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; AddInt(command, serverId); @@ -605,10 +650,12 @@ public static async Task> GetLatestFileIoStatsAsync( reader.IsDBNull(7) ? 0 : reader.GetInt64(7), reader.IsDBNull(8) ? 0 : reader.GetInt64(8), reader.IsDBNull(9) ? 0 : reader.GetInt64(9), - reader.IsDBNull(10) ? 0 : reader.GetInt64(10))); + reader.IsDBNull(10) ? 0 : reader.GetInt64(10), + reader.IsDBNull(11) ? null : reader.GetInt32(11))); + capturedAt ??= reader.GetDateTime(12); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── tempdb ─────────────────────────── */ @@ -667,24 +714,27 @@ public static async Task> GetTempDbTrendAsync( /// /// The latest perfmon counters — Lite's GetLatestPerfmonStatsAsync: counter_name / - /// instance_name / cntr_value / delta_cntr_value at the newest collection. $1 server_id. + /// instance_name / cntr_value / delta_cntr_value at the newest collection, with that collection's + /// collection_time trailing (#3541 A10, published once as captured_at). $1 server_id. /// public const string LatestPerfmonStatsSql = """ SELECT counter_name, instance_name, cntr_value, - delta_cntr_value + delta_cntr_value, + collection_time FROM v_perfmon_stats WHERE server_id = $1 AND collection_time = (SELECT MAX(collection_time) FROM v_perfmon_stats WHERE server_id = $1) ORDER BY counter_name """; - public static async Task> GetLatestPerfmonStatsAsync( + public static async Task> GetLatestPerfmonStatsAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { var rows = new List(); + DateTime? capturedAt = null; await using var command = postgres.CreateCommand(LatestPerfmonStatsSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; AddInt(command, serverId); @@ -696,9 +746,10 @@ public static async Task> GetLatestPerfmonStatsAsync( reader.IsDBNull(1) ? "" : reader.GetString(1), reader.IsDBNull(2) ? 0 : reader.GetInt64(2), reader.IsDBNull(3) ? 0 : reader.GetInt64(3))); + capturedAt ??= reader.GetDateTime(4); } - return rows; + return new LatestSnapshot(capturedAt, rows); } /* ─────────────────────────── top queries ─────────────────────────── */ @@ -706,11 +757,13 @@ public static async Task> GetLatestPerfmonStatsAsync( /// /// Top query-stats groups over the window — a focused projection of the viewer's TopQueriesSql /// (the columns Lite's get_top_queries_by_cpu returns): group by (database, query_hash), sum the - /// deltas + carry min/max spreads, rank by summed delta_elapsed_time descending, over-fetch by + /// deltas + carry min/max spreads, rank by summed delta_worker_time (CPU — the tool's promise; + /// #3523, the viewer's duration grid keeps its elapsed ranking) descending, over-fetch by /// 5 to drop WAITFOR shells via the latest-text LATERAL, cap at top. Summed bigints CAST back to bigint /// for the typed reader. The aggregate reads the base query_stats table (it projects no text); /// the text LATERAL reads v_query_stats, which resolves the #1767 payload dimension — the plan - /// tools read it the same way. $1 server_id, $2/$3 window (naive UTC), $4 top. + /// tools read it the same way. $1 server_id, $2/$3 window (naive UTC), $4 top, $5 database filter (NULL = all), + /// $6 lifetime max_dop floor (0 = no parallelism filter; #3541 A13). /// public const string TopQueriesSql = """ WITH ranked AS ( @@ -752,8 +805,17 @@ FROM query_stats (each proc-hosted statement groups under its own host object), while ad-hoc rows carry NULL and keep collapsing into one group per hash exactly as before. */ GROUP BY database_name, query_hash, host_object_name - HAVING SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0 - ORDER BY SUM(delta_elapsed_time) DESC + HAVING (SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0) + /* #3541 A13: the parallelism filter is part of the QUERY, applied to the grouped population + BEFORE the CPU ranking and the cap. It used to run in C# over the returned top-N page, so + parallel_only=true on a box whose twenty hottest plans were serial answered an empty page while + the window held parallel plans further down — and the engine's own CXPACKET advice sends agents + to exactly that call. $6 is the group's lifetime max_dop floor: 0 admits every group (the + unfiltered read, byte-identical in result to before), 2 is parallel_only, min_dop is itself. The + COALESCE keeps a group whose max_dop was never captured (NULL) out of a filtered page, which is + what the C# arm did too (null read as 0, and 0 > 1 is false). */ + AND COALESCE(MAX(max_dop), 0) >= $6 + ORDER BY SUM(delta_worker_time) DESC LIMIT $4 + 5 ) SELECT @@ -795,7 +857,7 @@ ORDER BY collection_time DESC LIMIT 1 ) AS t ON TRUE WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' - ORDER BY r.total_elapsed_us DESC + ORDER BY r.total_cpu_us DESC LIMIT $4 """; @@ -863,8 +925,12 @@ FROM query_stats without that arm every unrelated ad-hoc statement in a database would pool into one row. */ GROUP BY database_name, host_object_name, CASE WHEN host_object_name IS NULL THEN query_hash END - HAVING SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0 - ORDER BY SUM(delta_elapsed_time) DESC + HAVING (SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0) + /* #3541 A13: same in-query parallelism floor as TopQueriesSql — see its note. Under the rollup + the group's max_dop is the max across every fragment, so a procedure whose dynamic SQL went + parallel in ANY fragment passes parallel_only, which is the question being asked. */ + AND COALESCE(MAX(max_dop), 0) >= $6 + ORDER BY SUM(delta_worker_time) DESC LIMIT $4 + 5 ) SELECT @@ -907,13 +973,20 @@ ORDER BY collection_time DESC LIMIT 1 ) AS t ON TRUE WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' - ORDER BY r.total_elapsed_us DESC + ORDER BY r.total_cpu_us DESC LIMIT $4 """; + /// + /// The top-N groups by CPU, ranked over the population that passes every filter. + /// is the lifetime max_dop floor a group must reach to be ranked at all (#3541 A13): 0 for no + /// parallelism filter, 2 for parallel_only, the caller's min_dop otherwise — see + /// 's HAVING note. The filter is IN the statement so the page is the top-N of the + /// filtered population, not the filtered remainder of an unfiltered top-N. + /// public static async Task> GetTopQueriesByCpuAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int top, string? databaseName, - bool rollUpByHostObject = false, CancellationToken cancellationToken = default) + bool rollUpByHostObject = false, int minMaxDop = 0, CancellationToken cancellationToken = default) { var rows = new List(); /* #2235: same parameters, same columns, different GROUP BY — see TopQueriesByHostObjectSql. */ @@ -922,6 +995,7 @@ public static async Task> GetTopQueriesByCpuAsync( AddWindow(command, serverId, startUtc, endUtc); AddInt(command, top); AddNullableText(command, databaseName); + AddInt(command, minMaxDop); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -960,7 +1034,7 @@ public static async Task> GetTopQueriesByCpuAsync( /// Top procedure-stats groups over the window — a focused projection of the viewer's /// TopProceduresSql (the columns Lite's get_top_procedures_by_cpu returns): group by /// (database, schema, object, type), sum the deltas + carry min/max spreads, rank by summed - /// delta_elapsed_time descending, cap at top. Reads the base procedure_stats table + /// delta_worker_time (CPU — the tool's promise; #3523) descending, cap at top. Reads the base procedure_stats table /// (no v_ view). $1 server_id, $2/$3 window (naive UTC), $4 top. /// public const string TopProceduresSql = """ @@ -989,7 +1063,7 @@ FROM procedure_stats AND ($5::text IS NULL OR database_name = $5) GROUP BY database_name, schema_name, object_name, object_type HAVING SUM(delta_execution_count) > 0 OR SUM(delta_elapsed_time) > 0 - ORDER BY SUM(delta_elapsed_time) DESC + ORDER BY SUM(delta_worker_time) DESC LIMIT $4 """; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index ae7a07932..3cbb51755 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -264,16 +264,84 @@ GROUP BY server_id /// This view is the SQL Server extended-event capture and nothing else (#3017). /// v_deadlocks is SELECT * FROM deadlocks, and deadlocks is written by exactly one /// collector — DeadlocksCollector, whose TargetTable it is. A PostgreSQL target's deadlocks - /// go to pg_deadlocks instead, there is no v_pg_deadlocks, and nothing joins the two, so - /// this count is structurally zero for a PostgreSQL server no matter how many deadlocks its clusters - /// have. Zero is also what a genuinely quiet SQL Server reports, which is why the total ships with - /// beside it: the reading that needs no action and the reading that - /// does not cover the fleet are otherwise the same character. + /// never reach it, so this count is structurally zero for a PostgreSQL server no matter how many + /// deadlocks its clusters have; is that engine's read (#3539), and + /// takes one or the other by engine. Zero is also what a genuinely quiet SQL + /// Server reports, which is why the total ships with beside it: the + /// reading that needs no action and the reading whose collector is silent are otherwise the same + /// character. public const string FleetDeadlockSql = @" SELECT server_id, COUNT(*) AS cnt, MAX(deadlock_time) AS last_seen FROM v_deadlocks WHERE deadlock_time >= $1 AND deadlock_time <= $2 +GROUP BY server_id"; + + /// Deadlocks in the window per PostgreSQL server (#3539) — the same two columns as + /// , from the engine's own counter rather than a captured graph. $1 window + /// start, $2 window end (both naive UTC). + /// + /// A counter DIFFERENCE, never a sum of the column. pg_database_stats.deadlocks is + /// pg_stat_database.deadlocks stored raw: a lifetime counter per database, sampled every minute, + /// so the same 122 sits in every one of a day's 1,440 rows and SUM(deadlocks) over a window is a + /// number with no meaning (measured on one store: 8,006,912 across 50 servers with zero new deadlocks in + /// the window). What happened IN the window is each consecutive pair's difference, taken per + /// (server_id, database_name) series — the same LAG shape DarlingPgDatabaseReader.PgDatabaseSql + /// uses for get_pg_database_stats, and the two must agree on a server or the fleet card would + /// contradict the tool it sends a reader to. + /// + /// Clamped at zero across a reset. pg_stat_reset() or a crash restart rewinds the + /// counter, and a plain difference goes negative there; GREATEST(…, 0) drops that interval + /// rather than subtracting a lifetime from the window. The interval's real deadlocks (those between the + /// reset and the next sample) survive as the next difference. The tool reports how many intervals + /// clamped; this card does not — a band is not the place for a reset count, and + /// get_pg_database_stats is one call away. + /// + /// Summed across databases because the card is per SERVER and the SQL Server count it + /// sits beside is too: a deadlock graph names the databases involved, but v_deadlocks is counted + /// per server_id, and the band's tiers were measured per server-hour (#3368). The NULL-named + /// shared-relation row PostgreSQL emits is its own series under PARTITION BY (grouping + /// semantics), so it differences correctly and is summed in. + /// + /// last_seen is the sample that first showed the increase — the deadlock happened + /// somewhere in the preceding minute, which is the resolution a per-minute counter has. Bounded to the + /// window like the count, where the SQL Server "last seen" is the newest graph in the window too; the + /// unbounded "ever" the viewer's card shows for SQL Server has no cheap PostgreSQL twin (it would be a + /// scan of the whole counter series for its last step), and "last within the window" is what the + /// count-carrying chip actually renders. + /// + /// intervals is how many differences were taken — the count of consecutive-sample + /// pairs across every series in the window — and it is what makes the count a MEASUREMENT. A difference + /// needs two samples; a server with one row in the window, or none, has had no difference taken, and + /// its cnt of zero is the arithmetic of an empty set, not an observation that nothing deadlocked. + /// This is where the PostgreSQL arm parts from the SQL Server one on purpose: COUNT(*) over an + /// event table with no events IS an observation (the collector looked and found none), where + /// LAG over no samples is undefined. bands only when this is positive, + /// which is also what keeps #3539 A6 intact — an online PostgreSQL target nothing has collected from + /// measures nothing and reads Warning, not "Healthy — 1 of 6 measured". + /// + /// Bounded on collection_time, the hypertable's partitioning column, so chunk exclusion + /// keeps the read to the window's chunk(s): on the one-minute cadence the default hour is ~60 rows per + /// database per server, and the fleet's whole hour is tens of thousands of rows behind the + /// (server_id, collection_time) index — the same order as 's + /// two scans. + public const string FleetPgDeadlockSql = @" +WITH sampled AS +( + SELECT + server_id, + collection_time, + deadlocks - LAG(deadlocks) OVER (PARTITION BY server_id, database_name ORDER BY collection_time) AS raw_delta + FROM pg_database_stats + WHERE collection_time >= $1 + AND collection_time <= $2 +) +SELECT + server_id, + CAST(coalesce(SUM(GREATEST(raw_delta, 0)), 0) AS bigint) AS cnt, + MAX(collection_time) FILTER (WHERE raw_delta > 0) AS last_seen, + CAST(count(raw_delta) AS bigint) AS intervals +FROM sampled GROUP BY server_id"; /// The deadlock health band's two tiers from the singleton settings row (#3368, V120). @@ -399,6 +467,7 @@ public static async Task GetFleetOverviewAsync( var threads = await ReadThreadsAsync(postgres, cancellationToken); var blocking = await ReadBlockingAsync(postgres, windowStartUtc, windowEndUtc, cancellationToken); var deadlocks = await ReadDeadlocksAsync(postgres, windowStartUtc, windowEndUtc, cancellationToken); + var pgDeadlocks = await ReadPgDeadlocksAsync(postgres, windowStartUtc, windowEndUtc, cancellationToken); var lastCollection = await ReadLastCollectionAsync(postgres, now, cancellationToken); var failingCollectors = await ReadFailingCollectorCountsAsync(postgres, now, cancellationToken); var tags = await ReadTagsAsync(postgres, cancellationToken); @@ -422,6 +491,10 @@ and the band counts and the worst-first ranking would be derived from it. */ threads.TryGetValue(server.ServerId, out var t); blocking.TryGetValue(server.ServerId, out var b); deadlocks.TryGetValue(server.ServerId, out var deadlock); + /* A miss leaves default(PgDeadlockRow) - zero count, no last-seen, ZERO intervals - which for + a PostgreSQL target is "no difference was taken" (unmeasured, not quiet), and for a SQL + Server is a row BuildCard never looks at. */ + pgDeadlocks.TryGetValue(server.ServerId, out var pgDeadlock); /* Not `lastCollection.TryGetValue(..., out var lastColl)` — that leaves lastColl as default(DateTime) (0001-01-01) on a miss, and default(DateTime) is NOT null, so it does not hit ClassifyFreshness's NeverCollected branch: it falls through to the @@ -436,7 +509,7 @@ ancient timestamp. */ tags.TryGetValue(server.ServerId, out var serverTags); cards.Add(BuildCard( - server, c, pg, m, mp, t, b, deadlock, lastColl, collectors, serverTags, now, + server, c, pg, m, mp, t, b, deadlock, pgDeadlock, lastColl, collectors, serverTags, now, windowEndUtc - windowStartUtc, deadlockTiers)); } @@ -453,6 +526,13 @@ ancient timestamp. */ /// metric passes default for its row, which is why those types never have to be NAMED in a test /// — they are internal only because CS0051 requires every parameter type of an internal method to be /// at least as accessible as it. + /// The SQL Server extended-event count for the window (); + /// structurally zero for a PostgreSQL target and ignored for one. + /// The PostgreSQL counter-difference count for the window + /// (, #3539), with how many differences it was summed from; + /// structurally empty for a SQL Server and ignored for one. Two parameters rather than one pre-merged + /// row so this method — the step that decides what a card CLAIMS — is where the engine picks, and a + /// test can hand a card BOTH rows and assert which one it believed. internal static FleetServerCard BuildCard( FleetServerRow server, CpuRow cpu, @@ -462,6 +542,7 @@ internal static FleetServerCard BuildCard( ThreadsRow threads, BlockingRow blocking, DeadlockRow deadlock, + PgDeadlockRow pgDeadlock, DateTime? lastCollection, CollectorCounts collectors, List? tags, @@ -469,7 +550,6 @@ internal static FleetServerCard BuildCard( TimeSpan deadlockWindow, DeadlockRateThresholds deadlockTiers) { - var deadlockCount = deadlock.Count; /* Lite's XE-preferred / DMV-fallback, per server: XE when it has any row this window, else the DMV snapshot — both count and worst-wait come from whichever source wins. */ @@ -484,6 +564,20 @@ internal static FleetServerCard BuildCard( rather than beside ClassifyPlatform because the CPU source classification needs it. */ var (isPostgres, isAurora) = ClassifyEngineKind(server.EngineKind); + /* The engine's OWN deadlock reading (#3539): the extended-event graph count for a SQL Server, the + pg_stat_database counter difference for a PostgreSQL target. Chosen by engine rather than summed, + because the other engine's row is a structural zero and a sum would hide which instrument + answered; the card says which through deadlock_source. The collector band travels with the + count for the same reason - coverage asks about the collector that produced THIS number. */ + var deadlockCount = isPostgres ? pgDeadlock.Count : deadlock.Count; + var deadlockLastSeen = isPostgres ? pgDeadlock.LastSeen : deadlock.LastSeen; + var deadlockCollectorBand = isPostgres ? collectors.PgDeadlockBand : collectors.DeadlockBand; + /* Whether the count is a MEASUREMENT. On SQL Server it always is: COUNT(*) over the graph table is + an observation even at zero (#3272's engine-not-collector rule). On PostgreSQL the count is a + counter DIFFERENCE, and a difference of fewer than two samples is not zero deadlocks, it is no + reading - see FleetPgDeadlockSql's intervals paragraph. */ + var deadlockMeasured = !isPostgres || pgDeadlock.Intervals > 0; + /* One expression for "total non-idle host CPU, from whichever collector has it" (#3267), shared with the viewer's card so the two cannot drift on the fallback. cpuPercent stays the SQL-Server-process share and is NOT filled from the PostgreSQL arm: Performance Insights publishes only the host @@ -501,15 +595,24 @@ than leaving a consumer to infer it from which fields are null. */ var hasMemoryPressure = pressure.WaiterCount > 0 || pressure.TimeoutCount > 0 || pressure.ForcedCount > 0; var maxBlockedSeconds = maxBlockingWaitMs / 1000.0; - /* The three DMV-sourced readings, with "not measured" expressed as null for an engine that has no + /* The two DMV-sourced readings, with "not measured" expressed as null for an engine that has no row in the views behind them (#3272). The reads above produced zeros for such a target, and a zero here argued Healthy — a green dot for a metric nothing measured. The published COUNTS are - left exactly as they are: #3017's deadlock_source and the fleet coverage block explain a total - built out of those zeros, and nulling them would make the total's own denominator unreadable. - It is the BAND that stops claiming health. */ + left exactly as they are: the fleet coverage block explains a total built out of those zeros, and + nulling them would make the total's own denominator unreadable. It is the BAND that stops + claiming health. + + Deadlocks left this set in #3539: both engines now have a source behind the reading (the graph + capture or the server counter). The SQL Server arm keeps the engine-not-collector rule + ServerMetricSources states - a silent deadlocks collector reads zero and bands Healthy, with + deadlock_source and the coverage block disclosing the gap. The PostgreSQL arm is gated on the + instrument instead: its count is a difference, and a window with fewer than two samples per + series has had no difference taken, so the band reads Unknown there rather than a Healthy + computed from an empty set. That is also what keeps #3539 A6's card - an online PostgreSQL + target nothing has collected from - measuring nothing. */ var memoryPressureForBand = ServerMetricSources.DmvSourced(hasMemoryPressure, isPostgres); var blockingForBand = ServerMetricSources.DmvSourced(blockingCount, isPostgres); - var deadlocksForBand = ServerMetricSources.DmvSourced(deadlockCount, isPostgres); + int? deadlocksForBand = deadlockMeasured ? deadlockCount : null; var metrics = new ServerHealthMetrics { @@ -522,6 +625,9 @@ because cpuForAlert there is percent of an allocation that moves. The source is HasMemoryPressure = memoryPressureForBand, BlockingCount = blockingForBand, MaxBlockedSeconds = maxBlockedSeconds, + /* #3539 A3: the blocking count's own denominator — the same card window the deadlock count was + read over, because one pair of bounds windowed both reads. */ + BlockingWindow = deadlockWindow, DeadlockCount = deadlocksForBand, /* #3368: the count's own denominator and the store's tiers travel WITH it, because the band is a rate. Omitting either would leave the deadlock dot banded on one pair of numbers while the @@ -534,6 +640,8 @@ because cpuForAlert there is percent of an allocation that moves. The source is ThreadsWaitingForCpu = threads.RunnableTasks, RequestsWaitingForThreads = threads.WorkQueue, FailedCollectorCount = collectors.Failing, + /* #3539 A8d: the denominator that grades the failing count into a share. */ + CollectorCount = collectors.Total, }; /* Freshness -> the card's collection state, through the SAME mapping the WPF card and the sidebar @@ -547,6 +655,10 @@ because cpuForAlert there is percent of an allocation that moves. The source is var overall = ServerHealthClassifier.OverallMetricSeverity(metrics); var band = ServerHealthClassifier.ClassifyBand(isOnline, awaitingFirstCollection, collectionStale, overall); + /* #3528: the fold above skips Unknown, so an online server with five of six metrics structurally + Unknown still bands Healthy. The counts ride the card so every consumer of the band label can + qualify it ("Healthy — 1 of 6 measured") instead of rendering an unqualified green. */ + var (measuredMetrics, totalMetrics) = ServerHealthClassifier.MeasuredMetricCounts(metrics); /* Per-server platform (design D4): the reliable signal the composer's measure auto-greying matches a measure's appliesTo against — see ClassifyPlatform for the edition mapping and why AWS RDS / msdb are @@ -591,15 +703,21 @@ deliberately not surfaced. */ MemorySeverity = ServerHealthClassifier.MemorySeverity(memoryPressureForBand), BlockingCount = blockingCount, MaxBlockingWaitMs = maxBlockingWaitMs, - BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds), + /* blockingForBand, not the raw count, for the reason DeadlockRatePerHour below gives: a + PostgreSQL target's raw count is a structural zero, and a rate derived from it would publish + 0.0/hr against a severity that reads Unknown (#3539 A3). */ + BlockingRatePerHour = blockingForBand.HasValue + ? ServerHealthClassifier.BlockingRatePerHour(blockingForBand.Value, deadlockWindow) + : null, + BlockingWindow = deadlockWindow, + BlockingSeverity = ServerHealthClassifier.BlockingSeverity(blockingForBand, maxBlockedSeconds, deadlockWindow), DeadlockCount = deadlockCount, - DeadlockLastSeen = deadlock.LastSeen, - /* deadlocksForBand, not the raw count, for the same reason DeadlockSeverity below takes it: on a - PostgreSQL target there is no deadlock reading at all and the raw count is a STRUCTURAL zero, - so a rate derived from it publishes 0.0/hr - a measurement nobody took - while the severity on - the same card correctly reads Unknown. The card chip and the viewer detail line both render - this field on nothing but non-null, so the disclosure has to live here. #3017 fixed exactly - this confusion for the count and the band; the rate must not reintroduce it. */ + DeadlockLastSeen = deadlockLastSeen, + DeadlockMeasured = deadlockMeasured, + /* Through deadlocksForBand rather than the raw int so the rate and the severity below are + derived from ONE value: the chip and the viewer detail line render this on non-null alone, + and a rate published for a count the band did not see is the #3017 confusion one field + over. Null exactly when the PostgreSQL arm took no difference (#3539). */ DeadlockRatePerHour = deadlocksForBand.HasValue ? ServerHealthClassifier.DeadlockRatePerHour(deadlocksForBand.Value, deadlockWindow) : null, @@ -607,7 +725,7 @@ the same card correctly reads Unknown. The card chip and the viewer detail line DeadlockRateThresholds = deadlockTiers, DeadlockSeverity = ServerHealthClassifier.DeadlockSeverity( deadlocksForBand, deadlockWindow, deadlockTiers), - DeadlockCollectorBand = collectors.DeadlockBand, + DeadlockCollectorBand = deadlockCollectorBand, TotalThreads = threads.TotalThreads, CurrentWorkers = threads.CurrentWorkers, AvailableThreads = availableThreads, @@ -616,8 +734,11 @@ the same card correctly reads Unknown. The card chip and the viewer detail line ThreadsSeverity = ServerHealthClassifier.ThreadsSeverity(threads.TotalThreads, availableThreads, threads.RunnableTasks, threads.WorkQueue), HealthyCollectorCount = collectors.Healthy, FailedCollectorCount = collectors.Failing, - CollectorSeverity = ServerHealthClassifier.CollectorSeverity(collectors.Failing), + CollectorCount = collectors.Total, + CollectorSeverity = ServerHealthClassifier.CollectorSeverity(collectors.Failing, collectors.Total), OverallMetricSeverity = overall, + MeasuredMetricCount = measuredMetrics, + MetricCount = totalMetrics, }; } @@ -660,13 +781,20 @@ public static FleetOverviewResult BuildRollup( /* #3017's denominator, reduced from the CARDS for the same reason the totals above are: the coverage figure and the total it qualifies then reconcile by construction rather than by two - queries agreeing. Only Read is counted as read — every other arm, INCLUDING an enum value a - later build adds and this switch has never heard of, lands in the silent bucket. A new source - kind that inflated the read count would restore exactly the defect this exists to fix, where - one that lands in an uncovered bucket merely attributes a real gap imprecisely. */ + queries agreeing. Only the arms FleetDeadlockCoverage.IsCovered names count as read — every + other arm, INCLUDING an enum value a later build adds and this switch has never heard of, + lands in the silent bucket. A new source kind that inflated the read count would restore + exactly the defect this exists to fix, where one that lands in an uncovered bucket merely + attributes a real gap imprecisely. The PostgreSQL arm is covered AND tallied on its own + (#3539): the sub-count names the instrument, the read count names the coverage. */ + if (FleetDeadlockCoverage.IsCovered(card.DeadlockSource)) + { + deadlockSourcesRead++; + } + switch (card.DeadlockSource) { - case FleetDeadlockSource.Read: deadlockSourcesRead++; break; + case FleetDeadlockSource.Read: break; case FleetDeadlockSource.PostgresTarget: deadlockPostgresTargets++; break; case FleetDeadlockSource.CollectorDenied: deadlockCollectorsDenied++; break; default: deadlockCollectorsSilent++; break; @@ -779,7 +907,11 @@ cannot say it differently. */ if (c.BlockingSeverity >= HealthSeverity.Warning && c.BlockingCount > 0) { - parts.Add($"Blocking {c.BlockingCount}"); + /* #3539 A3: the deadlock line's rule, one metric over — the count is the countable fact, the + rate is the banded one, and an unrateable window prints the count alone. */ + parts.Add(c.BlockingRatePerHour.HasValue + ? $"Blocking {c.BlockingCount} ({c.BlockingRatePerHour.Value.ToString("0.0", CultureInfo.InvariantCulture)}/hr)" + : $"Blocking {c.BlockingCount}"); } if (c.DeadlockSeverity >= HealthSeverity.Warning && c.DeadlockCount > 0) @@ -796,7 +928,11 @@ 1 in an hour and 1 in a day are the same string. The count stays because it is t if (c.CollectorSeverity >= HealthSeverity.Warning) { - parts.Add($"{c.FailedCollectorCount} collector{(c.FailedCollectorCount == 1 ? "" : "s")} failing"); + /* #3539 A8d: the share is what grades the band, so the denominator is named when there is one + — "3 of 40 collectors failing" — and the bare count stands when none was declared. */ + parts.Add(c.CollectorCount > 0 + ? $"{c.FailedCollectorCount} of {c.CollectorCount} collectors failing" + : $"{c.FailedCollectorCount} collector{(c.FailedCollectorCount == 1 ? "" : "s")} failing"); } if (c.CollectionStale) @@ -804,9 +940,22 @@ 1 in an hour and 1 in a day are the same string. The count stays because it is t parts.Add("collection stale"); } + if (c.MetricCount > 0 && c.MeasuredMetricCount == 0) + { + /* #3539 A6: the card banded Warning because NOTHING on it was measured (OverallMetricSeverity's + nothing-measured arm), and no per-metric clause above can fire for a card whose every band is + Unknown — so without this the ranking would show "Needs attention" against a card that + cannot say why. The same words the viewer's reason uses. */ + parts.Add(NoMetricMeasuredReason); + } + return parts.Count > 0 ? string.Join(", ", parts) : "Needs attention"; } + /// The reason clause for a card on which no metric was measured (#3539 A6) — the viewer's + /// FleetRollup.BuildReason spells it identically, so the two surfaces read alike. + internal const string NoMetricMeasuredReason = "no metric measured yet"; + /// The card's status word. Delegates to the one ladder every Darling surface renders (#2473): /// this file's own copy agreed with the WPF card, but the WPF sidebar row's copy did not, and three /// agreeing copies plus one that does not is still four places where the answer is decided. @@ -1068,6 +1217,33 @@ private static async Task> ReadDeadlocksAsync( return map; } + /// The PostgreSQL twin of (#3539) — same carrier, same window, + /// the engine's own counter behind it. Read for the whole fleet in one pass like every other per-metric + /// read here; a SQL Server has no row in pg_database_stats and simply does not appear. + private static async Task> ReadPgDeadlocksAsync( + NpgsqlDataSource postgres, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) + { + var map = new Dictionary(); + await using var command = postgres.CreateCommand(FleetPgDeadlockSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + AddTimestamp(command, startUtc); + AddTimestamp(command, endUtc); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + /* The SUM comes back as bigint; the card's count is an int like the SQL Server arm's. A window + whose clamped deadlock differences overflow int is not a reading this card can render either + way, so saturate rather than wrap - a wrapped count could band a catastrophe Healthy. */ + var count = reader.IsDBNull(1) ? 0L : Convert.ToInt64(reader.GetValue(1)); + map[reader.GetInt32(0)] = new PgDeadlockRow( + (int)Math.Min(count, int.MaxValue), + reader.IsDBNull(2) ? null : reader.GetDateTime(2), + reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3))); + } + + return map; + } + private static async Task> ReadLastCollectionAsync(NpgsqlDataSource postgres, DateTime now, CancellationToken cancellationToken) { var map = new Dictionary(); @@ -1118,14 +1294,20 @@ private static async Task> ReadFailingCollector counts[serverId] = new CollectorCounts( existing.Healthy + (status == "HEALTHY" ? 1 : 0), existing.Failing + (status == "FAILING" ? 1 : 0), - /* #3017: the ONE collector whose band the deadlock total's coverage turns on, kept - alongside the Healthy/Failing tallies because it comes out of the same aggregate — no - extra round trip, which is what keeps this reader's fan-out bounded. Named from the + /* #3539 A8d: every banded row, whatever its band — the share's denominator. */ + existing.Total + 1, + /* #3017: the ONE collector per engine whose band the deadlock total's coverage turns on, + kept alongside the Healthy/Failing tallies because it comes out of the same aggregate — + no extra round trip, which is what keeps this reader's fan-out bounded. Named from the collector rather than as a literal so a rename cannot leave this silently matching - nothing and reporting every server uncovered. */ + nothing and reporting every server uncovered. Both engines' bands are kept on every + server because this aggregate does not know the engine; BuildCard picks. */ string.Equals(health.CollectorName, DeadlocksCollector.Instance.Name, StringComparison.Ordinal) ? status - : existing.DeadlockBand); + : existing.DeadlockBand, + string.Equals(health.CollectorName, PgDatabaseStatsCollector.Instance.Name, StringComparison.Ordinal) + ? status + : existing.PgDeadlockBand); } return counts; @@ -1154,12 +1336,25 @@ internal readonly record struct PgCpuRow( internal readonly record struct ThreadsRow(int? TotalThreads, int? CurrentWorkers, int RunnableTasks, long WorkQueue); internal readonly record struct BlockingRow(int XeCount, long XeMaxWait, int DmvCount, long DmvMaxWait); internal readonly record struct DeadlockRow(int Count, DateTime? LastSeen); + /// The PostgreSQL deadlock reading (#3539): the summed counter differences, the sample that + /// showed the newest step, and — how many differences the sum was taken + /// over. Its default is zero intervals, which reads as unmeasured: a + /// PostgreSQL target that fell out of the read (no rows in the window) must band Unknown, and a + /// struct whose default meant "measured zero" would make that the quiet outcome of a miss. + internal readonly record struct PgDeadlockRow(int Count, DateTime? LastSeen, long Intervals); /// The deadlocks collector's own 7-day band for this server, or null /// when that collector left no row in the health window at all (#3017). Null and /// mean the same thing to a reader and take the same /// action, but they arrive differently: null is the absent GROUP, NEVER_RUN would be a present group with /// no runs in it. - internal readonly record struct CollectorCounts(int Healthy, int Failing, string? DeadlockBand = null); + /// The pg_database_stats collector's band, same terms (#3539) — the + /// deadlock-source collector on a PostgreSQL target, where is always + /// null because that engine has no deadlocks collector. + /// Every collector banded for this server in the health window, on any band (#3539 + /// A8d) — the denominator grades the failing count + /// against. Healthy + Failing is NOT it: STALE, WARNING, STOPPED, NO_PERMISSIONS and EXTENSION_MISSING rows + /// are all banded collectors that are neither. + internal readonly record struct CollectorCounts(int Healthy, int Failing, int Total, string? DeadlockBand = null, string? PgDeadlockBand = null); } /// @@ -1331,10 +1526,37 @@ public sealed class FleetServerCard [JsonPropertyName("blocking_count")] public int BlockingCount { get; init; } [JsonPropertyName("max_blocking_wait_ms")] public long MaxBlockingWaitMs { get; init; } + /// Blocking events per HOUR over the card's window — the figure the count arm of + /// blocking_severity banded on (#3539 A3), or null when the window was too short to normalise. + /// Published beside the raw count for 's reason. + [JsonPropertyName("blocking_rate_per_hour")] public double? BlockingRatePerHour { get; init; } [JsonPropertyName("blocking_severity")] public HealthSeverity BlockingSeverity { get; init; } + /// The window blocking_count covers (#3539 A3) — carried, not serialized, for the reason + /// is; the roll-up already publishes the bounds once. + [JsonIgnore] + public TimeSpan BlockingWindow { get; init; } + + /// Deadlocks in the card's window, from the engine's own instrument: captured deadlock graphs + /// on SQL Server, the pg_stat_database.deadlocks counter differenced per database and summed on + /// PostgreSQL (#3539). says which, and whether the collector behind it was + /// actually running. [JsonPropertyName("deadlock_count")] public int DeadlockCount { get; init; } + + /// The newest deadlock in the window — the graph's own timestamp on SQL Server; on PostgreSQL + /// the sample that first showed the counter step, so "within the preceding minute". [JsonPropertyName("deadlock_last_seen")] public DateTime? DeadlockLastSeen { get; init; } + + /// Whether is a measurement this card banded on (#3539) — always + /// on SQL Server (a graph count is an observation even at zero), and on PostgreSQL only when at least + /// one counter difference was taken in the window (a difference of fewer than two samples is no + /// reading). Carried, not serialized: deadlock_rate_per_hour is null and + /// deadlock_severity is Unknown exactly when this is false on a rateable window, so the wire + /// already says it; this is for , which must hand the re-band the same + /// null the card banded on. Defaults to false so a card built by a path that did not decide reads + /// unmeasured — the direction that cannot claim health. + [JsonIgnore] + public bool DeadlockMeasured { get; init; } /// Deadlocks per HOUR over the card's window — the figure deadlock_severity banded on /// (#3368), or null when the window was too short to normalise. Published beside the raw count because a /// card that bands on a number it does not show leaves a reader unable to tell which tier was @@ -1356,11 +1578,11 @@ public sealed class FleetServerCard [JsonIgnore] public DeadlockRateThresholds DeadlockRateThresholds { get; init; } - /// This server's deadlocks collector band over the trailing seven days of collection - /// health (#3017) — the fact that explains a of zero. Null when that + /// This server's deadlock-source collector band over the trailing seven days of collection + /// health (#3017) — the fact that explains a of zero. The collector is + /// deadlocks on SQL Server and pg_database_stats on PostgreSQL (#3539). Null when that /// collector left no row in the health window, which is itself the answer rather than the absence of - /// one: nothing was read for this server. A PostgreSQL target has no deadlocks collector at all, - /// so it is null there too and answers on the engine instead. + /// one: nothing was read for this server. [JsonPropertyName("deadlock_collector_band")] public string? DeadlockCollectorBand { get; init; } /// Whether read a deadlock source for this server, and when it did @@ -1385,10 +1607,33 @@ public sealed class FleetServerCard [JsonPropertyName("healthy_collector_count")] public int HealthyCollectorCount { get; init; } [JsonPropertyName("failed_collector_count")] public int FailedCollectorCount { get; init; } + /// Every collector banded for this server in the health window, on any band (#3539 A8d) — the + /// denominator collector_severity grades failed_collector_count against. Not + /// healthy + failed: STALE, WARNING, STOPPED and the permission bands are banded collectors that are + /// neither. Zero with nothing failing bands collector_severity Unknown, not Healthy (#3539 + /// A6): no collector has been banded for this server, so there is no collection to call clean. + [JsonPropertyName("collector_count")] public int CollectorCount { get; init; } [JsonPropertyName("collector_severity")] public HealthSeverity CollectorSeverity { get; init; } + /// The worst per-metric band, or Unknown when NOT ONE metric on the card was measured (#3539 + /// A6) — which band then reads as Warning, the never-collected server's band, rather than + /// Healthy. [JsonPropertyName("overall_metric_severity")] public HealthSeverity OverallMetricSeverity { get; init; } + /// How many of the card's per-metric severities carried a real reading when it banded (#3528) + /// — the band's fold skips Unknown, so a card can read Healthy off one measured metric of six. When + /// this is below , the band label deserves the qualifier ("Healthy — 1 of 6 + /// measured"); the web fleet page renders exactly that. Purely descriptive: it feeds neither the band + /// nor the worst-first score, so rank-neutrality of Unknown is unchanged — with the one exception + /// #3539 A6 draws at zero: a card measuring NOTHING is not Healthy (see overall_metric_severity), + /// and its reason says so. + [JsonPropertyName("measured_metric_count")] public int MeasuredMetricCount { get; init; } + + /// The denominator for — how many per-metric severities the + /// card carries at all. Published rather than assumed at six so a consumer never hardcodes a figure the + /// next metric row changes. + [JsonPropertyName("metric_count")] public int MetricCount { get; init; } + /// The card's raw per-metric inputs, for re-scoring in the rollup (not serialized). [JsonIgnore] public ServerHealthMetrics ToHealthMetricsValue => ToHealthMetrics(); @@ -1408,7 +1653,12 @@ deliberately left as zeros (#3017) — reading them here would hand the ranking HasMemoryPressure = ServerMetricSources.DmvSourced(HasMemoryPressure, IsPostgres), BlockingCount = ServerMetricSources.DmvSourced(BlockingCount, IsPostgres), MaxBlockedSeconds = MaxBlockingWaitMs / 1000.0, - DeadlockCount = ServerMetricSources.DmvSourced(DeadlockCount, IsPostgres), + /* #3539 A3: the count's denominator, for the same reason the deadlock window travels below. */ + BlockingWindow = BlockingWindow, + /* The engine's own instrument's count since #3539, handed to the re-band exactly as the card + banded it: measured on every SQL Server card, and on a PostgreSQL card only when a difference + was taken - see DeadlockMeasured. */ + DeadlockCount = DeadlockMeasured ? DeadlockCount : null, /* #3368: the three travel together for the reason the CPU trio above does. Without the window the re-band would have no denominator and the worst-first score would rank every deadlocking server at Warning; without the tiers it would rank them against the shipped pair while the card's own @@ -1420,6 +1670,8 @@ dot used the store's. */ ThreadsWaitingForCpu = ThreadsWaitingForCpu, RequestsWaitingForThreads = RequestsWaitingForThreads, FailedCollectorCount = FailedCollectorCount, + /* #3539 A8d: the share's denominator, or the re-band would grade presence-flat again. */ + CollectorCount = CollectorCount, }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs index c006a8aa0..8e17a3ea5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingHealthReader.cs @@ -12,8 +12,8 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Analysis.Baselines; using PerformanceMonitor.Common; - using PerformanceMonitor.Darling.Storage; namespace PerformanceMonitor.Darling.Service.Mcp; @@ -43,6 +43,16 @@ internal static class DarlingHealthReader public sealed record ServerSummaryReadResult( double? CpuPercent, double? MemoryMb, int BlockingCount, int DeadlockCount, DateTime? LastCollectionTime) { + /// The collection_time of the CPU snapshot came from (#3541 A10) + /// — its own clock, distinct from , which is the newest collection of + /// ANY collector for the server. init rather than positional so the positional shape existing + /// callers construct is unchanged. Null when there is no CPU row. + public DateTime? CpuCapturedAt { get; init; } + + /// The collection_time of the memory snapshot came from (#3541 + /// A10). Null when there is no memory row. + public DateTime? MemoryCapturedAt { get; init; } + /// True when the server has no collected data at all (no CPU/memory snapshot and no collection /// log) — the tool surfaces the #1224 "unavailable" miss instead of an all-zero card. public bool HasNoData => @@ -55,15 +65,16 @@ public sealed record ServerSummaryReadResult( /// sample_time as the within-batch tiebreak, and no time predicate because sample_time is /// the monitored server's local wall clock. public const string ServerSummaryCpuSql = @" -SELECT sqlserver_cpu_utilization +SELECT sqlserver_cpu_utilization, collection_time FROM v_cpu_utilization_stats WHERE server_id = $1 ORDER BY collection_time DESC, sample_time DESC LIMIT 1"; - /// Latest total server memory (MB) for one server. $1 server_id. + /// Latest total server memory (MB) for one server, with the snapshot's own collection_time + /// (#3541 A10). $1 server_id. public const string ServerSummaryMemorySql = @" -SELECT CAST(total_server_memory_mb AS double precision) +SELECT CAST(total_server_memory_mb AS double precision), collection_time FROM v_memory_stats WHERE server_id = $1 ORDER BY collection_time DESC @@ -84,18 +95,27 @@ ORDER BY collection_time DESC public const string ServerSummaryLastCollectionSql = @" SELECT MAX(collection_time) FROM v_collection_log WHERE server_id = $1"; + /// The span the blocking and deadlock counts cover, ending at the read's clock — Lite's window. + /// Published by the tool so "recent" has a number. + public const int ServerSummaryCountsWindowHours = 1; + /// /// One server's one-shot health summary — the viewer's GetServerSummaryAsync reduced to the subset /// the same-named Lite tool serves. Blocking / deadlock counts use a one-hour window (Lite's window); CPU - /// and memory take the newest snapshot. + /// and memory take the newest snapshot, each carrying its own collection_time (#3541 A10) — the + /// payload used to publish ONE clock (last_collection, the newest collection of ANY collector) beside + /// two figures it did not stamp, so a CPU row from a collector that died yesterday read as current + /// because the collection log was fresh from the collectors still running. /// public static async Task GetServerSummaryAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) { - var windowStart = DateTime.UtcNow.AddHours(-1); + var windowStart = DateTime.UtcNow.AddHours(-ServerSummaryCountsWindowHours); double? cpuPercent = null; + DateTime? cpuCapturedAt = null; double? memoryMb = null; + DateTime? memoryCapturedAt = null; var blockingCount = 0; var deadlockCount = 0; DateTime? lastCollection = null; @@ -108,6 +128,7 @@ public static async Task GetServerSummaryAsync( if (await reader.ReadAsync(cancellationToken)) { cpuPercent = reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0)); + cpuCapturedAt = reader.GetDateTime(1); } } @@ -119,6 +140,7 @@ public static async Task GetServerSummaryAsync( if (await reader.ReadAsync(cancellationToken)) { memoryMb = reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0)); + memoryCapturedAt = reader.GetDateTime(1); } } @@ -160,7 +182,11 @@ public static async Task GetServerSummaryAsync( } } - return new ServerSummaryReadResult(cpuPercent, memoryMb, blockingCount, deadlockCount, lastCollection); + return new ServerSummaryReadResult(cpuPercent, memoryMb, blockingCount, deadlockCount, lastCollection) + { + CpuCapturedAt = cpuCapturedAt, + MemoryCapturedAt = memoryCapturedAt, + }; } /* ═══════════════════════════ daily summary ═══════════════════════════ */ @@ -172,19 +198,68 @@ public sealed record DailySummaryReadRow( long BlockingEvents, long HighCpuEvents, long CollectionErrors, long MemoryPressureEvents, long MemoryCriticalEvents, long AlertCount, long MaxBlockDurationMs, bool HasData) { + /// The store's deadlock-rate tiers (#3368/#3525) — stamped by the calendar-day reads so + /// bands on the pair get_alert_settings reports rather than the + /// shipped defaults. The default is the shipped pair, which is what a store at its V120 column + /// defaults holds anyway. + public DeadlockRateThresholds RateTiers { get; init; } = DeadlockRateThresholds.Default; + + /// The clock the still-forming day's window clamps against (#3525 review): anchored + /// reads hand their resolved window end so a backdated as_of clamps against its own "now"; + /// unanchored reads (the explicit-date tool, the viewer path) band against the wall clock. + public DateTime ReferenceUtc { get; init; } = DateTime.UtcNow; + + /// Collector runs of every status in the window (#3539 A2) — the denominator the + /// collection-error share bands on. An init member rather than a positional parameter so the + /// positional shape every existing constructor call uses is unchanged; the reader stamps it from the + /// trailing collection_runs column. + public long CollectionRuns { get; init; } + + /// + /// Whether this row's counts are a measurement or the shape retention left behind (#3541 A9) — see + /// . Stamped by the range reader from the day, the run count and the + /// store's retention horizon; the default is Collected so a row constructed without a reader (the + /// tests' hand-built rows, the fleet sweep's) bands as it always did. + /// + public DailySummaryDataState DataState { get; init; } = DailySummaryDataState.Collected; + + /// The horizon was judged against, carried so the single-day tool can + /// publish it beside a purged verdict; null on a row nobody judged. + public DateTime? RetentionHorizon { get; init; } + + /// How many of the seven per-signal sources hold at least one row for the day (#3541 A9) — + /// the aggregate's trailing signal_sources_present column, the fact that tells a purged shell + /// from a day the purge has not reached. + public int SignalSourcesPresent { get; init; } + public DailyHealthSignals ToSignals() => new() { - HasData = HasData, + /* #3541 A9: a purged or past-horizon day is a NoData day to the band, whatever the spine still + holds for it — the COALESCEd zeros it carries may be absences, and measured-zero-Healthy was the + lie. HasData alone said "a spine row exists", which the collection log's longer horizon made + true for a whole second month of purged signals. Inside retention (Collected, NoRunRecord) a + zero IS a measurement and the band stands. */ + HasData = HasData && DataState is not (DailySummaryDataState.Purged or DailySummaryDataState.PastHorizon), Deadlocks = DeadlockCount, CollectionErrors = CollectionErrors, + CollectionRuns = CollectionRuns, HighCpuEvents = HighCpuEvents, BlockingEvents = BlockingEvents, + /* #3539 A2: the day bands blocking through the card's BlockingSeverity, whose wait arm reads the + longest block — so the peak travels in the signals rather than only into the reasons line. */ + PeakBlockWaitMs = MaxBlockDurationMs, MemoryPressureEvents = MemoryPressureEvents, MemoryCriticalEvents = MemoryCriticalEvents, AlertCount = AlertCount, + /* #3525: a finished calendar day bands over its full 24 hours; the still-forming day clamps + to its elapsed portion against ReferenceUtc, or an active storm dilutes against hours that + have not happened yet (review finding on #3525). The fleet sweep does NOT read this + projection: it sums this row type's raw counts into signals windowed to its own span. */ + Window = DailyHealthBandCalculator.CalendarDayWindow(SummaryDate, ReferenceUtc), }; - public DailyHealthBand HealthBand => DailyHealthBandCalculator.Classify(ToSignals()); + public DailyHealthBand HealthBand => + DailyHealthBandCalculator.Classify(ToSignals(), new DailyHealthThresholds { DeadlockRates = RateTiers }); /// Human label for the band ("Healthy" / "Warning" / "Critical" / "No Data"). public string OverallHealth => DailyHealthBandCalculator.Label(HealthBand); @@ -197,15 +272,99 @@ public sealed record DailySummaryReadRow( /// public const string DailySummaryRangeSql = DailySummarySql.RangeSql; + /// The range read's rows plus the horizon they were judged against (#3541 A9). + /// One row per day the spine holds, oldest first, each stamped with its . + /// The oldest UTC day every signal source still holds — . + /// The retention (days) the horizon was computed from: the shortest effective horizon among the sources. + public sealed record DailySummaryRangeReadResult(List Rows, DateTime RetentionHorizon, int ShortestRetentionDays); + + /// + /// The collectors whose tables the daily aggregate reads as SIGNALS, by their schedule names — the + /// sources whose retention decides the horizon (#3541 A9). The collection log and the alert log are the + /// other two spine members; they are constants on and are folded in by + /// . query_stats is included even though old windows route + /// its CTE to a rollup with its own longer retention: the horizon is a floor over EVERY signal, and the + /// rollup keeps only the query count, not the band's inputs. + /// + internal static readonly string[] DailySummarySignalCollectors = + { + "wait_stats", "query_stats", "deadlocks", "blocked_process_report", "dmv_blocking_snapshot", + "cpu_utilization", "memory_pressure_events", + }; + + /// + /// The FLEET-WIDE retention overrides (server_id NULL) for the signal collectors — the same rows + /// StoreConfigProvider.ResolveFleetRetentionDays layers over CollectorScheduleDefaults for + /// the purge itself, so the horizon this reader publishes is the horizon the purge actually enforces + /// rather than the shipped default. A per-server override cannot apply to a shared-table purge, which is + /// why only fleet rows are read. $1 the collector names. + /// + public const string FleetRetentionOverridesSql = """ + SELECT collector_name, retention_days + FROM config_collector_schedules + WHERE server_id IS NULL + AND retention_days IS NOT NULL + AND collector_name = ANY($1) + """; + + /// + /// The shortest effective retention among the daily aggregate's sources, in days — the number the + /// horizon is measured back from. Pure: is the collector → + /// retention_days map the store holds (empty on an untouched store). + /// + /// The signal collectors resolve through + /// so an operator-shortened or -lengthened retention moves the horizon with it, with the same floor the + /// purge applies to the two baseline-serving raw tables ( for + /// cpu_utilization). The collection log and the alert log are folded in at their constants; on a + /// default store they are the LONGER horizons (60 and 90 days), which is exactly why a spine row can + /// outlive its signals and why the shortest one is the horizon. + /// + internal static int ShortestSignalRetentionDays(IReadOnlyList fleetOverrides) + { + var shortest = Math.Min(DarlingRetention.CollectionLogRetentionDays, DarlingRetention.AlertHistoryRetentionDays); + foreach (var collector in DailySummarySignalCollectors) + { + var days = StoreConfigProvider.ResolveFleetRetentionDays(collector, fleetOverrides); + if (DarlingRetention.BaselineServingRawCollectors.Contains(collector)) + { + days = Math.Max(days, BaselineMath.BaselineWindowDays); + } + + shortest = Math.Min(shortest, days); + } + + return Math.Max(1, shortest); + } + + private static async Task> ReadFleetRetentionOverridesAsync( + NpgsqlDataSource postgres, CancellationToken cancellationToken) + { + var overrides = new List(); + await using var command = postgres.CreateCommand(FleetRetentionOverridesSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + command.Parameters.Add(new NpgsqlParameter { TypedValue = DailySummarySignalCollectors }); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + overrides.Add(new ScheduleOverride(null, reader.GetString(0), null, reader.GetInt32(1), true, null)); + } + + return overrides; + } + /// One per collected day in the half-open [fromDate, toDate) /// window (the viewer's GetDailySummaryRangeAsync). /// /// #1661: routes to the same retention tier the viewer's calendar does. This matters beyond /// correctness — the calendar and this MCP tool answer the same question, so if only one routed they would /// report different query counts for the same day and there would be no way to tell which was right. + /// + /// #3541 A9: returns the rows AND the retention horizon they were judged against — see + /// and the horizon note in the body. /// - public static async Task> GetDailySummaryRangeAsync( - NpgsqlDataSource postgres, int serverId, DateTime fromDate, DateTime toDate, CancellationToken cancellationToken = default) + public static async Task GetDailySummaryRangeAsync( + NpgsqlDataSource postgres, int serverId, DateTime fromDate, DateTime toDate, + DateTime? referenceUtc = null, CancellationToken cancellationToken = default) { /* #1664: gate the age decision on the rollups actually existing — a plain-PostgreSQL store has none (and never drops raw, so raw is complete there). #1759: and on what they have MATERIALIZED, which is @@ -220,6 +379,21 @@ runs at human/model cadence and these are two small lookups. */ DateTime.UtcNow, fromDate, rollups.QueryGrainHourly, rollups.QueryGrainDaily, coverage.For(TimescaleSupport.QueryStatsHourlyView, TimescaleSupport.QueryStatsDailyView)); + /* #3525: the deadlock-rate tiers the day band evaluates, read ONCE per range rather than per row — + DarlingFleetReader's own hoist argument: a settings write mid-read must not band some days on the + old pair and the rest on the new one. */ + var rateTiers = await ReadDeadlockRateThresholdsAsync(postgres, cancellationToken); + + /* #3541 A9: the retention horizon, from the store's effective retention and the READER's wall clock. + The clock is deliberately NOT the caller's anchor — a purge is a wall-clock event and a backdated + as_of cannot un-purge a table; anchoring the horizon to as_of would let "as_of 25 days ago, + days_back 30" paint the purged stretch green again, which is the defect. The anchor still governs + the WINDOW (fromDate/toDate above) and the still-forming day's clamp (ReferenceUtc below); the + horizon is a property of the store. DailySummaryRetention.HorizonFor documents the date arithmetic. */ + var fleetOverrides = await ReadFleetRetentionOverridesAsync(postgres, cancellationToken); + var shortestRetentionDays = ShortestSignalRetentionDays(fleetOverrides); + var horizon = DailySummaryRetention.HorizonFor(DateTime.UtcNow, shortestRetentionDays); + var results = new List(); await using var command = postgres.CreateCommand(DailySummarySql.RangeSqlFor(tier)); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; @@ -230,10 +404,35 @@ runs at human/model cadence and these are two small lookups. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - results.Add(ReadDailySummaryRow(reader)); + var row = ReadDailySummaryRow(reader); + results.Add(row with + { + RateTiers = rateTiers, + ReferenceUtc = referenceUtc ?? DateTime.UtcNow, + DataState = DailySummaryRetention.StateFor(row.SummaryDate, row.CollectionRuns, row.SignalSourcesPresent, horizon), + RetentionHorizon = horizon, + }); } - return results; + return new DailySummaryRangeReadResult(results, horizon, shortestRetentionDays); + } + + /// The deadlock band's tiers from the store's singleton settings row (#3368, V120), or the + /// shipped pair when the row is absent — DarlingFleetReader.ReadDeadlockRateThresholdsAsync's + /// read, off the same published SQL, for the DAY surfaces (#3525). Values come back RAW; + /// clamps on read. + private static async Task ReadDeadlockRateThresholdsAsync( + NpgsqlDataSource postgres, CancellationToken cancellationToken) + { + await using var command = postgres.CreateCommand(DarlingFleetReader.FleetDeadlockRateThresholdSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (await reader.ReadAsync(cancellationToken)) + { + return new DeadlockRateThresholds(reader.GetDouble(0), reader.GetDouble(1)); + } + + return DeadlockRateThresholds.Default; } /// @@ -249,7 +448,8 @@ runs at human/model cadence and these are two small lookups. */ /// calendar, get_daily_summary and now the sweep all band from, so the sweep's verdicts and /// the day surfaces cannot disagree about the same signals. The SQL buckets by UTC day, so a span /// crossing midnight returns one row per day touched; the caller sums the rows, which is exact - /// because every signal is an additive count over the same half-open window. + /// because every COUNT is additive over the same half-open window — and takes the MAX of the one + /// magnitude, the peak block wait (#3539 A2), which is exact for the same reason. /// /// Throws on a store fault, deliberately — the engine-read posture /// (FleetSweepStore.GetLatestSweepAsync's reasoning): the sweep's caller must render a @@ -281,10 +481,16 @@ public static async Task GetDailySummaryAsync( NpgsqlDataSource postgres, int serverId, DateTime? summaryDate = null, CancellationToken cancellationToken = default) { var targetDate = summaryDate?.Date ?? DateTime.UtcNow.Date; - var rows = await GetDailySummaryRangeAsync(postgres, serverId, targetDate, targetDate.AddDays(1), cancellationToken); - return rows.Count > 0 - ? rows[0] - : new DailySummaryReadRow(targetDate, 0m, "", 0, 0, 0, 0, 0, 0, 0, 0, 0, HasData: false); + var range = await GetDailySummaryRangeAsync(postgres, serverId, targetDate, targetDate.AddDays(1), cancellationToken: cancellationToken); + return range.Rows.Count > 0 + ? range.Rows[0] + : new DailySummaryReadRow(targetDate, 0m, "", 0, 0, 0, 0, 0, 0, 0, 0, 0, HasData: false) + { + /* A day the spine does not hold at all is not "collected" either: before the horizon it is + purged like any other, inside it simply without a run record — so the single-day tool can say which. */ + DataState = DailySummaryRetention.StateFor(targetDate, 0, 0, range.RetentionHorizon), + RetentionHorizon = range.RetentionHorizon, + }; } private static DailySummaryReadRow ReadDailySummaryRow(DbDataReader reader) => new( @@ -300,5 +506,12 @@ public static async Task GetDailySummaryAsync( reader.IsDBNull(9) ? 0L : Convert.ToInt64(reader.GetValue(9)), reader.IsDBNull(10) ? 0L : Convert.ToInt64(reader.GetValue(10)), reader.IsDBNull(11) ? 0L : Convert.ToInt64(reader.GetValue(11)), - HasData: true); + HasData: true) + { + /* #3539 A2: the trailing collection_runs column, appended after peak_block_wait_ms so the eleven + positional reads above stay where they were. */ + CollectionRuns = reader.IsDBNull(12) ? 0L : Convert.ToInt64(reader.GetValue(12)), + /* #3541 A9: the signal-presence count, after collection_runs. */ + SignalSourcesPresent = reader.IsDBNull(13) ? 0 : Convert.ToInt32(reader.GetValue(13)), + }; } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLatchSpinlockReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLatchSpinlockReader.cs index 742183164..b9c0cb696 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLatchSpinlockReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLatchSpinlockReader.cs @@ -23,11 +23,14 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// /// Darling's latch_stats / spinlock_stats collectors are cumulative-counter delta collectors -/// (like wait_stats) that store the last interval's delta_* directly but NO -/// sample_interval_seconds, so the per-second rate is derived in SQL from the per-class LAG -/// interval (the truncate-then-diff epoch idiom the viewer's trend reads use). Each read aggregates the -/// window per class into one row (the top N by total delta wait time / total delta collisions), carrying the -/// latest interval's per-second rate and last delta. The Dashboard's severity / latch_description +/// (like wait_stats) that store the last interval's delta_* beside the measured +/// sample_interval_seconds it accrued over (V127, #3540 — before which the per-second rate could only +/// be derived from the per-class LAG interval, the truncate-then-diff epoch idiom the viewer's trend +/// reads use, and that derivation remains the fallback for pre-V127 rows). Each read aggregates the window +/// per class into one row (the top N by total delta wait time / total delta collisions), carrying the latest +/// interval's per-second rate and last delta; the rate is null when the latest interval was unknowable (a +/// stored 0: first sighting, counter reset, gap past the policy) rather than the 0.00 a restart used to read +/// as. The Dashboard's severity / latch_description /// / recommendation and spinlock_description are NOT stored columns — they are the Dashboard /// view's own CASE derivations (deterministic functions of latch_class / spinlock_name and the /// latest delta), reproduced VERBATIM as pure static helpers here so the tools serve the FULL Dashboard result @@ -40,25 +43,29 @@ internal static class DarlingLatchSpinlockReader /* ─────────────────────────── result rows ─────────────────────────── */ /// One latch class aggregated over the window: the summed deltas plus the latest interval's - /// per-second rate and last delta wait (the severity input). + /// per-second rate and last delta wait (the severity input). is the + /// span that last delta accrued over (#3541 A10 — the interval the severity band was computed from, published + /// beside the window totals so the two are distinguishable); null when the interval was unknowable. public sealed record LatchStatRow( string LatchClass, long TotalDeltaWaitTimeMs, long TotalDeltaWaitingRequests, - double WaitsPerSecond, double WaitMsPerSecond, long LatestDeltaWaitTimeMs, DateTime LatestCollectionTime); + double? WaitsPerSecond, double? WaitMsPerSecond, long LatestDeltaWaitTimeMs, DateTime LatestCollectionTime, + double? LatestIntervalSeconds); /// One spinlock aggregated over the window: the summed deltas plus the latest interval's /// per-second collision/spin rates. public sealed record SpinlockStatRow( string SpinlockName, long TotalDeltaCollisions, long TotalDeltaSpins, long TotalDeltaBackoffs, - double CollisionsPerSecond, double SpinsPerSecond, DateTime LatestCollectionTime); + double? CollisionsPerSecond, double? SpinsPerSecond, DateTime LatestCollectionTime); /* ─────────────────────────── latch stats (top N over the window) ─────────────────────────── */ /// /// The top-N latch classes over the window, one row per class — mirroring the Dashboard's /// get_latch_stats per-class aggregation (SUM of the last interval's deltas, top by total delta - /// wait time). The per-second rate comes from the latest interval via the per-class LAG interval - /// (Darling stores no sample_interval_seconds), the same idiom the viewer's LatchTrendSql - /// uses. latest_delta_wait_time_ms feeds the reproduced severity CASE. Runs on the + /// wait time). The per-second rate comes from the latest interval's stored sample_interval_seconds + /// (the per-class LAG interval only for pre-V127 rows), the same idiom the viewer's + /// LatchTrendSql uses; null when that interval was unknowable. latest_delta_wait_time_ms + /// feeds the reproduced severity CASE. Runs on the /// v_latch_stats passthrough view. $1 server_id, $2 window start, $3 window end (naive UTC), $4 top. /// public const string LatchStatsTopNSql = """ @@ -69,7 +76,13 @@ WITH windowed AS collection_time, delta_waiting_requests_count, delta_wait_time_ms, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY latch_class ORDER BY collection_time)))) AS interval_seconds + /* #3540: the STORED interval where the row has one; 0 (no delta knowable) becomes NULL through NULLIF + so the latest-interval rates below are NULL — reported as null, never 0.00 — when the newest + collection was a restart. NULL (a pre-V127 row) falls back to the LAG. */ + CASE WHEN sample_interval_seconds IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY latch_class ORDER BY collection_time)))) + ELSE NULLIF(sample_interval_seconds, 0) + END AS interval_seconds FROM v_latch_stats WHERE server_id = $1 AND collection_time >= $2 @@ -90,8 +103,11 @@ latest AS SELECT DISTINCT ON (latch_class) latch_class, delta_wait_time_ms AS latest_delta_wait_time_ms, - CASE WHEN interval_seconds > 0 THEN CAST(delta_waiting_requests_count AS double precision) / interval_seconds ELSE 0 END AS waits_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS double precision) / interval_seconds ELSE 0 END AS wait_ms_per_second + CASE WHEN interval_seconds > 0 THEN CAST(delta_waiting_requests_count AS double precision) / interval_seconds END AS waits_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(delta_wait_time_ms AS double precision) / interval_seconds END AS wait_ms_per_second, + /* #3541 A10: the span the severity-banded delta accrued over, so the tool can publish the + interval the band came from beside the window totals it does NOT come from. */ + CASE WHEN interval_seconds > 0 THEN CAST(interval_seconds AS double precision) END AS latest_interval_seconds FROM windowed ORDER BY latch_class, collection_time DESC ) @@ -102,7 +118,8 @@ FROM windowed l.waits_per_second, l.wait_ms_per_second, l.latest_delta_wait_time_ms, - a.latest_collection_time + a.latest_collection_time, + l.latest_interval_seconds FROM agg AS a JOIN latest AS l ON l.latch_class = a.latch_class ORDER BY a.total_delta_wait_time_ms DESC @@ -124,10 +141,11 @@ public static async Task> GetLatchStatsTopNAsync( reader.IsDBNull(0) ? "" : reader.GetString(0), reader.IsDBNull(1) ? 0 : reader.GetInt64(1), reader.IsDBNull(2) ? 0 : reader.GetInt64(2), - reader.IsDBNull(3) ? 0 : reader.GetDouble(3), - reader.IsDBNull(4) ? 0 : reader.GetDouble(4), + reader.IsDBNull(3) ? null : reader.GetDouble(3), + reader.IsDBNull(4) ? null : reader.GetDouble(4), reader.IsDBNull(5) ? 0 : reader.GetInt64(5), - reader.GetDateTime(6))); + reader.GetDateTime(6), + reader.IsDBNull(7) ? null : reader.GetDouble(7))); } return rows; @@ -138,8 +156,9 @@ public static async Task> GetLatchStatsTopNAsync( /// /// The top-N spinlocks over the window, one row per spinlock — the collision analog of /// , mirroring the Dashboard's get_spinlock_stats per-name - /// aggregation (top by total delta collisions). Per-second collision/spin rates from the latest interval - /// via the per-name LAG interval. Runs on v_spinlock_stats. $1 server_id, $2 start, $3 end + /// aggregation (top by total delta collisions). Per-second collision/spin rates from the latest interval's + /// stored sample_interval_seconds (per-name LAG for pre-V127 rows), null when unknowable. + /// Runs on v_spinlock_stats. $1 server_id, $2 start, $3 end /// (naive UTC), $4 top. /// public const string SpinlockStatsTopNSql = """ @@ -151,7 +170,13 @@ WITH windowed AS delta_collisions, delta_spins, delta_backoffs, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY spinlock_name ORDER BY collection_time)))) AS interval_seconds + /* #3540: the STORED interval where the row has one; 0 (no delta knowable) becomes NULL through NULLIF + so the latest-interval rates below are NULL — reported as null, never 0.00 — when the newest + collection was a restart. NULL (a pre-V127 row) falls back to the LAG. */ + CASE WHEN sample_interval_seconds IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (PARTITION BY spinlock_name ORDER BY collection_time)))) + ELSE NULLIF(sample_interval_seconds, 0) + END AS interval_seconds FROM v_spinlock_stats WHERE server_id = $1 AND collection_time >= $2 @@ -172,8 +197,8 @@ latest AS ( SELECT DISTINCT ON (spinlock_name) spinlock_name, - CASE WHEN interval_seconds > 0 THEN CAST(delta_collisions AS double precision) / interval_seconds ELSE 0 END AS collisions_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(delta_spins AS double precision) / interval_seconds ELSE 0 END AS spins_per_second + CASE WHEN interval_seconds > 0 THEN CAST(delta_collisions AS double precision) / interval_seconds END AS collisions_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(delta_spins AS double precision) / interval_seconds END AS spins_per_second FROM windowed ORDER BY spinlock_name, collection_time DESC ) @@ -207,8 +232,8 @@ public static async Task> GetSpinlockStatsTopNAsync( reader.IsDBNull(1) ? 0 : reader.GetInt64(1), reader.IsDBNull(2) ? 0 : reader.GetInt64(2), reader.IsDBNull(3) ? 0 : reader.GetInt64(3), - reader.IsDBNull(4) ? 0 : reader.GetDouble(4), - reader.IsDBNull(5) ? 0 : reader.GetDouble(5), + reader.IsDBNull(4) ? null : reader.GetDouble(4), + reader.IsDBNull(5) ? null : reader.GetDouble(5), reader.GetDateTime(6))); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLongQueryReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLongQueryReader.cs index a0392da74..6f4a71f1c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLongQueryReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingLongQueryReader.cs @@ -21,7 +21,8 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// keyed by server_id and windowed on the naive-UTC collection_time prefix, over the BASE /// long_query_completions table (a post-V14 collector has no v_* passthrough view). Ordered by /// duration DESC (NULLS LAST so attentions, whose duration is NULL, follow the ranked completions). The SQL -/// is a public const so Darling.Tests can pin the dialect + columns without a live Postgres. +/// is a public const so Darling.Tests can pin the dialect + columns without a live Postgres. $1 server_id, +/// $2/$3 window (naive UTC), $4 row cap. /// internal static class DarlingLongQueryReader { @@ -76,11 +77,22 @@ FROM long_query_completions AND collection_time >= $2 AND collection_time <= $3 ORDER BY duration_microseconds DESC NULLS LAST, event_time DESC - LIMIT 200 + LIMIT $4 """; + /// + /// The SLOWEST completions over the window (attentions, whose duration is NULL, + /// sort last). Callers detecting truncation pass limit + 1 and read the extra row as the signal. + /// + /// The cap is a PARAMETER, not a literal (#3541 A3). It was LIMIT 200 under a tool that + /// advertised limit and applied it with Take(limit) — harmless while nobody asked for more + /// than 200, invisible when they did. The ORDERING is the load-bearing half of this read and is the one + /// both SKUs now share: the population a completions tool must keep is the window's slowest, because a + /// newest-first cap re-ranked afterwards can omit the slowest run in the window entirely, which is what + /// Lite's twin did until this change. + /// public static async Task> GetRecentLongQueryCompletionsAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) { var rows = new List(); @@ -88,6 +100,7 @@ public static async Task> GetRecentLongQueryCompletionsAs await using var command = new NpgsqlCommand(LongQueryCompletionsSql, connection); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + DarlingMcpReadParameters.AddInt(command, cap); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs index f63b07c28..11bfb7c4b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpAlertTools.cs @@ -75,13 +75,17 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpAlertTools { - [McpServerTool(Name = "get_alert_history"), Description("Gets recent alert history from the alert log: what alerts fired, when, for which server, the current vs threshold value, whether email/webhook delivery succeeded, and whether the alert was muted. Omit server_name to see the whole fleet (each row names its server); pass one to scope to a single server. notification_type is the delivery disposition and is the ONLY field that says why a row did not deliver: 'email'/'webhook'/'email+webhook' delivered on that channel; 'throttled' means the delivery cooldown was still inside this alert's window so nothing was attempted (the throttle working, not a fault); 'folded' means a repeat was rolled onto another server's post for the same metric and is named there under 'Other Servers Affected', so it WAS reported; 'failed' means a channel was attempted and came back unsuccessful, with send_error carrying the first failing channel's text; 'unconfigured' means no email or webhook channel is set up; 'muted' means a mute rule suppressed it; 'none' is a resolution row, which no channel applies to. Do NOT split the not-delivered rows on send_error: it is null on 'throttled' and 'folded' rows and on every row written before those values existed, so a null error is not evidence of a working cooldown. 'undelivered' is a retained legacy value that means throttled OR folded OR failed with nothing in the row to say which — count those rows separately rather than attributing them.")] + [McpServerTool(Name = "get_alert_history"), Description("Gets recent alert history from the alert log, NEWEST FIRST: what alerts fired, when, for which server, the current vs threshold value, whether email/webhook delivery succeeded, and whether the alert was muted. Omit server_name to see the whole fleet (each row names its server); pass one to scope to a single server. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: alerts_returned is how many rows you got, truncated says the window held more than limit, and oldest_returned_alert_time / newest_returned_alert_time bound the page — under newest-first ordering the oldest stamp IS how far back this read reached, so on a noisy fleet a 24-hour request at the default limit may cover minutes. Raise limit or narrow hours_back when truncated is true; widening hours_back cannot help. BY DEFAULT THIS READ EXCLUDES DISMISSED ALERTS — rows an operator acknowledged in the Viewer's Alert History grid. Dismissal says nothing about whether the alert fired or mattered, so an incident reconstruction that ignores it can miss the very critical someone already looked at: dismissed_excluded says whether the filter applied and dismissed_excluded_count is how many rows in the window it removed, and include_dismissed = true returns them, each labelled dismissed = true. notification_type is the delivery disposition and is the ONLY field that says why a row did not deliver: 'email'/'webhook'/'email+webhook' delivered on that channel; 'throttled' means the delivery cooldown was still inside this alert's window so nothing was attempted (the throttle working, not a fault); 'folded' means a repeat was rolled onto another server's post for the same metric and is named there under 'Other Servers Affected', so it WAS reported; 'failed' means a channel was attempted and came back unsuccessful, with send_error carrying the first failing channel's text; 'unconfigured' means no email or webhook channel is set up; 'muted' means a mute rule suppressed it; 'none' is a resolution row, which no channel applies to. Do NOT split the not-delivered rows on send_error: it is null on 'throttled' and 'folded' rows and on every row written before those values existed, so a null error is not evidence of a working cooldown. 'undelivered' is a retained legacy value that means throttled OR folded OR failed with nothing in the row to say which — count those rows separately rather than attributing them. severity is the row's tier — 'critical', 'warning', 'info' or 'resolution' — and severity_source says where it came from: 'fired' when the row persisted the tier the alert actually fired at (graded alerts such as Poison Wait, Volume Free Space and Database State fire Warning OR Critical by measurement), 'metric_name' when the row carries no tier and the metric's name is the only evidence (rows written before the tier was persisted, alerts whose severity is fixed per metric, and every resolution row). Do not infer a graded alert's tier from its name: a 'Poison Wait' row with severity 'warning' fired as a warning.")] public static async Task GetAlertHistory( NpgsqlDataSource postgres, [Description("Server name or display name. Omit to return alerts across all servers (the fleet default).")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 50.")] int limit = 50, - [Description(McpHelpers.AsOfDescription)] string? as_of = null) + [Description("Maximum rows to return, newest first. Default 50. This is what bounds the page — read truncated to know whether the window held more.")] int limit = 50, + [Description(McpHelpers.AsOfDescription)] string? as_of = null, + /* Appended after as_of for the reason get_collection_log's filters are: MCP invokes by name, the + /api/read dispatch passes as_of by name, and a trailing optional is the one position no existing + positional C# caller can be re-bound by. */ + [Description("Include alerts an operator has dismissed in the Viewer. Default false, which is the Alert History grid's own read. Dismissal is an acknowledgement, not a verdict — a dismissed critical still fired — so set this when reconstructing an incident rather than triaging what is still open. Each row then carries dismissed so the two populations stay distinguishable.")] bool include_dismissed = false) { var hoursError = McpHelpers.ValidateWindow(hours_back, as_of, out var windowEnd); if (hoursError != null) return hoursError; @@ -103,30 +107,79 @@ public static async Task GetAlertHistory( try { var since = windowEnd.AddHours(-hours_back); - var rows = await DarlingAlertReader.GetAlertHistoryAsync(postgres, since, windowEnd, serverId, limit); - if (rows.Count == 0) - return McpHelpers.Status("empty", "No alerts found in the specified time range."); - var alerts = rows.Select(r => new + /* #3541 A3: over-fetch by one so truncation is OBSERVED rather than inferred from count == limit, + the pattern get_collection_log and get_query_heatmap use. The cap was already the caller's + here; what was missing was any way to tell a window of exactly `limit` alerts from a busier + one, and any statement that the dismissed rows had been removed. */ + var rows = await DarlingAlertReader.GetAlertHistoryPageAsync(postgres, since, windowEnd, serverId, limit + 1, include_dismissed); + var truncated = rows.Count > limit; + var page = truncated ? rows.Take(limit).ToList() : rows; + + /* The hidden filter, measured: how many rows in this window and scope it removed. Zero is a real + answer (nothing was hidden) and is what a caller who never sends include_dismissed most needs + to see beside a clean-looking page. Not probed when the filter is off, because then it removed + nothing by construction. */ + var dismissedExcludedCount = include_dismissed + ? 0L + : await DarlingAlertReader.CountDismissedAlertsAsync(postgres, since, windowEnd, serverId); + + if (page.Count == 0) { - alert_time = r.AlertTime.ToString("o"), - server_id = r.ServerId, - server_name = r.ServerName, - metric_name = r.MetricName, - current_value = r.CurrentValue, - threshold_value = r.ThresholdValue, - alert_sent = r.AlertSent, - notification_type = r.NotificationType, - send_error = r.SendError, - muted = r.Muted, - detail_text = r.DetailText + /* An empty default page over a window that DOES hold dismissed rows is not "no alerts": it is + "every alert here was acknowledged", and the one-sentence quiet-window answer would send + the caller off widening a window whose contents they were never shown. */ + return dismissedExcludedCount > 0 + ? McpHelpers.Status( + "empty", + $"No undismissed alerts found in the specified time range, but {dismissedExcludedCount} dismissed alert(s) were excluded by the default filter. Re-run with include_dismissed = true to see them — a dismissed alert still fired.") + : McpHelpers.Status("empty", "No alerts found in the specified time range."); + } + + var alerts = page.Select(r => + { + var (severity, severitySource) = AlertHistoryRowSeverity.Describe(r.MetricName, r.ContextJson); + return new + { + alert_time = r.AlertTime.ToString("o"), + server_id = r.ServerId, + server_name = r.ServerName, + metric_name = r.MetricName, + current_value = r.CurrentValue, + threshold_value = r.ThresholdValue, + alert_sent = r.AlertSent, + notification_type = r.NotificationType, + send_error = r.SendError, + muted = r.Muted, + /* Per row, so a page that mixes the two populations labels each one. Always false on the + default read, which is a true statement about every row on it. */ + dismissed = r.Dismissed, + /* #3539 A8e: the tier the alert FIRED at where the row persisted one ("fired"), else what + the metric NAME implies ("metric_name") — the same two arms both Alert History grids + colour rows by, so a caller reading "Poison Wait" here sees the Warning it fired at + rather than the red the name used to earn every row. The source is published because + the two are not equal evidence; see AlertHistoryRowSeverity.Describe. */ + severity, + severity_source = severitySource, + detail_text = r.DetailText, + }; }); return JsonSerializer.Serialize(new { server = scope, hours_back, - total_alerts = rows.Count, + /* #3541 A3: `total_alerts` is gone — it was the page count under a name that promised the + window. What is published is what was measured: the page, whether the window held more, + the span the page covers (newest-first, so the oldest stamp IS the reach), and the filter + that shaped the population together with how much it removed. */ + alerts_returned = page.Count, + truncated, + oldest_returned_alert_time = page.Min(r => r.AlertTime).ToString("o"), + newest_returned_alert_time = page.Max(r => r.AlertTime).ToString("o"), + order = "alert_time_desc", + dismissed_excluded = !include_dismissed, + dismissed_excluded_count = dismissedExcludedCount, alerts }, McpHelpers.JsonOptions); } @@ -136,7 +189,7 @@ public static async Task GetAlertHistory( } } - [McpServerTool(Name = "get_alert_settings"), Description("Gets the current alert configuration the service is using: which alerts are enabled and their thresholds (CPU, blocking, deadlocks, poison waits, long-running queries/jobs, tempdb, low disk, failed jobs, database state, Availability Group health, connection loss), the cooldown, excluded databases, the deadlock/blocking delivery mode and cooldown, the scheduled-analysis cadence, and the fleet-sweep cadence. TWO different cooldowns are reported and they govern different stages: top-level cooldown_minutes gates whether the alert engine FIRES at all, while delivery.cooldown_minutes bounds the resulting Slack/Teams/PagerDuty/webhook/email post twice over: once per alert FINGERPRINT, and once per METRIC across the whole fleet for a RE-notification. The second bound is why one fault on forty servers does not cost forty posts an hour; the servers it holds back are named on the post that does go out, under an 'Other Servers Affected' section. A first notice is never held back by either bound, and PerEvent delivery mode opts out of the per-metric one. A channel going quiet with alerts still in get_alert_history is delivery.cooldown_minutes, not cooldown_minutes. The self_alerts group holds the thresholds for alerts about the MONITOR STORE itself rather than a monitored server — those arrive with Server: 'Monitor Store' by default, or with the store's own peers.storeName label when the operator set that file-only field (a multi-store estate names each store on its own self-alerts), so an alert naming either spelling is tuned here and nowhere else, including Retention Held's warn/critical ratios. Mute rules match the alert row's server spelling, so on a store with storeName set, scope self-alert mutes to that label, not to 'Monitor Store'. The health_bands group is NOT an alert: its two tiers decide what band a server's card, the worst-first ranking and get_fleet_overview's counts read, in deadlocks per HOUR normalised over whatever window was asked for — so the same pair means the same condition on a 1-hour read and a 24-hour one. Tuning deadlocks.count_threshold does not move the band and tuning health_bands does not move the alert. The fleet_sweep group is NOT an alert family either, and its cadence is a SECOND cadence, separate from the scheduled-analysis one: fleet_sweep.enabled turns the scheduled whole-fleet sweep report on or off, and fleet_sweep.interval_minutes (15–1440, default 60 — hourly) is how often it runs. The alerts_enabled master switch deliberately does not govern sweep production, only delivery: sweeps keep running under alerts_enabled: false — that is when they carry the would-have-paged ledger — so muting the fleet does not blind the report surface. Separately, deadlocks.pg_count_threshold and blocking.pg_count_threshold are the PostgreSQL versions of those two alerts' count gates, reported inside those same groups, and they are deliberately NOT the same numbers as the count_threshold beside them: a PostgreSQL server has no deadlock or blocking health band to calibrate against, and its blocking count is a periodic SAMPLE of pg_stat_activity rather than engine-recorded reports. The enabled switch in each group governs BOTH engines; the two thresholds do not move each other. On a store with no PostgreSQL targets both PostgreSQL keys are inert. SMTP/webhook delivery credentials are managed separately and are not reported here — configure them in the standalone Darling Viewer app's Settings window (Notifications section), which connects to this store (including remotely, not just localhost) rather than requiring desktop access to this specific box.")] + [McpServerTool(Name = "get_alert_settings"), Description("Gets the current alert configuration the service is using: which alerts are enabled and their thresholds (CPU, blocking, deadlocks, poison waits, long-running queries/jobs, tempdb, low disk, failed jobs, database state, Availability Group health, connection loss), the cooldown, excluded databases, the deadlock/blocking delivery mode and cooldown, the scheduled-analysis cadence, and the fleet-sweep cadence. TWO different cooldowns are reported and they govern different stages: top-level cooldown_minutes gates whether the alert engine FIRES at all, while delivery.cooldown_minutes bounds the resulting Slack/Teams/PagerDuty/webhook/email post twice over: once per alert FINGERPRINT, and once per METRIC across the whole fleet for a RE-notification. The second bound is why one fault on forty servers does not cost forty posts an hour; the servers it holds back are named on the post that does go out, under an 'Other Servers Affected' section. A first notice is never held back by either bound, and PerEvent delivery mode opts out of the per-metric one. A channel going quiet with alerts still in get_alert_history is delivery.cooldown_minutes, not cooldown_minutes. The self_alerts group holds the thresholds for alerts about the MONITOR STORE itself rather than a monitored server — those arrive with Server: 'Monitor Store' by default, or with the store's own peers.storeName label when the operator set that file-only field (a multi-store estate names each store on its own self-alerts), so an alert naming either spelling is tuned here and nowhere else, including Retention Held's warn/critical ratios. Mute rules match the alert row's server spelling, so on a store with storeName set, scope self-alert mutes to that label, not to 'Monitor Store'. The health_bands group is NOT an alert: its two tiers decide what band a server's card, the worst-first ranking and get_fleet_overview's counts read, in deadlocks per HOUR normalised over whatever window was asked for — so the same pair means the same condition on a 1-hour read and a 24-hour one. Tuning deadlocks.count_threshold does not move the band and tuning health_bands does not move the alert. The file_growth group's rise_mb is megabytes per HOUR, averaged over file_growth.lookback_minutes — a rate, not a total for the window: 10240 means 10 GB/hr whether the lookback is 5 minutes or 24 hours, and the engine scales it to the window (a 5-minute lookback asks for 853 MB inside it, a 24-hour one for 240 GB). The fleet_sweep group is NOT an alert family either, and its cadence is a SECOND cadence, separate from the scheduled-analysis one: fleet_sweep.enabled turns the scheduled whole-fleet sweep report on or off, and fleet_sweep.interval_minutes (15–1440, default 60 — hourly) is how often it runs. The alerts_enabled master switch deliberately does not govern sweep production, only delivery: sweeps keep running under alerts_enabled: false — that is when they carry the would-have-paged ledger — so muting the fleet does not blind the report surface. Separately, deadlocks.pg_count_threshold and blocking.pg_count_threshold are the PostgreSQL versions of those two alerts' count gates, reported inside those same groups, and they are deliberately NOT the same numbers as the count_threshold beside them: a PostgreSQL server has no deadlock or blocking health band to calibrate against, and its blocking count is a periodic SAMPLE of pg_stat_activity rather than engine-recorded reports. The enabled switch in each group governs BOTH engines; the two thresholds do not move each other. On a store with no PostgreSQL targets both PostgreSQL keys are inert. SMTP/webhook delivery credentials are managed separately and are not reported here — configure them in the standalone Darling Viewer app's Settings window (Notifications section), which connects to this store (including remotely, not just localhost) rather than requiring desktop access to this specific box.")] public static async Task GetAlertSettings( NpgsqlDataSource postgres) { @@ -257,6 +310,10 @@ invite tuning one and expecting the other to move. */ self_alerts = new { disk_free_warn_percent = s.SelfDiskFreeWarnPercent, + /* #3528 (V126): the percent's GB floor — pressure requires BOTH the percent above breached + AND free space below this many GB (0 removes the floor), so a large store volume at a low + percent stops paging CRITICAL. The pvs.floor_gb composition, not low_disk's OR pair. */ + disk_free_warn_gb = s.SelfDiskFreeWarnGb, collection_stale_minutes = s.CollectionStaleMinutes, collection_failure_threshold = s.CollectionFailureThreshold, /* #2136: the Store Job Over Cadence warning percent (Critical is fixed at 100). */ @@ -278,6 +335,10 @@ be enabled only by UPDATEing config_alert_settings by hand. Reported by @gotqn. file_growth = new { enabled = s.FileGrowthEnabled, + /* #3539 A8c: MB per HOUR, averaged over lookback_minutes — a rate, not the in-window delta the key's + spelling suggests. The key keeps its name (a rename breaks every client that reads or writes it, + and Lite's McpAlertSettingsKeyTests derive its shape from this source); the unit is stated in + both tool descriptions, which is where an agent reads it. */ rise_mb = s.FileGrowthRiseMb, volume_percent = s.FileGrowthVolumePercent, lookback_minutes = s.FileGrowthLookbackMinutes @@ -429,7 +490,8 @@ and the second one is a mute somebody INTENDED that is no longer in force. "per hour is the tightest setting that is still a rate, so no value here can restore the 'any deadlock " + "is Critical' reading these tiers replaced. Setting critical BELOW warn is accepted and means every " + "banded rate is Critical. " + - "Two keys govern the PostgreSQL versions of the two count alerts and are NOT the same numbers as their SQL Server neighbours: deadlocks.pg_count_threshold and blocking.pg_count_threshold, both accepting 1 upward. They sit inside those groups rather than a section of their own so both engines' figures are visible together, but tuning deadlocks.count_threshold does NOT move the PostgreSQL gate and tuning deadlocks.pg_count_threshold does NOT move the SQL Server one. The enabled switch in each group DOES govern both engines. They are separate because the reason to move the SQL Server deadlock figure is agreement with health_bands.deadlock_warn_per_hour, and a PostgreSQL server has no deadlock band at all - its deadlocks are served by get_pg_deadlocks and are structurally absent from the fleet deadlock total - while on the blocking side the SQL Server count is engine-recorded blocked-process reports and the PostgreSQL one is distinct root blockers in a periodic SAMPLE of pg_stat_activity. Both PostgreSQL keys are ignored on a store with no PostgreSQL targets. " + + "file_growth.rise_mb is megabytes per HOUR averaged over file_growth.lookback_minutes (a rate — the same 10240 is 10 GB/hr on any lookback; the engine scales it to the window), so shortening the lookback does not tighten the rise gate and lengthening it does not loosen it; only the rate does. " + + "Two keys govern the PostgreSQL versions of the two count alerts and are NOT the same numbers as their SQL Server neighbours: deadlocks.pg_count_threshold and blocking.pg_count_threshold, both accepting 1 upward. They sit inside those groups rather than a section of their own so both engines' figures are visible together, but tuning deadlocks.count_threshold does NOT move the PostgreSQL gate and tuning deadlocks.pg_count_threshold does NOT move the SQL Server one. The enabled switch in each group DOES govern both engines. They are separate because the two engines count with different instruments - SQL Server's figure is captured deadlock graphs, the PostgreSQL one is deadlocks parsed from the server log (get_pg_deadlocks) - and an operator tuning one should not silently move the other. Both engines' fleet cards now band deadlocks through the SAME health_bands.deadlock_warn_per_hour tiers (the PostgreSQL card differences the server's own pg_stat_database.deadlocks counter over the window, #3539), so the #3444 move - raising a fire gate to meet the band's Warning bar so a page and an amber dot describe the same server - is available on either knob. On the blocking side the SQL Server count is engine-recorded blocked-process reports and the PostgreSQL one is distinct root blockers in a periodic SAMPLE of pg_stat_activity. Both PostgreSQL keys are ignored on a store with no PostgreSQL targets. " + "The fleet_sweep group is NOT an alert family and the alerts_enabled master switch does not govern it: " + "fleet_sweep.enabled turns the scheduled whole-fleet sweep report on or off, and " + "fleet_sweep.interval_minutes (15\u20131440, default 60) is its cadence. Sweeps deliberately keep running " + @@ -1317,7 +1379,10 @@ letting one silently win. */ switch (k) { case "enabled": AddBool("blocking_enabled", n, "blocking.enabled"); break; - case "count_threshold": AddInt("blocking_count_threshold", n, "blocking.count_threshold", 1, int.MaxValue); break; + /* #3528: the floor is the named constant rather than the literal 1 it always + was, because the read side now clamps to it — the same structural parity the + pg twin below has held since V122. */ + case "count_threshold": AddInt("blocking_count_threshold", n, "blocking.count_threshold", PostgresAlertEvaluator.CountThresholdFloor, int.MaxValue); break; /* #2417: get_alert_settings has emitted this key since #1839 and the writer never took it, so handing a whole read payload back -- the round trip this tool's own description tells the caller to perform -- was rejected with @@ -1328,8 +1393,7 @@ keeps disabling the second gate rather than becoming invalid. */ /* #3444 (V122): the PostgreSQL count gate. The floor is the SAME named constant DarlingAlertSettings clamps to, not a retyped 1, so this writer cannot ACCEPT a value the read-side clamp then rewrites. The twin above - takes the identical bound and has no read-side clamp; that gap is named on - DarlingAlertSettings rather than reproduced here. */ + holds the identical bound-and-clamp pair since #3528. */ case "pg_count_threshold": AddInt("pg_blocking_count_threshold", n, "blocking.pg_count_threshold", PostgresAlertEvaluator.CountThresholdFloor, int.MaxValue); break; default: error = $"Unknown field 'blocking.{k}'."; break; } @@ -1342,7 +1406,8 @@ DarlingAlertSettings rather than reproduced here. */ switch (k) { case "enabled": AddBool("deadlock_enabled", n, "deadlocks.enabled"); break; - case "count_threshold": AddInt("deadlock_count_threshold", n, "deadlocks.count_threshold", 1, int.MaxValue); break; + /* #3528: the named constant for its blocking sibling's reason. */ + case "count_threshold": AddInt("deadlock_count_threshold", n, "deadlocks.count_threshold", PostgresAlertEvaluator.CountThresholdFloor, int.MaxValue); break; /* #3444 (V122): the PostgreSQL count gate — same bound sourcing as its blocking sibling. */ case "pg_count_threshold": AddInt("pg_deadlock_count_threshold", n, "deadlocks.pg_count_threshold", PostgresAlertEvaluator.CountThresholdFloor, int.MaxValue); break; @@ -1443,6 +1508,9 @@ the value the sweep uses. */ switch (k) { case "disk_free_warn_percent": AddInt("self_disk_free_warn_percent", n, "self_alerts.disk_free_warn_percent", 0, 100); break; + /* #3528: bound mirrors DarlingAlertSettings' Math.Max(0, ...) — 0 is IN range + because it removes the floor (the pvs.floor_gb reading), not nonsense. */ + case "disk_free_warn_gb": AddInt("self_disk_free_warn_gb", n, "self_alerts.disk_free_warn_gb", 0, int.MaxValue); break; case "collection_stale_minutes": AddInt("collection_stale_minutes", n, "self_alerts.collection_stale_minutes", 5, 1440); break; case "collection_failure_threshold": AddInt("collection_failure_threshold", n, "self_alerts.collection_failure_threshold", 1, 1000); break; case "store_job_cadence_warn_percent": AddInt("store_job_cadence_warn_percent", n, "self_alerts.store_job_cadence_warn_percent", 5, 100); break; @@ -1484,7 +1552,8 @@ including why the ceiling exists rather than leaving the knob open. */ [0,100] on the volume percent, [5,1440] on the lookback. If these drift apart the tool accepts a value the engine then silently rewrites, which reads as the setting not sticking. Zero on either gate disables that gate rather than being invalid (#2349), - which is why the rise floor is 0 and not 1. */ + which is why the rise floor is 0 and not 1. rise_mb is MB per HOUR (#3539 A8c); the + column takes the same integer it always did, and the engine scales it to the window. */ case "file_growth": Group(prop.Value, "file_growth", (k, n) => { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs index 2696d199d..c01ef3818 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpBlockingTools.cs @@ -44,13 +44,13 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpBlockingTools { - [McpServerTool(Name = "get_blocking"), Description("Gets blocking events captured by the blocked process report extended event (plus the always-on DMV blocking-snapshot fallback). Shows the blocked and blocking sessions, wait types, wait times, and query text for both. Use this first for a quick overview, then use get_blocked_process_xml for deep analysis of prolonged blocking. Every timestamp here is UTC: event_time already was, and the six blocked_/blocking_ last_tran/last_batch stamps are de-skewed from the monitored server's local clock by this read, so comparing them against event_time to see whether a transaction predates the block is direct.")] + [McpServerTool(Name = "get_blocking"), Description("Gets blocking events captured by the blocked process report extended event (plus the always-on DMV blocking-snapshot fallback), NEWEST FIRST. Shows the blocked and blocking sessions, wait types, wait times, and query text for both. Use this first for a quick overview, then use get_blocked_process_xml for deep analysis of prolonged blocking. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: hours_back is the window you ASKED for, events_returned is how many rows you GOT, truncated says the window held more than limit, and oldest_returned_event_time / newest_returned_event_time bound the page you are looking at. Because the page is a contiguous newest-first slice, oldest_returned_event_time IS how far back this read reached — on a server blocking steadily, a 24-hour request at the default limit is answered by the newest few minutes, and nothing in the rows themselves says so. When truncated is true, raise limit or narrow hours_back (or anchor as_of) before drawing a conclusion about the window; widening hours_back cannot help, because the cap is on rows, not time. With dedup_key the read scans the window for the fingerprint BEFORE limit applies (so a matching incident is never lost to the cap), up to a stated scan ceiling: rows_examined is how many rows were fingerprinted and scan_truncated says whether the window held more than the scan could reach. Every timestamp here is UTC: event_time already was, and the six blocked_/blocking_ last_tran/last_batch stamps are de-skewed from the monitored server's local clock by this read, so comparing them against event_time to see whether a transaction predates the block is direct.")] public static async Task GetBlocking( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 30.")] int limit = 30, - [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null, + [Description("Maximum rows to return, newest first. Default 30. This is what bounds the page — read truncated to know whether the window held more.")] int limit = 30, + [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects. The fingerprint scan runs over the window BEFORE limit, up to the scan ceiling the payload reports as rows_examined / scan_truncated.")] string? dedup_key = null, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveWithFingerprintNameAsync(postgres, server_name); @@ -64,15 +64,39 @@ public static async Task GetBlocking( try { var now = windowEnd; + var filtering = !DarlingIncidentFingerprint.NoFilter(dedup_key); + + /* + #3541 A3: the cap is the CALLER'S, and truncation is OBSERVED rather than inferred. + + The reader used to cap at 200 rows newest-first whatever `limit` said, and this tool then + took `limit` of those and published the 200 as `total_events`. Two lies in one payload: the + count was neither the window's total nor the page's, and a 24-hour request on a server + blocking steadily was answered from its newest few minutes with nothing saying so. Fetching + limit + 1 and reading the extra row as the signal is the pattern get_collection_log and + get_query_heatmap already use; comparing count to the cap cannot tell a window holding + exactly `limit` events from one holding more. + + Under a dedup_key the fetch is the FINGERPRINT SCAN, not the page: #2159's promise is that + the filter runs over the window before `limit`, and the hidden 200-row cap was quietly + breaking it for every incident older than the newest 200 rows. The scan is bounded by a + STATED ceiling, over-fetched by one for the same reason, so the no-match answer can say the + scan ran out rather than implying the window was searched. + */ + var fetch = filtering ? DarlingBlockingReader.FingerprintScanCeiling + 1 : limit + 1; var rows = await DarlingBlockingReader.GetRecentBlockedProcessReportsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, fetch); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "blocked_process_report") ?? McpHelpers.Status("empty", "No blocking events found in the specified time range."); - /* #2159: fingerprint the WHOLE window, then filter, then cap. Capping first would let `limit` + var scanTruncated = filtering && rows.Count > DarlingBlockingReader.FingerprintScanCeiling; + if (scanTruncated) rows = rows.Take(DarlingBlockingReader.FingerprintScanCeiling).ToList(); + + /* #2159: fingerprint the WHOLE scan, then filter, then cap. Capping first would let `limit` discard the very incident the key names — the caller asked for one specific incident, not for - the newest `limit` rows that happen to include it. */ + the newest `limit` rows that happen to include it. Without a key the scan IS the page plus its + one sentinel row, so the keys computed here are the ones the page emits. */ var examined = rows.Count; var keys = DarlingIncidentFingerprint.BlockingKeys( resolved.FingerprintName, @@ -80,19 +104,26 @@ the newest `limit` rows that happen to include it. */ r.DatabaseName, r.ContentiousObject, r.BlockedSqlText, r.BlockingSqlText, r.WaitTimeMs, r.LockMode)).ToList()); - if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + if (filtering) { var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); var kept = rows.Where((_, i) => keys[i] == wanted).ToList(); if (kept.Count == 0) return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( - "blocking events", dedup_key!, resolved.FingerprintName, examined)); + "blocking events", dedup_key!, resolved.FingerprintName, examined) + + ScanCeilingClause(scanTruncated)); keys = kept.Select(r => wanted).Cast().ToList(); rows = kept; } - var result = rows.Take(limit).Select((r, i) => new + /* The page: `limit` rows of whatever survived, and the row past it is the truncation signal. + Under a key this is "more matching rows than limit"; without one it is "more rows in the + window than limit" — the same field, and both sentences are true of what it measures. */ + var truncated = rows.Count > limit; + var page = rows.Take(limit).ToList(); + + var result = page.Select((r, i) => new { event_time = r.EventTime?.ToString("o"), source = r.Source, @@ -136,9 +167,32 @@ the newest `limit` rows that happen to include it. */ return JsonSerializer.Serialize(new { server = resolved.ServerName, + /* The span REQUESTED. Kept under its shipped name, and no longer the only span on the page. */ hours_back, - dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), - total_events = rows.Count, + dedup_key = filtering ? DarlingIncidentFingerprint.NormalizeKey(dedup_key) : null, + /* + #3541 A3: `total_events` is gone. It was the reader's capped row count — neither the + window's total nor the page's — under a name that promised the first. What is published + now is what was measured: how many rows this page holds, whether the window held more, + and the time span the page actually covers. Newest-first makes the page a contiguous + slice of the window's tail, so oldest_returned_event_time IS the reach of this read — + the #3287 figure, and the field a caller has to read before believing that a quiet page + describes a quiet window. Under a dedup_key the page is the matching rows and the two + stamps bound the INCIDENT rather than the reach; the scan fields below carry the reach. + */ + events_returned = page.Count, + truncated, + /* Min/Max over the rows rather than rows[0] / rows[^1]: those coincide only under time + ordering, and a dedup_key page is the matching rows rather than a contiguous slice. + Enumerable.Min over DateTime? skips nulls and yields null for a page with no stamps. */ + oldest_returned_event_time = page.Min(r => r.EventTime)?.ToString("o"), + newest_returned_event_time = page.Max(r => r.EventTime)?.ToString("o"), + order = "event_time_desc", + /* The fingerprint scan, stated only when one ran: how many window rows were fingerprinted + and whether the window held more than the scan could reach. Null rather than 0 without a + key, because no scan was made — 0 would read as "a scan found nothing". */ + rows_examined = filtering ? examined : (int?)null, + scan_truncated = filtering ? scanTruncated : (bool?)null, events = result }, McpHelpers.JsonOptions); } @@ -148,13 +202,25 @@ the newest `limit` rows that happen to include it. */ } } - [McpServerTool(Name = "get_deadlocks"), Description("Gets recent deadlock events with victim process info. Deadlocks occur when two or more sessions permanently block each other. Use get_deadlock_detail for the full deadlock graph XML.")] + /// + /// The sentence appended to a no-match answer when the fingerprint scan hit + /// . Empty otherwise, so the shared + /// stays word-for-word what it was for a scan that + /// did cover the window — the three causes it names are the whole story in that case, and this one is the + /// fourth that only exists once the scan can run out. + /// + private static string ScanCeilingClause(bool scanTruncated) => + scanTruncated + ? $" The window held MORE rows than the {DarlingBlockingReader.FingerprintScanCeiling}-row fingerprint scan could reach, so this is not proof the incident is absent from the window — anchor as_of at the alert time with a narrow hours_back and retry." + : string.Empty; + + [McpServerTool(Name = "get_deadlocks"), Description("Gets recent deadlock events with victim process info, NEWEST FIRST. Deadlocks occur when two or more sessions permanently block each other. Use get_deadlock_detail for the full deadlock graph XML. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: deadlocks_returned is how many rows you got, truncated says the window held more than limit, and oldest_returned_deadlock_time / newest_returned_deadlock_time bound the page — under the newest-first ordering the oldest stamp IS how far back this read reached, so a truncated page says nothing about the earlier part of the window. Raise limit or narrow hours_back when truncated is true; widening hours_back cannot help, because the cap is on rows. With dedup_key the fingerprint scan runs over the window BEFORE limit, up to a stated ceiling (rows_examined / scan_truncated).")] public static async Task GetDeadlocks( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 20.")] int limit = 20, - [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null, + [Description("Maximum rows to return, newest first. Default 20. This is what bounds the page — read truncated to know whether the window held more.")] int limit = 20, + [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects. The fingerprint scan runs over the window BEFORE limit, up to the scan ceiling the payload reports as rows_examined / scan_truncated.")] string? dedup_key = null, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveWithFingerprintNameAsync(postgres, server_name); @@ -168,8 +234,14 @@ public static async Task GetDeadlocks( try { var now = windowEnd; + var filtering = !DarlingIncidentFingerprint.NoFilter(dedup_key); + + /* #3541 A3: see get_blocking — the caller's limit + 1 as the page fetch, the stated scan ceiling + + 1 as the fingerprint fetch, and the extra row in either case as the observed signal. The + reader's own cap was 50 here, which a caller asking for 100 deadlocks never saw. */ + var fetch = filtering ? DarlingBlockingReader.FingerprintScanCeiling + 1 : limit + 1; var rows = await DarlingBlockingReader.GetRecentDeadlocksAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, fetch); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "deadlocks") /* #2546: capability first (permanent), then the runtime precondition (fixable), then the @@ -179,24 +251,31 @@ did not deadlock — the one answer nobody should be given without being told. * ?? await DarlingRuntimePrecondition.StatusAsync(postgres, resolved.ServerId, resolved.ServerName, "deadlocks") ?? McpHelpers.Status("empty", "No deadlocks found in the specified time range."); - /* #2159: see get_blocking — fingerprint the window, filter, then cap. */ + var scanTruncated = filtering && rows.Count > DarlingBlockingReader.FingerprintScanCeiling; + if (scanTruncated) rows = rows.Take(DarlingBlockingReader.FingerprintScanCeiling).ToList(); + + /* #2159: see get_blocking — fingerprint the scan, filter, then cap. */ var examined = rows.Count; var keys = DarlingIncidentFingerprint.DeadlockKeys( resolved.FingerprintName, rows.Select(r => r.DeadlockGraphXml)); - if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + if (filtering) { var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); var kept = rows.Where((_, i) => keys[i] == wanted).ToList(); if (kept.Count == 0) return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( - "deadlocks", dedup_key!, resolved.FingerprintName, examined)); + "deadlocks", dedup_key!, resolved.FingerprintName, examined) + + ScanCeilingClause(scanTruncated)); keys = kept.Select(r => wanted).Cast().ToList(); rows = kept; } - var result = rows.Take(limit).Select((r, i) => new + var truncated = rows.Count > limit; + var page = rows.Take(limit).ToList(); + + var result = page.Select((r, i) => new { collection_time = r.CollectionTime.ToString("o"), deadlock_time = r.DeadlockTime?.ToString("o"), @@ -211,8 +290,18 @@ did not deadlock — the one answer nobody should be given without being told. * { server = resolved.ServerName, hours_back, - dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), - total_deadlocks = rows.Count, + dedup_key = filtering ? DarlingIncidentFingerprint.NormalizeKey(dedup_key) : null, + /* #3541 A3: the page described as a page — see get_blocking for why `total_deadlocks` went. + The ORDER BY is deadlock_time, so the bounds are on that stamp rather than collection_time, + and a deadlock whose stamp did not parse (null) is skipped by Min/Max rather than read as + the epoch. */ + deadlocks_returned = page.Count, + truncated, + oldest_returned_deadlock_time = page.Min(r => r.DeadlockTime)?.ToString("o"), + newest_returned_deadlock_time = page.Max(r => r.DeadlockTime)?.ToString("o"), + order = "deadlock_time_desc", + rows_examined = filtering ? examined : (int?)null, + scan_truncated = filtering ? scanTruncated : (bool?)null, deadlocks = result }, McpHelpers.JsonOptions); } @@ -222,12 +311,12 @@ did not deadlock — the one answer nobody should be given without being told. * } } - [McpServerTool(Name = "get_deadlock_detail"), Description("Gets the full deadlock graph XML for a specific time range. Returns the raw XML that can be analyzed for lock resources, process details, and deadlock chains.")] + [McpServerTool(Name = "get_deadlock_detail"), Description("Gets the full deadlock graph XML for a specific time range, NEWEST FIRST. Returns the raw XML that can be analyzed for lock resources, process details, and deadlock chains. Only deadlocks that CARRY a graph are counted against limit, so the page is limit graphs rather than limit rows; deadlocks_returned, truncated and oldest_returned_deadlock_time / newest_returned_deadlock_time describe the page the same way get_deadlocks does, and truncated means the window held more graphs than limit.")] public static async Task GetDeadlockDetail( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum deadlocks to return. Default 5.")] int limit = 5, + [Description("Maximum deadlocks WITH a graph to return, newest first. Default 5. Read truncated to know whether the window held more.")] int limit = 5, [Description("Optional #1140 alert fingerprint (the alert's Dedup Key). When supplied, returns only the incident with that key — paste it straight from an alert or ticket instead of scanning the window. The key is scoped to the server's display name and the incident's involved objects.")] string? dedup_key = null, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { @@ -242,13 +331,27 @@ public static async Task GetDeadlockDetail( try { var now = windowEnd; - var rows = await DarlingBlockingReader.GetRecentDeadlocksAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + + var filtering = !DarlingIncidentFingerprint.NoFilter(dedup_key); + + /* + #3541 A3: the graph predicate moved INTO the SQL (graphOnly), so the page fetch can be the + caller's limit + 1 over exactly the rows this tool can return. It used to filter + has_deadlock_xml in C# after a fixed 50-row fetch, so a caller asking for five graphs had + at most fifty rows to find them in, and a run of graph-less rows at the newest end read as + "no deadlock XML in the window" while older graphs sat behind the cap. Under a dedup_key the + fetch is the stated fingerprint scan ceiling + 1, as on get_deadlocks. + */ + var fetch = filtering ? DarlingBlockingReader.FingerprintScanCeiling + 1 : limit + 1; + var candidates = await DarlingBlockingReader.GetRecentDeadlocksAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, fetch, graphOnly: true); + var scanTruncated = filtering && candidates.Count > DarlingBlockingReader.FingerprintScanCeiling; + if (scanTruncated) candidates = candidates.Take(DarlingBlockingReader.FingerprintScanCeiling).ToList(); /* #2159: the XML filter runs BEFORE the cap and before the fingerprint, because a row without a graph has no objects to fingerprint — it could never match a key, and including it would only - consume one of the `limit` slots the caller wanted spent on real graphs. */ - var candidates = rows.Where(r => r.HasDeadlockXml).ToList(); + consume one of the `limit` slots the caller wanted spent on real graphs. It is now the SQL's + predicate rather than a Where() here, for the reason above. */ if (candidates.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "deadlocks") ?? McpHelpers.Status("empty", "No deadlock XML available in the specified time range."); @@ -257,18 +360,20 @@ consume one of the `limit` slots the caller wanted spent on real graphs. */ var keys = DarlingIncidentFingerprint.DeadlockKeys( resolved.FingerprintName, candidates.Select(r => r.DeadlockGraphXml)); - if (!DarlingIncidentFingerprint.NoFilter(dedup_key)) + if (filtering) { var wanted = DarlingIncidentFingerprint.NormalizeKey(dedup_key); var kept = candidates.Where((_, i) => keys[i] == wanted).ToList(); if (kept.Count == 0) return McpHelpers.Status("empty", DarlingIncidentFingerprint.NoMatchMessage( - "deadlocks with a graph", dedup_key!, resolved.FingerprintName, examined)); + "deadlocks with a graph", dedup_key!, resolved.FingerprintName, examined) + + ScanCeilingClause(scanTruncated)); keys = kept.Select(r => wanted).Cast().ToList(); candidates = kept; } + var truncated = candidates.Count > limit; var withXml = candidates.Take(limit).ToList(); var result = withXml.Select((r, i) => new @@ -284,7 +389,17 @@ consume one of the `limit` slots the caller wanted spent on real graphs. */ { server = resolved.ServerName, hours_back, - dedup_key = DarlingIncidentFingerprint.NoFilter(dedup_key) ? null : DarlingIncidentFingerprint.NormalizeKey(dedup_key), + dedup_key = filtering ? DarlingIncidentFingerprint.NormalizeKey(dedup_key) : null, + /* #3541 A3: the page bounds, on the same names as get_deadlocks. The page here is graphs, so + truncated means "more deadlocks WITH a graph than limit", which is the sentence this tool's + caller needs. */ + deadlocks_returned = withXml.Count, + truncated, + oldest_returned_deadlock_time = withXml.Min(r => r.DeadlockTime)?.ToString("o"), + newest_returned_deadlock_time = withXml.Max(r => r.DeadlockTime)?.ToString("o"), + order = "deadlock_time_desc", + rows_examined = filtering ? examined : (int?)null, + scan_truncated = filtering ? scanTruncated : (bool?)null, deadlocks = result }, McpHelpers.JsonOptions); } @@ -294,12 +409,12 @@ consume one of the `limit` slots the caller wanted spent on real graphs. */ } } - [McpServerTool(Name = "get_blocked_process_xml"), Description("Gets the raw blocked process report XML from extended events. Contains full detail about both the blocked and blocking sessions for deep analysis.")] + [McpServerTool(Name = "get_blocked_process_xml"), Description("Gets the raw blocked process report XML from extended events, NEWEST FIRST. Contains full detail about both the blocked and blocking sessions for deep analysis. Only rows that CARRY a report (the XE capture; the DMV fallback never has one) are counted against limit; reports_returned, truncated and oldest_returned_event_time / newest_returned_event_time describe the page the same way get_blocking does, and truncated means the window held more reports than limit.")] public static async Task GetBlockedProcessXml( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum reports to return. Default 5.")] int limit = 5, + [Description("Maximum reports WITH XML to return, newest first. Default 5. Read truncated to know whether the window held more.")] int limit = 5, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -313,9 +428,16 @@ public static async Task GetBlockedProcessXml( try { var now = windowEnd; - var rows = await DarlingBlockingReader.GetRecentBlockedProcessReportsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); - var withXml = rows.Where(r => r.HasReportXml).Take(limit).ToList(); + + /* #3541 A3: same shape as get_deadlock_detail — the report-XML predicate is in the SQL, the XE + arm alone is read (the DMV fallback never carries a report), and the fetch is the caller's + limit + 1. It used to take the merged 200-row page and Where() it for XML in C#, so a caller + asking for five reports had at most the newest 200 merged rows to find them in, DMV rows + included, and nothing said so. */ + var candidates = await DarlingBlockingReader.GetRecentBlockedProcessReportsWithXmlAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); + var truncated = candidates.Count > limit; + var withXml = candidates.Take(limit).ToList(); if (withXml.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "blocked_process_report") /* #2546: same order and same reason as get_deadlocks — a blocked-process capture whose @@ -337,6 +459,13 @@ session is gone is indistinguishable here from a server that never blocked. */ { server = resolved.ServerName, hours_back, + /* #3541 A3: the page bounds, on get_blocking's names. truncated means "more reports WITH + XML in the window than limit". */ + reports_returned = withXml.Count, + truncated, + oldest_returned_event_time = withXml.Min(r => r.EventTime)?.ToString("o"), + newest_returned_event_time = withXml.Max(r => r.EventTime)?.ToString("o"), + order = "event_time_desc", reports = result }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs index f48837065..38a8a9fe9 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigHistoryTools.cs @@ -187,7 +187,7 @@ public static async Task GetTraceFlagChanges( } } - [McpServerTool(Name = "get_database_scoped_config"), Description("Gets database-scoped configuration settings (sys.database_scoped_configurations). Shows MAXDOP, legacy CE, parameter sniffing, and other per-database settings.")] + [McpServerTool(Name = "get_database_scoped_config"), Description("Gets database-scoped configuration settings (sys.database_scoped_configurations). Shows MAXDOP, legacy CE, parameter sniffing, and other per-database settings. LATEST IS A TIME: captured when the collector connects, not on a schedule - captured_at is the instant these settings are as of.")] public static async Task GetDatabaseScopedConfig( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -198,14 +198,14 @@ public static async Task GetDatabaseScopedConfig( try { - var rows = await DarlingConfigHistoryReader.GetLatestDatabaseScopedConfigAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingConfigHistoryReader.GetLatestDatabaseScopedConfigAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "database_scoped_config") ?? McpHelpers.Status( "unavailable", "No database-scoped configuration data available. The config collector may not have run yet."); - IEnumerable filtered = rows; + IEnumerable filtered = snapshot.Rows; if (!string.IsNullOrEmpty(database_name)) filtered = filtered.Where(r => r.DatabaseName.Equals(database_name, StringComparison.OrdinalIgnoreCase)); @@ -225,6 +225,8 @@ public static async Task GetDatabaseScopedConfig( return JsonSerializer.Serialize(new { server = resolved.ServerName, + /* #3541 A10: the connect-time capture these settings are as of. */ + captured_at = snapshot.CapturedAt!.Value.ToString("o"), database_count = grouped.Count, databases = grouped }, McpHelpers.JsonOptions); @@ -235,7 +237,7 @@ public static async Task GetDatabaseScopedConfig( } } - [McpServerTool(Name = "get_query_store_health"), Description("Gets per-database Query Store health (sys.database_query_store_options): actual vs desired state, readonly_reason (decoded), storage used vs cap, cleanup mode and thresholds, and the runtime-stats interval length. The classic silent failure is desired READ_WRITE with actual READ_ONLY after the storage cap hit — check this when Query Store data looks stale or missing. Collected hourly; OFF is recorded as OFF (an absent database means not collected, never off).")] + [McpServerTool(Name = "get_query_store_health"), Description("Gets per-database Query Store health (sys.database_query_store_options): actual vs desired state, readonly_reason (decoded), storage used vs cap, cleanup mode and thresholds, and the runtime-stats interval length. The classic silent failure is desired READ_WRITE with actual READ_ONLY after the storage cap hit — check this when Query Store data looks stale or missing. Collected hourly; OFF is recorded as OFF (an absent database means not collected, never off). LATEST IS A TIME: this is the newest hourly capture, and captured_at is its instant.")] public static async Task GetQueryStoreHealth( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -246,14 +248,14 @@ public static async Task GetQueryStoreHealth( try { - var rows = await DarlingConfigHistoryReader.GetLatestQueryStoreHealthAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingConfigHistoryReader.GetLatestQueryStoreHealthAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "query_store_health") ?? McpHelpers.Status( "unavailable", "No Query Store health data available. The query_store_health collector runs hourly (SQL Server 2016+); a server with no rows either predates Query Store or has not completed a cycle yet."); - IEnumerable filtered = rows; + IEnumerable filtered = snapshot.Rows; if (!string.IsNullOrEmpty(database_name)) filtered = filtered.Where(r => r.DatabaseName.Equals(database_name, StringComparison.OrdinalIgnoreCase)); @@ -278,6 +280,7 @@ public static async Task GetQueryStoreHealth( return JsonSerializer.Serialize(new { server = resolved.ServerName, + captured_at = snapshot.CapturedAt!.Value.ToString("o"), database_count = result.Count, databases = result }, McpHelpers.JsonOptions); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigTools.cs index 5c98d5547..52d09370c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpConfigTools.cs @@ -32,7 +32,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpConfigTools { - [McpServerTool(Name = "get_server_config"), Description("Gets the current SQL Server instance configuration (sys.configurations). Shows all sp_configure settings with configured and in-use values. Useful for checking CTFP, MAXDOP, max memory, and other instance-level settings right now (unlike get_server_config_changes, which shows only what changed between connect snapshots).")] + [McpServerTool(Name = "get_server_config"), Description("Gets the current SQL Server instance configuration (sys.configurations). Shows all sp_configure settings with configured and in-use values. Useful for checking CTFP, MAXDOP, max memory, and other instance-level settings right now (unlike get_server_config_changes, which shows only what changed between connect snapshots). LATEST IS A TIME: configuration is captured when the collector CONNECTS, not on a schedule, so 'current' here means 'as of the last capture' - captured_at is that instant, and a value can be days old on a server the monitor has stayed connected to.")] public static async Task GetServerConfig( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -42,8 +42,8 @@ public static async Task GetServerConfig( try { - var rows = await DarlingCurrentConfigReader.GetLatestServerConfigAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingCurrentConfigReader.GetLatestServerConfigAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "server_config") ?? McpHelpers.Status( "unavailable", @@ -52,8 +52,10 @@ public static async Task GetServerConfig( return JsonSerializer.Serialize(new { server = resolved.ServerName, - setting_count = rows.Count, - settings = rows.Select(r => new + /* #3541 A10: the connect-time capture this "current" configuration is as of. */ + captured_at = snapshot.CapturedAt!.Value.ToString("o"), + setting_count = snapshot.Count, + settings = snapshot.Rows.Select(r => new { name = r.ConfigurationName, value_configured = r.ValueConfigured, @@ -70,7 +72,7 @@ public static async Task GetServerConfig( } } - [McpServerTool(Name = "get_database_config"), Description("Gets database-level configuration for all databases (sys.databases). Shows recovery model, RCSI, auto-shrink, auto-close, Query Store, compatibility level, page verify, and other settings. Critical for identifying misconfigured databases.")] + [McpServerTool(Name = "get_database_config"), Description("Gets database-level configuration for all databases (sys.databases). Shows recovery model, RCSI, auto-shrink, auto-close, Query Store, compatibility level, page verify, and other settings. Critical for identifying misconfigured databases. LATEST IS A TIME: captured when the collector connects, not on a schedule - captured_at is the instant these settings are as of, and a database created or altered since is not reflected until the next connect.")] public static async Task GetDatabaseConfig( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -81,14 +83,14 @@ public static async Task GetDatabaseConfig( try { - var rows = await DarlingCurrentConfigReader.GetLatestDatabaseConfigAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingCurrentConfigReader.GetLatestDatabaseConfigAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "database_config") ?? McpHelpers.Status( "unavailable", "No database configuration data available. The config collector may not have run yet."); - IEnumerable filtered = rows; + IEnumerable filtered = snapshot.Rows; if (!string.IsNullOrEmpty(database_name)) filtered = filtered.Where(r => r.DatabaseName.Equals(database_name, StringComparison.OrdinalIgnoreCase)); @@ -119,6 +121,7 @@ public static async Task GetDatabaseConfig( return JsonSerializer.Serialize(new { server = resolved.ServerName, + captured_at = snapshot.CapturedAt!.Value.ToString("o"), database_count = result.Count, databases = result }, McpHelpers.JsonOptions); @@ -129,7 +132,7 @@ public static async Task GetDatabaseConfig( } } - [McpServerTool(Name = "get_trace_flags"), Description("Gets active trace flags on the SQL Server instance. Shows flag number, enabled status, and whether the flag is global or session-scoped.")] + [McpServerTool(Name = "get_trace_flags"), Description("Gets active trace flags on the SQL Server instance. Shows flag number, enabled status, and whether the flag is global or session-scoped. LATEST IS A TIME: captured when the collector connects, not on a schedule - captured_at is the instant these flags are as of; a flag turned on or off since is not reflected until the next connect.")] public static async Task GetTraceFlags( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -139,16 +142,17 @@ public static async Task GetTraceFlags( try { - var rows = await DarlingCurrentConfigReader.GetLatestTraceFlagsAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingCurrentConfigReader.GetLatestTraceFlagsAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "trace_flags") ?? McpHelpers.Status("empty", "No trace flags found (none enabled, or the config collector has not run yet)."); return JsonSerializer.Serialize(new { server = resolved.ServerName, - trace_flag_count = rows.Count, - trace_flags = rows.Select(r => new + captured_at = snapshot.CapturedAt!.Value.ToString("o"), + trace_flag_count = snapshot.Count, + trace_flags = snapshot.Rows.Select(r => new { trace_flag = r.TraceFlag, enabled = r.Status, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomAlertTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomAlertTools.cs index 488600bc2..db785afc5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomAlertTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomAlertTools.cs @@ -177,7 +177,9 @@ public static async Task CreateCustomAlertRule( [McpServerTool(Name = "update_custom_alert_rule"), Description( "Updates an existing custom alert rule in place - a PARTIAL update: send only the fields you want to " + "change (name, description, definition, enabled), and every field you omit keeps its current value " + - "(omitting description does NOT clear it). Provide at least one field. A new definition is VALIDATED " + + "(omitting description does NOT clear it). To CLEAR the description, send it as an empty string \"\" - " + + "the one write vocabulary shared with update_custom_view: omitted = unchanged, empty = cleared. Provide at " + + "least one field. A new definition is VALIDATED " + "first; an invalid one returns {status:\"invalid\", ...} and changes nothing. Pass the 'version' you " + "last read via get_custom_alert_rule - if someone else changed the rule since, this returns " + "{status:\"conflict\", ...} rather than silently overwriting their edit (reload and re-apply). A " + @@ -189,7 +191,7 @@ public static async Task UpdateCustomAlertRule( [Description("The version you last read from get_custom_alert_rule (optimistic concurrency; a mismatch is a conflict, not an overwrite).")] int version, [Description("New rule name (unique, max 200 characters). Omit to keep the current name.")] string? name = null, [Description("New rule definition JSON. Validate it with validate_custom_alert_rule first. Omit to keep the current definition.")] string? definition = null, - [Description("New human-readable description. Omit to keep the current description (this cannot clear it).")] string? description = null, + [Description("New human-readable description. Omit to keep the current description; send an empty string \"\" to clear it.")] string? description = null, [Description("Whether the rule is active. Omit to keep the current enabled state; false pauses it, true resumes it.")] bool? enabled = null) { try @@ -223,7 +225,7 @@ blocked by a measure that has since drifted out of the catalog. */ var result = await store.UpdateAsync( rule_id, name ?? row.Name, - description ?? row.Description, + ResolveOptionalText(description, row.Description), definition ?? row.DefinitionJson, enabled ?? row.Enabled, version, @@ -428,6 +430,34 @@ public static Task ListCustomAlertTemplates() return Task.FromResult(new JsonObject { ["templates"] = templates }.ToJsonString(McpHelpers.JsonOptions)); } + /// + /// The ONE vocabulary for an optional text field on a partial update, shared by every Darling MCP update tool + /// that has one (this rule's description, and update_custom_view's): omitted (null) means + /// unchanged; an empty or whitespace-only string means cleared (stored as NULL); anything else is the new + /// value. + /// + /// Why this shape. Over MCP an omitted argument and an explicit JSON null both arrive as a C# + /// null, so null cannot carry two meanings and "unchanged" is the one it must carry - a caller who + /// sends {enabled: false} to pause a rule is not asking to lose its description. That leaves the + /// empty string as the only in-band way to say "clear", and an empty description is not a value anyone + /// stores on purpose, so nothing is lost by taking it. Before #3541 A14 the two update tools disagreed - + /// update_custom_alert_rule preserved an omitted description while update_custom_view treated + /// the same omission as a clear - and neither offered a clear at all, which is a contract that cannot be + /// honored on one side and a silent data loss on the other. + /// + /// Internal (not private) so the view tool calls this exact method rather than restating the rule; a + /// census test pins both call sites to it. + /// + internal static string? ResolveOptionalText(string? sent, string? current) + { + if (sent is null) + { + return current; + } + + return string.IsNullOrWhiteSpace(sent) ? null : sent; + } + /// A small {status, message} envelope for a non-data write outcome (conflict / invalid / /// not_found / error) - the same shape and /// use, so an MCP client can branch on the outcome kind. A successful diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomViewTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomViewTools.cs index 2d3372af8..d04f1c251 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomViewTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpCustomViewTools.cs @@ -155,7 +155,10 @@ public static async Task CreateCustomView( } [McpServerTool(Name = "update_custom_view"), Description( - "Updates an existing custom view in place (a full replacement of name/description/definition). The " + + "Updates an existing custom view in place: name and definition are REQUIRED and replace the stored ones; " + + "description is optional and follows the one write vocabulary shared with update_custom_alert_rule - " + + "OMITTED means UNCHANGED (the stored description is kept), an EMPTY string \"\" means CLEARED, any other " + + "text replaces it. The " + "definition is VALIDATED first; an invalid one returns {status:\"invalid\", ...} and changes nothing. Pass " + "the 'version' you last read via get_custom_view — if someone else changed the view since, this returns " + "{status:\"conflict\", ...} rather than silently overwriting their edit (reload and re-apply). A missing id " + @@ -167,7 +170,7 @@ public static async Task UpdateCustomView( [Description("The view name (unique, max 200 characters).")] string name, [Description("The full replacement view definition JSON. Validate it with validate_custom_view first.")] string definition, [Description("The version you last read from get_custom_view (optimistic concurrency; a mismatch is a conflict, not an overwrite).")] int version, - [Description("Optional human-readable description.")] string? description = null) + [Description("Optional human-readable description. Omit to keep the current description; send an empty string \"\" to clear it.")] string? description = null) { try { @@ -178,8 +181,23 @@ public static async Task UpdateCustomView( } var store = new CustomViewStore(postgres); + + /* #3541 A14: read the row first so an OMITTED description is carried forward rather than written as + NULL. The store's UpdateAsync is a full replacement (right for the web editor, which always sends + the whole form); over MCP an omitted argument arrives as null, and before this a rename or a + definition edit that did not restate the description silently erased it - while the sibling + update_custom_alert_rule kept it. Same rule, same helper, on both tools now: omitted = unchanged, + empty = cleared (see DarlingMcpCustomAlertTools.ResolveOptionalText). The extra read is one + indexed primary-key SELECT before a write the caller has already paid a round trip for. */ + var current = await store.GetAsync(view_id); + if (current is not CustomViewResult.Ok currentOk || currentOk.View is null) + { + return Outcome("not_found", $"No custom view with id {view_id}."); + } + var result = await store.UpdateAsync( - view_id, name, description, definition, version, updatedBy: DarlingWebEndpoints.McpEditorPrincipal); + view_id, name, DarlingMcpCustomAlertTools.ResolveOptionalText(description, currentOk.View.Description), + definition, version, updatedBy: DarlingWebEndpoints.McpEditorPrincipal); return result switch { CustomViewResult.Ok ok => DarlingWebEndpoints.BuildFullViewNode(ok.View!).ToJsonString(McpHelpers.JsonOptions), diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs index 7771c91a1..72d51a31d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpDataTools.cs @@ -101,12 +101,12 @@ public static async Task GetCpuUtilization( } } - [McpServerTool(Name = "get_wait_stats"), Description("Gets the top SQL Server wait types aggregated over a time period. Wait stats reveal what SQL Server spends time waiting on — high signal waits indicate CPU pressure, high resource waits indicate I/O or lock contention. Use this first to identify the dominant wait category, then drill into specific tools based on the wait type.")] + [McpServerTool(Name = "get_wait_stats"), Description("Gets the top SQL Server wait types aggregated over a time period, heaviest total wait first. Wait stats reveal what SQL Server spends time waiting on — high signal waits indicate CPU pressure, high resource waits indicate I/O or lock contention. Use this first to identify the dominant wait category, then drill into specific tools based on the wait type. THE PAGE IS BOUNDED BY limit: wait_types_returned is how many wait types you got and truncated says the window observed more than limit — the rows you have are the heaviest, and the ones past the cap are lighter, but a sum over the page is a sum over the page rather than over the server.")] public static async Task GetWaitStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum rows to return. Default 20.")] int limit = 20, + [Description("Maximum wait types to return, heaviest first. Default 20. This is what bounds the page — read truncated to know whether the window observed more.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -120,12 +120,17 @@ public static async Task GetWaitStats( try { var now = windowEnd; - var rows = await DarlingDataReader.GetWaitStatsAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now); + /* #3541 A3: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal. The reader's LIMIT 50 sat under a limit the tool accepts up to 1,000. */ + var rows = await DarlingDataReader.GetWaitStatsAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "wait_stats") ?? McpHelpers.Status("unavailable", "No wait stats data available for the specified time range."); - var result = rows.Take(limit).Select(r => + var truncated = rows.Count > limit; + var page = truncated ? rows.Take(limit).ToList() : rows; + + var result = page.Select(r => { var signalPct = r.TotalWaitTimeMs > 0 ? (double)r.TotalSignalWaitTimeMs / r.TotalWaitTimeMs * 100 : 0; return new @@ -143,6 +148,11 @@ public static async Task GetWaitStats( { server = resolved.ServerName, hours_back, + /* #3541 A3: the page described as a page. No time bounds here — the rows are per-type + aggregates over the whole window, so there is no page reach to report, only a cap. */ + wait_types_returned = page.Count, + truncated, + order = "total_wait_time_ms_desc", waits = result }, McpHelpers.JsonOptions); } @@ -274,7 +284,7 @@ public static async Task GetWaitTrend( } } - [McpServerTool(Name = "get_memory_stats"), Description("Gets the latest memory statistics snapshot: physical memory, buffer pool size, plan cache size, memory utilization %, and SQL Server memory model. Use this for a quick memory health check; use get_memory_clerks to see detailed breakdown by component.")] + [McpServerTool(Name = "get_memory_stats"), Description("Gets the latest memory statistics snapshot: physical memory, buffer pool size, plan cache size, memory utilization %, and SQL Server memory model. Use this for a quick memory health check; use get_memory_clerks to see detailed breakdown by component. LATEST IS A TIME: this reads one snapshot, not a window, and captured_at is the instant that snapshot was collected - read it before treating any figure as current, because the newest row a store holds can be minutes or days old.")] public static async Task GetMemoryStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -296,7 +306,8 @@ public static async Task GetMemoryStats( return JsonSerializer.Serialize(new { server = resolved.ServerName, - collection_time = stats.CollectionTime.ToString("o"), + /* #3541 A10: the one stamp every latest-snapshot read publishes, under the one name. */ + captured_at = stats.CollectionTime.ToString("o"), total_physical_memory_mb = stats.TotalPhysicalMemoryMb, available_physical_memory_mb = stats.AvailablePhysicalMemoryMb, memory_utilization_pct = Math.Round(utilization, 1), @@ -314,7 +325,7 @@ public static async Task GetMemoryStats( } } - [McpServerTool(Name = "get_memory_clerks"), Description("Gets the top memory consumers by memory clerk type — shows which SQL Server components are using the most memory.")] + [McpServerTool(Name = "get_memory_clerks"), Description("Gets the top memory consumers by memory clerk type — shows which SQL Server components are using the most memory. LATEST IS A TIME: this reads the newest clerk snapshot, not a window, and captured_at is the instant it was collected.")] public static async Task GetMemoryClerks( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -324,9 +335,9 @@ public static async Task GetMemoryClerks( try { - var rows = await DarlingDataReader.GetLatestMemoryClerksAsync(postgres, resolved.ServerId); + var snapshot = await DarlingDataReader.GetLatestMemoryClerksAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + if (snapshot.IsEmpty) /* ONE branch here, deliberately, and it is the reason this read gets no existence probe. The read is "every clerk at MAX(collection_time)", so zero rows back is logically the @@ -340,7 +351,7 @@ with the read by construction and tell the caller nothing it did not already hav "unavailable", $"No memory-clerk snapshot is available for {resolved.ServerName}. This read returns the LATEST snapshot rather than a window, so an empty result is never a quiet period — a live SQL Server always has memory clerks. It means nothing the memory_clerks collector stored is still retained, either because it has not run for this server or because its rows have aged out. Check get_collection_health and get_collection_log for the memory_clerks collector."); - var result = rows.Select(r => new + var result = snapshot.Rows.Select(r => new { clerk_type = r.ClerkType, memory_mb = Math.Round(r.MemoryMb, 2) @@ -349,6 +360,7 @@ with the read by construction and tell the caller nothing it did not already hav return JsonSerializer.Serialize(new { server = resolved.ServerName, + captured_at = snapshot.CapturedAt!.Value.ToString("o"), clerks = result }, McpHelpers.JsonOptions); } @@ -358,7 +370,7 @@ with the read by construction and tell the caller nothing it did not already hav } } - [McpServerTool(Name = "get_file_io_stats"), Description("Gets the latest file I/O statistics per database file: read/write counts, bytes, stall times, and calculated latency. High read latency (>20ms) or write latency (>10ms for data, >2ms for log) often indicates storage bottlenecks.")] + [McpServerTool(Name = "get_file_io_stats"), Description("Gets the latest file I/O statistics per database file: read/write counts, bytes, stall times, and calculated latency. High read latency (>20ms) or write latency (>10ms for data, >2ms for log) often indicates storage bottlenecks. Each row carries sample_interval_seconds, the measured seconds its deltas accrued over; a 0 means no delta was knowable for that file at this collection (first sighting, counter reset, or a gap past the delta policy — typically a restart) and its latencies are null rather than 0. LATEST IS A TIME: this reads the newest file-I/O snapshot, not a window, and captured_at is the instant it was collected; the deltas cover the sample_interval_seconds ending there.")] public static async Task GetFileIoStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -368,12 +380,12 @@ public static async Task GetFileIoStats( try { - var rows = await DarlingDataReader.GetLatestFileIoStatsAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingDataReader.GetLatestFileIoStatsAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "file_io_stats") ?? McpHelpers.Status("unavailable", "No file I/O stats available."); - var result = rows.Select(r => new + var result = snapshot.Rows.Select(r => new { database_name = r.DatabaseName, file_name = r.FileName, @@ -386,13 +398,20 @@ public static async Task GetFileIoStats( delta_write_bytes = r.DeltaWriteBytes, delta_stall_read_ms = r.DeltaStallReadMs, delta_stall_write_ms = r.DeltaStallWriteMs, - avg_read_latency_ms = Math.Round(r.DeltaReads > 0 ? (double)r.DeltaStallReadMs / r.DeltaReads : 0, 2), - avg_write_latency_ms = Math.Round(r.DeltaWrites > 0 ? (double)r.DeltaStallWriteMs / r.DeltaWrites : 0, 2) + /* #3540: the measured seconds the deltas accrued over, handed to the caller as the perfmon + tools hand theirs. 0 means no delta on this row was knowable — the collector's first + sighting of the file, a counter reset, or a gap past the policy — and the latencies below + are null for it rather than the "0.00 ms" a restart used to read as. null on the interval + itself is a pre-V127 row that never recorded one. */ + sample_interval_seconds = r.SampleIntervalSeconds, + avg_read_latency_ms = r.IsUnknowable ? (double?)null : Math.Round(r.DeltaReads > 0 ? (double)r.DeltaStallReadMs / r.DeltaReads : 0, 2), + avg_write_latency_ms = r.IsUnknowable ? (double?)null : Math.Round(r.DeltaWrites > 0 ? (double)r.DeltaStallWriteMs / r.DeltaWrites : 0, 2) }); return JsonSerializer.Serialize(new { server = resolved.ServerName, + captured_at = snapshot.CapturedAt!.Value.ToString("o"), files = result }, McpHelpers.JsonOptions); } @@ -448,7 +467,7 @@ public static async Task GetTempDbTrend( } } - [McpServerTool(Name = "get_perfmon_stats"), Description("Gets the latest SQL Server performance counter values: batch requests/sec, compilations/sec, deadlocks/sec, and more. Provides throughput context to distinguish a busy server from a sick one. Use counter_name or instance_name to filter results.")] + [McpServerTool(Name = "get_perfmon_stats"), Description("Gets the latest SQL Server performance counter values: batch requests/sec, compilations/sec, deadlocks/sec, and more. Provides throughput context to distinguish a busy server from a sick one. Use counter_name or instance_name to filter results. LATEST IS A TIME: this reads the newest counter snapshot, not a window, and captured_at is the instant it was collected; use get_perfmon_trend for a counter over time.")] public static async Task GetPerfmonStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -460,12 +479,12 @@ public static async Task GetPerfmonStats( try { - var rows = await DarlingDataReader.GetLatestPerfmonStatsAsync(postgres, resolved.ServerId); - if (rows.Count == 0) + var snapshot = await DarlingDataReader.GetLatestPerfmonStatsAsync(postgres, resolved.ServerId); + if (snapshot.IsEmpty) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "perfmon_stats") ?? McpHelpers.Status("unavailable", "No perfmon stats available."); - IEnumerable filtered = rows; + IEnumerable filtered = snapshot.Rows; if (!string.IsNullOrEmpty(counter_name)) filtered = filtered.Where(r => r.CounterName.Contains(counter_name, StringComparison.OrdinalIgnoreCase)); if (!string.IsNullOrEmpty(instance_name)) @@ -482,6 +501,7 @@ public static async Task GetPerfmonStats( return JsonSerializer.Serialize(new { server = resolved.ServerName, + captured_at = snapshot.CapturedAt!.Value.ToString("o"), counters = result }, McpHelpers.JsonOptions); } @@ -493,7 +513,7 @@ public static async Task GetPerfmonStats( /* ═══════════════════════════ query performance ═══════════════════════════ */ - [McpServerTool(Name = "get_top_queries_by_cpu"), Description("Gets expensive queries from sys.dm_exec_query_stats (plan cache). Best for: currently cached queries with detailed per-execution stats, DOP, spills, and query_hash for trending. Returns query_hash, query_plan_hash, sql_handle, plan_handle, and host_object (the hosting procedure/function for proc-hosted statements, null for ad-hoc) — groups key on (database, query_hash, host_object), so INSERT...EXEC callers in different procedures report separately with their own text. distinct_texts counts statement texts merged into a group (>1 = ad-hoc literal variants or pre-upgrade history; query_text is one representative, 0 means only rows predating the text dimension). Set group_by='host_object' to roll all of a procedure's statements into one row — necessary when dynamic SQL with per-value literals fragments one statement across many hashes, which no top-N-by-hash ranking can surface. Supports database and parallelism filtering. min/max_cpu_ms and min/max_elapsed_ms are LIFETIME extremes for the plan's time in cache (same semantics as max_dop), not windowed — totals and avgs are windowed deltas; rows where an extreme provably predates the window carry extremes_note. Also returns cpu_attribution: the returned rows' summed CPU-seconds against the SQL process's measured CPU-seconds for the window (avg cpu_utilization % x core count x window) - attributed_cpu_ratio says how much of the box the ranking explains; when the CPU series or core count is missing, or covers too little of the window, the ratio is omitted rather than invented.")] + [McpServerTool(Name = "get_top_queries_by_cpu"), Description("Gets expensive queries from sys.dm_exec_query_stats (plan cache). Best for: currently cached queries with detailed per-execution stats, DOP, spills, and query_hash for trending. Returns query_hash, query_plan_hash, sql_handle, plan_handle, and host_object (the hosting procedure/function for proc-hosted statements, null for ad-hoc) — groups key on (database, query_hash, host_object), so INSERT...EXEC callers in different procedures report separately with their own text. distinct_texts counts statement texts merged into a group (>1 = ad-hoc literal variants or pre-upgrade history; query_text is one representative, 0 means only rows predating the text dimension). Set group_by='host_object' to roll all of a procedure's statements into one row — necessary when dynamic SQL with per-value literals fragments one statement across many hashes, which no top-N-by-hash ranking can surface. Supports database and parallelism filtering; every filter is applied IN the query before the ranking and the cap, so the page is the top-N of the FILTERED population (filter_applied names the parallelism floor in force, null when none), and an empty page under parallel_only/min_dop is the window's answer rather than a page artefact. min/max_cpu_ms and min/max_elapsed_ms are LIFETIME extremes for the plan's time in cache (same semantics as max_dop), not windowed — totals and avgs are windowed deltas; rows where an extreme provably predates the window carry extremes_note. Also returns cpu_attribution: the returned rows' summed CPU-seconds against the SQL process's measured CPU-seconds for the window (avg cpu_utilization % x core count x window) - attributed_cpu_ratio says how much of the box the ranking explains; when the CPU series or core count is missing, or covers too little of the window, the ratio is omitted rather than invented.")] public static async Task GetTopQueriesByCpu( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -523,18 +543,39 @@ the exact wrong conclusion this option exists to prevent. */ validation = McpHelpers.ValidateTop(top, "top"); if (validation != null) return validation; + /* #3541 A13: the parallelism filter goes INTO the read as a lifetime max_dop floor on the grouped + population, applied before the CPU ranking and the cap (see TopQueriesSql's HAVING note). It used + to be a .Where over the returned top-N page: parallel_only=true on a box whose twenty hottest plans + were serial came back EMPTY while the window held parallel plans, and the engine's own CXPACKET + advice steers agents to exactly that call. The floor is 2 for parallel_only (the smallest DOP that + is parallel), min_dop when the caller set one above that, 0 (admit all) otherwise; min_dop implies + parallel filtering, as its description has always said. */ + var minMaxDop = min_dop > 1 ? min_dop : parallel_only ? 2 : 0; + var filterApplied = minMaxDop > 0 + ? $"lifetime max_dop >= {minMaxDop} (applied in SQL before the top-{top} ranking; the page is the top-{top} of the parallel population)" + : null; + try { var now = windowEnd; var rows = await DarlingDataReader.GetTopQueriesByCpuAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name, rollUpByHostObject: rollUp); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, top, database_name, rollUpByHostObject: rollUp, minMaxDop: minMaxDop); if (rows.Count == 0) + { + /* A filtered miss is not a collection miss: with the floor in the query, an empty page under + parallel_only means the window held no group whose plan ever ran parallel, and saying + "no query stats available" for that would send the caller to collection health. */ + if (minMaxDop > 0) + { + return McpHelpers.Status( + "empty", + $"No query-stats group on {resolved.ServerName} in the last {hours_back} hour(s) has a cached plan with lifetime max_dop >= {minMaxDop}. The filter was applied in SQL over the whole window, so this is the window's answer rather than a page artefact — drop parallel_only / min_dop to see the unfiltered ranking, or confirm current parallelism with analyze_query_plan.", + new { filter_applied = filterApplied }); + } + return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "query_stats") ?? McpHelpers.Status("unavailable", "No query stats available for the specified time range."); - - var filtered = rows - .Where(r => !(parallel_only || min_dop > 1) || (r.MaxDop > 1 && r.MaxDop >= (min_dop > 1 ? min_dop : 2))) - .ToList(); + } /* #2320: what fraction of the box's measured CPU the RETURNED rows explain — numerator is the caller-visible ranking (post top-N, post filters), denominator is measured, and the @@ -546,12 +587,12 @@ ratio is omitted rather than invented when a denominator piece is missing. The t var cpuAggregate = await cpuAggregateTask; var properties = await propertiesTask; var attribution = CpuAttribution.Compute( - filtered.Sum(r => r.TotalCpuUs) / 1_000_000.0, + rows.Sum(r => r.TotalCpuUs) / 1_000_000.0, now.AddHours(-hours_back), now, cpuAggregate.SampleCount, cpuAggregate.FirstSample, cpuAggregate.LastSample, cpuAggregate.AvgSqlCpuPercent, properties?.CpuCount ?? 0); - var result = filtered.Select(r => new + var result = rows.Select(r => new { database_name = r.DatabaseName, query_hash = r.QueryHash, @@ -609,6 +650,8 @@ ratio is omitted rather than invented when a denominator piece is missing. The t /* #2235: echoed so a stored or pasted payload cannot be misread as the other grouping — the two answer different questions and the rows look alike. */ group_by = rollUp ? "host_object" : "query_hash", + /* #3541 A13: the filter that shaped the population, stated on the payload; null when none. */ + filter_applied = filterApplied, cpu_attribution = new { ranked_cpu_seconds = attribution.RankedCpuSeconds, @@ -1368,7 +1411,7 @@ internal static (string State, string Message) FleetMaintenanceLogMiss( public static async Task GetCollectionLog( NpgsqlDataSource postgres, [Description("Server name or display name, or the reserved name (fleet) for the fleet-maintenance run-records.")] string? server_name = null, - [Description("Hours of history. Default 24.")] int hours_back = 24, + [Description("Hours of history. Default 24. No upper bound (this read exists to look further back than the 168-hour reads allow); a negative or zero value is refused rather than read as its absolute value.")] int hours_back = 24, [Description("Maximum rows to return. Default 200. Applied AFTER the two filters, so it caps the matching rows rather than the window.")] int limit = 200, [Description(McpHelpers.AsOfDescription)] string? as_of = null, /* @@ -1401,18 +1444,19 @@ every existing one meaning what it already meant. var invalidFloor = McpHelpers.ValidateMinMs(min_duration_ms, "min_duration_ms"); if (invalidFloor != null) return invalidFloor; - /* ResolveAsOf here, deliberately NOT ValidateWindow. These three reads have never capped - hours_back -- they Math.Abs() it and window on the result -- so routing them through the - shared validator would impose the 168-hour ceiling every other read carries, and take reach - away from exactly the read whose premise is looking FURTHER back than the default. The anchor - is validated because it is new; the span keeps the behaviour callers already have. */ - var anchorError = McpHelpers.ResolveAsOf(as_of, out var windowEnd); + /* ValidateUncappedWindow, deliberately NOT ValidateWindow. These three reads have never capped + hours_back, so routing them through the shared validator would impose the 168-hour ceiling every + other read carries and take reach away from exactly the read whose premise is looking FURTHER back + than the default. What they no longer do is Math.Abs() a negative span (#3541 A13): a window that + ends before it starts is a caller error, and flipping the sign answered a different question with + nothing to say so. Refused, like every other unusable parameter here. */ + var anchorError = McpHelpers.ValidateUncappedWindow(hours_back, as_of, out var windowEnd); if (anchorError != null) return anchorError; try { var end = windowEnd; - var start = end.AddHours(-Math.Abs(hours_back)); + var start = end.AddHours(-hours_back); /* Over-fetch by one so truncation is OBSERVED rather than inferred. Comparing count to the cap cannot tell a window holding exactly `limit` runs from one holding more, and this @@ -1465,7 +1509,7 @@ FleetMaintenanceLogMiss. The server branches below are untouched. */ if (resolved.ServerId == DarlingObservability.FleetServerId) { var (state, text) = FleetMaintenanceLogMiss( - everCollected, collector_name, min_duration_ms, Math.Abs(hours_back)); + everCollected, collector_name, min_duration_ms, hours_back); return McpHelpers.Status(state, text); } @@ -1481,12 +1525,12 @@ FleetMaintenanceLogMiss. The server branches below are untouched. */ { return McpHelpers.Status( "empty", - $"No collector runs on {resolved.ServerName} in the last {Math.Abs(hours_back)} hour(s) matched {McpHelpers.DescribeCollectionLogFilters(collector_name, min_duration_ms)}. This says nothing about the window as a whole — the filters were applied, so unfiltered runs may well exist. Drop them to see what the window holds, and check collector_name against the names get_collection_health lists, since it is matched exactly."); + $"No collector runs on {resolved.ServerName} in the last {hours_back} hour(s) matched {McpHelpers.DescribeCollectionLogFilters(collector_name, min_duration_ms)}. This says nothing about the window as a whole — the filters were applied, so unfiltered runs may well exist. Drop them to see what the window holds, and check collector_name against the names get_collection_health lists, since it is matched exactly."); } return McpHelpers.Status( "empty", - $"No collector runs recorded for {resolved.ServerName} in the last {Math.Abs(hours_back)} hour(s). This server HAS collected before, so this window is genuinely quiet rather than broken — widen hours_back to find the most recent runs."); + $"No collector runs recorded for {resolved.ServerName} in the last {hours_back} hour(s). This server HAS collected before, so this window is genuinely quiet rather than broken — widen hours_back to find the most recent runs."); } var result = rows.Select(r => new @@ -1613,7 +1657,7 @@ emits its two sub-lines. Emitted raw rather than pre-divided into ms-per-id -- t server = resolved.ServerName, /* The span REQUESTED. Kept under its shipped name, and no longer the only span reported -- see the two timestamps below. */ - hours_back = Math.Abs(hours_back), + hours_back = hours_back, run_count = rows.Count, /* Observed by the over-fetch above, not inferred from the row count. */ truncated, @@ -1672,25 +1716,26 @@ description is the only other place that coupling is written down. */ public static async Task GetCurrentWaitsTrend( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Hours of history. Default 4.")] int hours_back = 4, + [Description("Hours of history. Default 4. No upper bound (this read exists to look further back than the 168-hour reads allow); a negative or zero value is refused rather than read as its absolute value.")] int hours_back = 4, [Description("Limit the blocked-session series to one database. Omit for all databases.")] string? database_name = null, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); if (error != null) return error; - /* ResolveAsOf here, deliberately NOT ValidateWindow. These three reads have never capped - hours_back -- they Math.Abs() it and window on the result -- so routing them through the - shared validator would impose the 168-hour ceiling every other read carries, and take reach - away from exactly the read whose premise is looking FURTHER back than the default. The anchor - is validated because it is new; the span keeps the behaviour callers already have. */ - var anchorError = McpHelpers.ResolveAsOf(as_of, out var windowEnd); + /* ValidateUncappedWindow, deliberately NOT ValidateWindow. These three reads have never capped + hours_back, so routing them through the shared validator would impose the 168-hour ceiling every + other read carries and take reach away from exactly the read whose premise is looking FURTHER back + than the default. What they no longer do is Math.Abs() a negative span (#3541 A13): a window that + ends before it starts is a caller error, and flipping the sign answered a different question with + nothing to say so. Refused, like every other unusable parameter here. */ + var anchorError = McpHelpers.ValidateUncappedWindow(hours_back, as_of, out var windowEnd); if (anchorError != null) return anchorError; try { var end = windowEnd; - var start = end.AddHours(-Math.Abs(hours_back)); + var start = end.AddHours(-hours_back); var waits = await DarlingDataReader.GetWaitingTaskTrendAsync(postgres, resolved.ServerId, start, end); var blocked = await DarlingDataReader.GetBlockedSessionTrendAsync( @@ -1713,7 +1758,7 @@ waiting_tasks collector never ran. A caller told all-clear stops looking. return everCollected ? McpHelpers.Status( "empty", - $"Nothing was waiting on {resolved.ServerName} in the last {Math.Abs(hours_back)} hour(s). The collector HAS sampled this server, so this is a genuine all-clear for the window rather than missing data.") + $"Nothing was waiting on {resolved.ServerName} in the last {hours_back} hour(s). The collector HAS sampled this server, so this is a genuine all-clear for the window rather than missing data.") : McpHelpers.Status( "unavailable", $"No waiting-task samples have EVER been recorded for {resolved.ServerName}, so this is NOT an all-clear — there is nothing to read. Check that collection is running for this server before concluding it was quiet."); @@ -1722,7 +1767,7 @@ waiting_tasks collector never ran. A caller told all-clear stops looking. return JsonSerializer.Serialize(new { server = resolved.ServerName, - hours_back = Math.Abs(hours_back), + hours_back = hours_back, database_name, /* Two series in one payload because they are read together: a wait-type spike with no @@ -1753,24 +1798,25 @@ across two tools a caller can fetch one and draw the wrong conclusion. public static async Task GetBlockingStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Hours of history. Default 24.")] int hours_back = 24, + [Description("Hours of history. Default 24. No upper bound (this read exists to look further back than the 168-hour reads allow); a negative or zero value is refused rather than read as its absolute value.")] int hours_back = 24, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); if (error != null) return error; - /* ResolveAsOf here, deliberately NOT ValidateWindow. These three reads have never capped - hours_back -- they Math.Abs() it and window on the result -- so routing them through the - shared validator would impose the 168-hour ceiling every other read carries, and take reach - away from exactly the read whose premise is looking FURTHER back than the default. The anchor - is validated because it is new; the span keeps the behaviour callers already have. */ - var anchorError = McpHelpers.ResolveAsOf(as_of, out var windowEnd); + /* ValidateUncappedWindow, deliberately NOT ValidateWindow. These three reads have never capped + hours_back, so routing them through the shared validator would impose the 168-hour ceiling every + other read carries and take reach away from exactly the read whose premise is looking FURTHER back + than the default. What they no longer do is Math.Abs() a negative span (#3541 A13): a window that + ends before it starts is a caller error, and flipping the sign answered a different question with + nothing to say so. Refused, like every other unusable parameter here. */ + var anchorError = McpHelpers.ValidateUncappedWindow(hours_back, as_of, out var windowEnd); if (anchorError != null) return anchorError; try { var end = windowEnd; - var start = end.AddHours(-Math.Abs(hours_back)); + var start = end.AddHours(-hours_back); var blocking = await DarlingDataReader.GetBlockingDurationStatsAsync(postgres, resolved.ServerId, start, end); @@ -1804,7 +1850,7 @@ await DarlingBlockingTrendReader.HasAnyBlockingCollectorRunAsync(postgres, resol return everRan ? McpHelpers.Status( "empty", - $"No blocking or deadlocks recorded for {resolved.ServerName} in the last {Math.Abs(hours_back)} hour(s). The blocking collectors HAVE run successfully for this server, so the window is genuinely clear rather than blind.") + $"No blocking or deadlocks recorded for {resolved.ServerName} in the last {hours_back} hour(s). The blocking collectors HAVE run successfully for this server, so the window is genuinely clear rather than blind.") : McpHelpers.Status( "unavailable", $"The blocking collectors have NEVER run successfully for {resolved.ServerName}, so this is NOT a clean bill of health — nothing looked. Blocked-process reports need the XE session running, or the DMV blocking snapshot collector enabled; check those before concluding this server does not block."); @@ -1813,7 +1859,7 @@ await DarlingBlockingTrendReader.HasAnyBlockingCollectorRunAsync(postgres, resol return JsonSerializer.Serialize(new { server = resolved.ServerName, - hours_back = Math.Abs(hours_back), + hours_back = hours_back, /* Severity, not counts. get_blocking_trend already answers how OFTEN; ten one-second blocks and one ten-minute block share a count and are different problems. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFleetTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFleetTools.cs index c849527fa..e4ab91ff9 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFleetTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFleetTools.cs @@ -50,7 +50,17 @@ public sealed class DarlingMcpFleetTools "buffer_pool_mb and the whole threads block are null for " + "the same structural reason and no band is claimed for them — those metrics are SQL Server DMV " + "readings with no PostgreSQL equivalent collected; get_pg_buffer_usage, get_pg_kernel_stats and " + - "get_pg_session_states are the reads that answer the nearest PostgreSQL questions.")] + "get_pg_session_states are the reads that answer the nearest PostgreSQL questions. A PostgreSQL " + + "target's deadlock_count IS measured: it is the server's own pg_stat_database.deadlocks counter, " + + "differenced per database over the window (a statistics reset clamps to zero, never subtracts) and " + + "summed, banded through the same deadlock_warn_per_hour / deadlock_critical_per_hour tiers as SQL " + + "Server's graph count; deadlock_source reads PostgresTarget for it, which since #3539 means COUNTED " + + "from that counter (deadlock_coverage.postgres_servers is a sub-count of servers_read, not a gap), " + + "and get_pg_deadlocks has the parsed deadlock reports themselves. Its blocking_severity stays " + + "Unknown on purpose: PostgreSQL blocking is a once-a-minute SAMPLE of pg_stat_activity, and the " + + "blocking band's count tiers were measured in engine-recorded reports per hour, so a sighting count " + + "through them would band on a denominator they were never measured against — the PostgreSQL " + + "Blocking alert speaks for that condition until a sampled-shape band is measured.")] public static async Task GetFleetOverview( NpgsqlDataSource postgres, [Description("Hours of blocking/deadlock history the per-server cards and fleet totals window over. Default 1.")] int hours_back = 1) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthParserTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthParserTools.cs index 7c288e3cb..49284156f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthParserTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthParserTools.cs @@ -42,6 +42,15 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// parsed-table architecture) have no analog in the parse-on-read record and are omitted. Severe-error /// database_name is resolved from the collected size-stats mapping (the DB-free shred left it null). /// +/// +/// +/// Every one of the nine publishes its SOURCE WITNESS (#3541 A12): source_observed — whether the +/// collector has ever stored a system_health event of any type for this server, i.e. whether the ring buffer +/// has ever been read into the store — and last_captured_at, the collector's newest capture. A zero-row +/// window is then one of four nothings () and says which; a server whose session has +/// never been read answers unavailable, never empty. Before this, eight of the nine answered a +/// dead session with the same word a healthy quiet hour earns. +/// /// [McpServerToolType] public sealed class DarlingMcpHealthParserTools @@ -54,7 +63,7 @@ public sealed class DarlingMcpHealthParserTools /// private const string SystemHealthCollectorName = "system_health_events"; - [McpServerTool(Name = "get_health_parser_system_health"), Description("Gets parsed system_health extended event data: overall health indicators captured by sp_HealthParser.")] + [McpServerTool(Name = "get_health_parser_system_health"), Description("Gets parsed system_health extended event data: overall health indicators captured by sp_HealthParser. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetSystemHealth( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -72,13 +81,15 @@ GetSystemHealthAsync keeps every SYSTEM snapshot that has a timestamp. */ r => r.EventTime.HasValue); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No system health data found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.SpServerDiagnosticsEvent, + "none carried a SYSTEM component result with a timestamp (the other four sp_server_diagnostics components feed the sibling reads)"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), total_entries = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), entries = c.Rows.Take(limit).Select(r => new @@ -105,7 +116,7 @@ GetSystemHealthAsync keeps every SYSTEM snapshot that has a timestamp. */ catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_system_health", ex); } } - [McpServerTool(Name = "get_health_parser_severe_errors"), Description("Gets severe errors from system_health: stack dumps, non-yielding schedulers, and other critical SQL Server events.")] + [McpServerTool(Name = "get_health_parser_severe_errors"), Description("Gets severe errors from system_health: stack dumps, non-yielding schedulers, and other critical SQL Server events. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetSevereErrors( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -127,6 +138,7 @@ public static async Task GetSevereErrors( var xmls = await DarlingSystemHealthReader.ReadEventXmlAsync( postgres, resolved.ServerId, now.AddHours(-hours_back), now, SystemHealthParser.ErrorReportedEvent); var map = await mapTask; + var lastCapturedAt = await DarlingSystemHealthReader.GetLastCaptureAsync(postgres, resolved.ServerId); var rows = xmls .Select(SystemHealthParser.ParseSevereError) @@ -134,13 +146,17 @@ public static async Task GetSevereErrors( .Select(r => r!) .ToList(); if (rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No severe errors found in the requested time range."); + return await EmptyAsync( + postgres, new Collected(null, resolved.ServerId, resolved.ServerName, rows, xmls.Count, lastCapturedAt), + hours_back, SystemHealthParser.ErrorReportedEvent, + $"none was a significant severe error (severity {SystemHealthSignificance.SevereErrorMinSeverity}+ and off the benign connection-reset list)"); return JsonSerializer.Serialize(new { server = resolved.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(lastCapturedAt), error_count = rows.Count, shown = Math.Min(rows.Count, limit), errors = rows.Take(limit).Select(r => new @@ -158,7 +174,7 @@ public static async Task GetSevereErrors( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_severe_errors", ex); } } - [McpServerTool(Name = "get_health_parser_io_issues"), Description("Gets I/O-related issues from system_health: 15-second I/O warnings, long I/O requests, and stalled I/O subsystems.")] + [McpServerTool(Name = "get_health_parser_io_issues"), Description("Gets I/O-related issues from system_health: 15-second I/O warnings, long I/O requests, and stalled I/O subsystems. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetIOIssues( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -175,13 +191,15 @@ public static async Task GetIOIssues( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No I/O issues found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.SpServerDiagnosticsEvent, + "none was an IO_SUBSYSTEM component result in the WARNING state"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), issue_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), issues = c.Rows.Take(limit).Select(r => new @@ -199,7 +217,7 @@ public static async Task GetIOIssues( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_io_issues", ex); } } - [McpServerTool(Name = "get_health_parser_scheduler_issues"), Description("Gets scheduler issues from system_health: non-yielding schedulers, deadlocked schedulers, and scheduler monitor events.")] + [McpServerTool(Name = "get_health_parser_scheduler_issues"), Description("Gets scheduler issues from system_health: non-yielding schedulers, deadlocked schedulers, and scheduler monitor events. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetSchedulerIssues( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -215,13 +233,15 @@ public static async Task GetSchedulerIssues( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No scheduler issues found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.SchedulerMonitorEvent, + "none was a scheduler-monitor record in the WARNING state"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), issue_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), issues = c.Rows.Take(limit).Select(r => new @@ -241,7 +261,7 @@ public static async Task GetSchedulerIssues( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_scheduler_issues", ex); } } - [McpServerTool(Name = "get_health_parser_memory_conditions"), Description("Gets memory condition events from system_health: low memory notifications, memory broker adjustments, and memory pressure indicators.")] + [McpServerTool(Name = "get_health_parser_memory_conditions"), Description("Gets memory condition events from system_health: low memory notifications, memory broker adjustments, and memory pressure indicators. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetMemoryConditions( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -257,13 +277,15 @@ public static async Task GetMemoryConditions( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No memory condition events found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.SpServerDiagnosticsEvent, + "none was a RESOURCE component result carrying a low-memory (RESOURCE_MEMPHYSICAL_LOW) notification"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), event_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), events = c.Rows.Take(limit).Select(r => new @@ -306,7 +328,7 @@ public static async Task GetMemoryConditions( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_memory_conditions", ex); } } - [McpServerTool(Name = "get_health_parser_cpu_tasks"), Description("Gets CPU task events from system_health: long-running CPU-bound tasks, high CPU worker threads, and process utilization snapshots.")] + [McpServerTool(Name = "get_health_parser_cpu_tasks"), Description("Gets CPU task events from system_health: long-running CPU-bound tasks, high CPU worker threads, and process utilization snapshots. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetCPUTasks( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -322,13 +344,15 @@ public static async Task GetCPUTasks( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No CPU task events found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.SpServerDiagnosticsEvent, + $"none was a QUERY_PROCESSING component result in the WARNING state with at least {SystemHealthSignificance.CpuTaskMinPendingTasks} pending tasks"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), event_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), events = c.Rows.Take(limit).Select(r => new @@ -350,7 +374,7 @@ public static async Task GetCPUTasks( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_cpu_tasks", ex); } } - [McpServerTool(Name = "get_health_parser_memory_broker"), Description("Gets memory broker events from system_health: cache shrink/grow notifications, memory clerk adjustments, and broker-mediated memory redistribution.")] + [McpServerTool(Name = "get_health_parser_memory_broker"), Description("Gets memory broker events from system_health: cache shrink/grow notifications, memory clerk adjustments, and broker-mediated memory redistribution. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetMemoryBroker( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -366,13 +390,15 @@ public static async Task GetMemoryBroker( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No memory broker events found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.MemoryBrokerEvent, + "none carried a low-memory notification (broker adjustments that are not a shrink under pressure are routine)"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), event_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), events = c.Rows.Take(limit).Select(r => new @@ -396,7 +422,7 @@ public static async Task GetMemoryBroker( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_memory_broker", ex); } } - [McpServerTool(Name = "get_health_parser_memory_node_oom"), Description("Gets memory node OOM events from system_health: out-of-memory conditions on specific NUMA nodes.")] + [McpServerTool(Name = "get_health_parser_memory_node_oom"), Description("Gets memory node OOM events from system_health: out-of-memory conditions on specific NUMA nodes. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetMemoryNodeOOM( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -414,13 +440,15 @@ public static async Task GetMemoryNodeOOM( SystemHealthSignificance.IsSignificant); if (c.EarlyReturn != null) return c.EarlyReturn; if (c.Rows.Count == 0) - return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status("empty", "No memory node OOM events found in the requested time range."); + return await EmptyAsync(postgres, c, hours_back, SystemHealthParser.MemoryNodeOomEvent, + "none shredded to a memory-node OOM record (this category is ungated, so a captured OOM event that parsed would be here)"); return JsonSerializer.Serialize(new { server = c.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(c.LastCapturedAt), event_count = c.Rows.Count, shown = Math.Min(c.Rows.Count, limit), events = c.Rows.Take(limit).Select(r => new @@ -459,7 +487,7 @@ public static async Task GetMemoryNodeOOM( catch (Exception ex) { return McpHelpers.FormatError("get_health_parser_memory_node_oom", ex); } } - [McpServerTool(Name = "get_health_parser_significant_waits"), Description("Gets significant individual waits from system_health: one row per wait_info event where a real session's non-BACKUP statement waited at least 500 ms on a wait type that is not idle/background — the wait type, total and signal duration, the wait resource, the session id and the waiting statement. get_wait_stats gives the instance-wide totals and can never name the statement that paid them; this is the individual waits, with their SQL text.")] + [McpServerTool(Name = "get_health_parser_significant_waits"), Description("Gets significant individual waits from system_health: one row per wait_info event where a real session's non-BACKUP statement waited at least 500 ms on a wait type that is not idle/background — the wait type, total and signal duration, the wait resource, the session id and the waiting statement. get_wait_stats gives the instance-wide totals and can never name the statement that paid them; this is the individual waits, with their SQL text. Every answer carries source_observed (whether this server's system_health session has EVER been read into the store) and last_captured_at (the collector's newest capture): an empty window on a server whose session was never read is status unavailable, not a clean bill; an empty window on one that has been read says whether the category was captured and gated out, captured before this window, or never recorded by the engine.")] public static async Task GetSignificantWaits( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -484,6 +512,7 @@ so the two would arrive indistinguishable. var now = windowEnd; var xmls = await DarlingSystemHealthReader.ReadEventXmlAsync( postgres, resolved.ServerId, now.AddHours(-hours_back), now, SystemHealthParser.WaitInfoEvent); + var lastCapturedAt = await DarlingSystemHealthReader.GetLastCaptureAsync(postgres, resolved.ServerId); var rows = xmls .Select(SystemHealthParser.ParseSignificantWait) @@ -494,46 +523,26 @@ so the two would arrive indistinguishable. if (rows.Count == 0) { /* - Three different nothings, and only one of them is good news. Events captured but none - significant is the healthy state and costs no extra query -- we already counted them. - Nothing captured in the window needs the probe to tell a quiet window from a server + The read this family's empty ladder was modelled on (#2484): events captured but none + significant is the healthy state and costs no extra query -- we already counted them; + nothing captured in the window needs the probe to tell a quiet window from a server whose wait_info has never been collected, because "no significant waits" is exactly - what an operator wants to hear and a caller who believes it stops looking. - */ - if (xmls.Count > 0) - { - return McpHelpers.Status( - "empty", - $"{xmls.Count} wait_info event(s) were captured for {resolved.ServerName} in the last {hours_back} hour(s) and none was significant (needs a real session, a non-BACKUP statement, at least {SystemHealthSignificance.SignificantWaitMinDurationMs} ms, and a wait type off the idle list). Events ARE being captured, so this is the healthy answer for this read rather than missing data."); - } - - var everCaptured = await DarlingSystemHealthReader.HasAnyEventOfTypeAsync( - postgres, resolved.ServerId, SystemHealthParser.WaitInfoEvent); - if (everCaptured) - { - return McpHelpers.Status( - "empty", - $"No wait_info events were captured for {resolved.ServerName} in the last {hours_back} hour(s). This server HAS captured them before, so the window is genuinely quiet rather than blind — widen hours_back to reach the most recent events."); - } - - /* - #2511 adds a FOURTH nothing, and it is the one that was being mis-explained. On an engine - whose system_health collector is gated off there is no session to start and no collection - to check, so the advice below is advice about something that cannot exist. The engine - answer goes first because it is the stronger claim; the text after it stays exactly right - for every engine that DOES collect this. + what an operator wants to hear and a caller who believes it stops looking. Since #3541 + A12 the ladder lives in EmptyAsync and all nine reads climb it; only the gate's own + description (the four conditions) is this tool's to word. */ - return await DarlingEngineCapability.NotCollectedStatusAsync( - postgres, resolved.ServerId, resolved.ServerName, SystemHealthCollectorName) - ?? McpHelpers.Status( - "unavailable", - $"No wait_info events have EVER been captured for {resolved.ServerName}, so this is NOT an all-clear — there is nothing here to be clear about. This read is served from the collected system_health ring buffer: check that collection is running for this server and that its system_health session is started before concluding nothing was waiting."); + return await EmptyAsync( + postgres, new Collected(null, resolved.ServerId, resolved.ServerName, rows, xmls.Count, lastCapturedAt), + hours_back, SystemHealthParser.WaitInfoEvent, + $"none was significant (needs a real session, a non-BACKUP statement, at least {SystemHealthSignificance.SignificantWaitMinDurationMs} ms, and a wait type off the idle list)"); } return JsonSerializer.Serialize(new { server = resolved.ServerName, hours_back, + source_observed = true, + last_captured_at = Stamp(lastCapturedAt), wait_count = rows.Count, shown = Math.Min(rows.Count, limit), waits = rows.Take(limit).Select(r => new @@ -561,7 +570,8 @@ signal close to the total is CPU pressure wearing a wait type's name. */ /// . The id rides along for the #2511 engine-capability probe on the zero-row path — /// re-resolving the name there would be a second chance to match a DIFFERENT server, since resolution is /// first-wins over a partial. - private readonly record struct Collected(string? EarlyReturn, int ServerId, string ServerName, List Rows); + private readonly record struct Collected( + string? EarlyReturn, int ServerId, string ServerName, List Rows, int RawEventCount, DateTime? LastCapturedAt); /// /// Resolves the server, validates hours_back + as_of + limit, reads the raw event_xml for @@ -570,20 +580,26 @@ signal close to the total is CPU pressure wearing a wait type's name. */ /// accepts. The seven gated categories pass their /// predicate; System Health passes an EventTime-present /// predicate (ungated, matching the viewer's chart read). + /// Also carries the two facts the payload owes under contract rule 5 (#3541 A12): how many raw + /// events of the type the window held BEFORE the gate (so zero survivors out of a hundred captured + /// reads as healthy, and zero out of zero does not), and the collector's newest capture of any type + /// (so a caller can see whether the source was ever observed at all). The witness is one index-walk + /// per call — see . /// private static async Task> CollectAsync( NpgsqlDataSource postgres, string? serverName, int hoursBack, int limit, string? asOf, string eventType, Func> shred, Func significant) where T : class { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, serverName); - if (error != null) return new Collected(error, 0, "", new List()); + if (error != null) return new Collected(error, 0, "", new List(), 0, null); var validation = McpHelpers.ValidateWindow(hoursBack, asOf, out var windowEnd) ?? McpHelpers.ValidateTop(limit); - if (validation != null) return new Collected(validation, 0, "", new List()); + if (validation != null) return new Collected(validation, 0, "", new List(), 0, null); var now = windowEnd; var xmls = await DarlingSystemHealthReader.ReadEventXmlAsync( postgres, resolved.ServerId, now.AddHours(-hoursBack), now, eventType); + var lastCapturedAt = await DarlingSystemHealthReader.GetLastCaptureAsync(postgres, resolved.ServerId); var rows = new List(); foreach (var xml in xmls) @@ -595,9 +611,96 @@ private static async Task> CollectAsync( } } - return new Collected(null, resolved.ServerId, resolved.ServerName, rows); + return new Collected(null, resolved.ServerId, resolved.ServerName, rows, xmls.Count, lastCapturedAt); } + /* ─────────────────────────── the four nothings (#3541 A12) ─────────────────────────── */ + + /// + /// What zero rows means for one system_health category, which is four different things — and only the + /// first two are good news. Modelled on get_health_parser_significant_waits' three-way ladder (#2484), + /// which was the ONE read of the nine that refused to call a never-read session a clean bill; the other + /// eight answered empty to everything, so a dead system_health session, a collector that + /// never ran, and a healthy quiet hour all read as "no severe errors". Contract rule 5: zero is a + /// measurement, and an absence must say what it is an absence OF. + /// + /// Rung 1 — captured and gated out. Events of the type WERE stored in the window; the + /// shred + significance gate kept none. Healthy, and free: the raw count was taken on the data read. + /// Rung 2 — captured before, not in this window. Quiet window; widening reaches the most recent + /// events, and the message says when the last one was stored so the caller knows how far. + /// Rung 3 — this type never, but the session IS being read. Other categories have been stored, so + /// the ring buffer is reachable and the engine has simply never recorded one of these — for a + /// memory-node OOM or a severe error that is the healthy measurement, not a blind spot, and it must not + /// be called unavailable. Rung 4 — nothing of any type, ever. A dead session or a collector that + /// never ran: unavailable, the #3524 shape, never empty. The #2511 engine-capability probe + /// goes first on this rung because it is the stronger claim (an Azure SQL Database has no session to + /// start), and its text stays exactly right for every engine that does collect this. + /// + /// Every rung carries the same two witness keys the data envelope carries + /// (source_observed, last_captured_at) plus the rung's own evidence, at the top level + /// beside status — the trend family's precedent (#3541 A2): a caller reads the witness without + /// first checking which branch answered. The type-scoped probe runs only on the empty path, so + /// the healthy data path costs one witness query, not two. + /// + private static async Task EmptyAsync( + NpgsqlDataSource postgres, Collected c, int hoursBack, string eventType, string noneQualifiedBecause) + { + /* The type-scoped probe runs on every rung: on rung 1 the type exists in the window so the backward + index walk stops at its first row, and the stamp it returns is THIS type's newest capture rather + than the server-level witness standing in for it. */ + var lastOfType = await DarlingSystemHealthReader.GetLastCaptureOfTypeAsync(postgres, c.ServerId, eventType); + if (c.RawEventCount > 0) + { + return WitnessStatus( + "empty", + $"{c.RawEventCount} {eventType} event(s) were captured for {c.ServerName} in the last {hoursBack} hour(s) and {noneQualifiedBecause}. Events ARE being captured, so this is the healthy answer for this read rather than missing data.", + sourceObserved: true, c.LastCapturedAt, lastCapturedOfTypeAt: lastOfType, eventsInWindow: c.RawEventCount); + } + + if (lastOfType is DateTime seen) + { + return WitnessStatus( + "empty", + $"No {eventType} events were captured for {c.ServerName} in the last {hoursBack} hour(s). This server HAS captured them before (the newest was stored at {Stamp(seen)}), so the window is genuinely quiet rather than blind — widen hours_back to reach the most recent events.", + sourceObserved: true, c.LastCapturedAt, lastCapturedOfTypeAt: seen, eventsInWindow: 0); + } + + if (c.LastCapturedAt is DateTime alive) + { + return WitnessStatus( + "empty", + $"No {eventType} events have been captured for {c.ServerName} at any time, but its system_health session IS being read — the collector last stored an event of another type at {Stamp(alive)} — so for this category the absence is a measurement: the engine has not recorded one. Not a blind spot, and a wider window would not change it.", + sourceObserved: true, alive, lastCapturedOfTypeAt: null, eventsInWindow: 0); + } + + return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, c.ServerId, c.ServerName, SystemHealthCollectorName) + ?? WitnessStatus( + "unavailable", + $"No system_health events of ANY type have EVER been captured for {c.ServerName}, so this is NOT an all-clear — there is nothing here to be clear about. This read is served from the collected system_health ring buffer: check that collection is running for this server and that its system_health session is started before concluding nothing happened.", + sourceObserved: false, lastCapturedAt: null, lastCapturedOfTypeAt: null, eventsInWindow: 0); + } + + /// + /// with the source witness beside status and message: the + /// same source_observed / last_captured_at pair the data envelope carries, plus what this + /// rung measured (last_captured_of_type_at, events_in_window). Top-level rather than under + /// hints so the keys sit in one place whichever branch answered. + /// + private static string WitnessStatus( + string status, string message, bool sourceObserved, DateTime? lastCapturedAt, DateTime? lastCapturedOfTypeAt, int eventsInWindow) + => JsonSerializer.Serialize(new + { + status, + message, + source_observed = sourceObserved, + last_captured_at = Stamp(lastCapturedAt), + last_captured_of_type_at = Stamp(lastCapturedOfTypeAt), + events_in_window = eventsInWindow, + }, McpHelpers.JsonOptions); + + /// The store's naive-UTC stamp in the same ISO shape the rows' event_time uses; null stays null. + private static string? Stamp(DateTime? stamp) => stamp?.ToString("o"); + /// Wraps a single-record shred (0-or-1) as the 0..n sequence expects. private static IEnumerable One(T? record) where T : class => record is null ? Enumerable.Empty() : new[] { record }; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthTools.cs index 39564f525..b77c18670 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHealthTools.cs @@ -45,7 +45,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpHealthTools { - [McpServerTool(Name = "get_server_summary"), Description("Gets a quick health overview for a SQL Server instance: current CPU %, memory usage, recent blocking count, and deadlock count. Use this for a fast health check before drilling into specific areas.")] + /// get_server_summary's description, VERBATIM Lite's (#3541 A10); the cross-SKU census pins them + /// equal. Names the three clocks the payload carries, because the payload used to carry one. + internal const string ServerSummaryDescription = + "Gets a quick health overview for a SQL Server instance: current CPU %, memory usage, recent blocking count, and deadlock count. Use this for a fast health check before drilling into specific areas. THREE CLOCKS, NAMED: cpu_percent is the newest CPU snapshot and cpu_captured_at is its instant; memory_mb is the newest memory snapshot and memory_captured_at is its instant; last_collection is the newest collection of ANY collector for this server - the store's freshness, NOT the age of the two figures above, which can be far older when their own collectors have stopped. blocking_count and deadlock_count cover the counts_window_hours ending now."; + + [McpServerTool(Name = "get_server_summary"), Description(ServerSummaryDescription)] public static async Task GetServerSummary( NpgsqlDataSource postgres, [Description("Server name or display name. Optional if only one server is configured.")] string? server_name = null) @@ -65,9 +70,15 @@ public static async Task GetServerSummary( { server = resolved.ServerName, cpu_percent = summary.CpuPercent, + /* #3541 A10: each latest figure carries ITS OWN clock. last_collection below is the newest + collection of ANY collector — a live collection log beside a dead CPU collector made a + day-old cpu_percent read as current, because the only stamp on the payload was fresh. */ + cpu_captured_at = summary.CpuCapturedAt?.ToString("o"), memory_mb = summary.MemoryMb, + memory_captured_at = summary.MemoryCapturedAt?.ToString("o"), blocking_count = summary.BlockingCount, deadlock_count = summary.DeadlockCount, + counts_window_hours = DarlingHealthReader.ServerSummaryCountsWindowHours, last_collection = summary.LastCollectionTime?.ToString("o") }, McpHelpers.JsonOptions); } @@ -77,26 +88,42 @@ public static async Task GetServerSummary( } } - [McpServerTool(Name = "get_daily_summary"), Description("Gets a daily health summary: overall composite health band (Healthy/Warning/Critical), total wait time, top wait type, unique query count, deadlocks, blocking events, memory pressure (and severe memory pressure), high-CPU samples, collection errors, and actionable alert count for one day. Use this for a quick overview to decide which areas need investigation.")] + [McpServerTool(Name = "get_daily_summary"), Description("Gets a daily health summary: overall composite health band (Healthy/Warning/Critical), total wait time, top wait type, unique query count, deadlocks, blocking events, memory pressure (and severe memory pressure), high-CPU samples, collection errors, and actionable alert count for one day. Use this for a quick overview to decide which areas need investigation. A day before the store's retention_horizon (the oldest day the shortest-lived signal table still holds) returns status=unavailable with data_state=purged rather than a health band: its per-signal counts would be COALESCEd zeros, not measurements, and a zero is only a measurement inside retention. A returned day carries data_state=collected (a verdict), past_horizon (before the horizon but some signal table still holds rows — the purge has not reached it; No Data, non-zero counts real) or no_run_record (inside retention, no collector run recorded — banded on the counts as read, which are measurements there; the collection-error share has no denominator).")] public static async Task GetDailySummary( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Summary date (yyyy-MM-dd), interpreted as a UTC day. Default is today.")] string? summary_date = null) + [Description("Summary date, ISO-8601 yyyy-MM-dd ONLY (e.g. 2026-07-09), interpreted as a UTC day; any other spelling is refused rather than guessed at. Default is today.")] string? summary_date = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); if (error != null) return error; - DateTime? date = null; - if (!string.IsNullOrEmpty(summary_date)) - { - if (!DateTime.TryParse(summary_date, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out var parsed)) - return $"Invalid date format '{summary_date}'. Use yyyy-MM-dd format (e.g., 2026-07-09)."; - date = parsed; - } + /* #3541 A9: exact ISO-8601, refused otherwise — McpHelpers.ParseSummaryDate says why the general + parse this replaced was the wrong tool in a file that already held as_of's strict allowlist. */ + var dateError = McpHelpers.ParseSummaryDate(summary_date, out var date); + if (dateError != null) return dateError; try { var row = await DarlingHealthReader.GetDailySummaryAsync(postgres, resolved.ServerId, date); + + /* #3541 A9: a day before the retention horizon is "unavailable" in the miss vocabulary's own + sense — it existed and is not retrievable now — and it is told apart from a never-collected + day because the two send a caller to different places (nowhere useful, versus collection + health). What the spine still holds for it rides in the hints, named for what it is. */ + if (row.DataState == DailySummaryDataState.Purged) + return McpHelpers.Status( + "unavailable", + $"{row.SummaryDate:yyyy-MM-dd} is before {resolved.ServerName}'s retention_horizon ({row.RetentionHorizon:yyyy-MM-dd}): the per-signal tables the health band reads (deadlocks, blocking, CPU, memory, waits) have been purged for that day, so no health verdict is possible and the counts would be zeros by construction, not by measurement. Longer-lived sources may still record the day — collection_runs and alert_count below are real where non-zero.", + new + { + summary_date = row.SummaryDate.ToString("yyyy-MM-dd"), + overall_health = row.OverallHealth, + data_state = DailySummaryRetention.Label(row.DataState), + retention_horizon = row.RetentionHorizon?.ToString("yyyy-MM-dd"), + collection_runs = row.CollectionRuns, + alert_count = row.AlertCount, + }); + if (!row.HasData) return McpHelpers.Status( "empty", @@ -109,6 +136,11 @@ public static async Task GetDailySummary( summary_date = row.SummaryDate.ToString("yyyy-MM-dd"), overall_health = row.OverallHealth, health_band = row.HealthBand.ToString(), + /* #3541 A9: collected, past_horizon or no_run_record here (purged returned above); the note + says what the zeros are on a non-collected day, null on a collected one. */ + data_state = DailySummaryRetention.Label(row.DataState), + data_note = row.RetentionHorizon is { } horizon ? DailySummaryRetention.Note(row.DataState, horizon, row.SignalSourcesPresent) : null, + retention_horizon = row.RetentionHorizon?.ToString("yyyy-MM-dd"), total_wait_time_sec = row.TotalWaitTimeSec, top_wait_type = row.TopWaitType, unique_queries = row.UniqueQueries, @@ -119,7 +151,13 @@ public static async Task GetDailySummary( memory_critical_events = row.MemoryCriticalEvents, collection_errors = row.CollectionErrors, alert_count = row.AlertCount, - max_block_duration_ms = row.MaxBlockDurationMs + max_block_duration_ms = row.MaxBlockDurationMs, + /* #3539 A2/A3, additive: the figures the band read that the counts alone cannot show — the + run total the error share is a share OF, and the blocking rate over the day's window (null + when the window was too short to normalise). The window is the row's own clamp against + its ReferenceUtc, so an anchored read rates against its as_of, not the process clock. */ + collection_runs = row.CollectionRuns, + blocking_rate_per_hour = ServerHealthClassifier.BlockingRatePerHour(row.BlockingEvents, row.ToSignals().Window), }, McpHelpers.JsonOptions); } catch (Exception ex) @@ -128,7 +166,7 @@ public static async Task GetDailySummary( } } - [McpServerTool(Name = "get_daily_summary_range"), Description("Gets the daily health summary for a SPAN of days rather than one: one row per collected day, each with its composite health band (Healthy/Warning/Critical), total wait time, top wait type, unique query count, deadlocks, blocking events with the peak block wait, high-CPU samples, memory pressure, collection errors and actionable alert count. This is what the desktop viewer's Performance Calendar month grid draws, and it is the read to use when the question is WHICH day rather than how one day went — scan the bands, then call get_daily_summary for the day that stands out. A day on which anything at all was collected appears here even if every signal was quiet (that day is Healthy, not missing), so a gap in the returned days is a gap in COLLECTION.")] + [McpServerTool(Name = "get_daily_summary_range"), Description("Gets the daily health summary for a SPAN of days rather than one: one row per collected day, each with its composite health band (Healthy/Warning/Critical), total wait time, top wait type, unique query count, deadlocks, blocking events with the peak block wait, high-CPU samples, memory pressure, collection errors and actionable alert count. This is what the desktop viewer's Performance Calendar month grid draws, and it is the read to use when the question is WHICH day rather than how one day went — scan the bands, then call get_daily_summary for the day that stands out. A day on which anything at all was collected appears here even if every signal was quiet (that day is Healthy, not missing), so a gap in the returned days is a gap in COLLECTION — INSIDE RETENTION. The per-signal tables age out at the store's shortest retention while the collection log and alert log live longer, so retention_horizon is the oldest day every signal can still answer for; a returned day before it carries data_state=purged (no signal table holds it) or past_horizon (some still do — the purge has not reached it), health_band=NoData and a data_note, NEVER Healthy — a purged day's zeros are absences, and days_before_horizon counts both kinds. A day inside retention with no collector run recorded is data_state=no_run_record — it keeps its band (an alert-only day is Warning), with the caveat that the error share has no denominator. Purged and past_horizon rows carry no verdict.")] public static async Task GetDailySummaryRange( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -162,8 +200,11 @@ means the anchor day is the last one included rather than the first one excluded var fromDate = lastDay.AddDays(-(days_back - 1)); var toDate = lastDay.AddDays(1); - var rows = await DarlingHealthReader.GetDailySummaryRangeAsync( - postgres, resolved.ServerId, fromDate, toDate); + /* The anchor is also the clock the still-forming day's window clamps against (#3525 review): + a backdated as_of must clamp its own "today" against ITSELF, not the process clock. */ + var range = await DarlingHealthReader.GetDailySummaryRangeAsync( + postgres, resolved.ServerId, fromDate, toDate, referenceUtc: windowEnd); + var rows = range.Rows; if (rows.Count == 0) { @@ -199,14 +240,23 @@ edge table it cannot report a healthy server as uncollected. otherwise tell which days they were given from the days they got. */ from_date = fromDate.ToString("yyyy-MM-dd"), to_date = lastDay.ToString("yyyy-MM-dd"), - /* Days WITH data, not days in the span. The two differ exactly where collection has a hole, - and that difference is the most useful thing on this payload. */ + /* Days the spine holds, not days in the span. The two differ exactly where collection has a + hole, and that difference is the most useful thing on this payload — read it together with + days_before_horizon, because a held day before the horizon is a shell, not a collected day. */ day_count = rows.Count, + /* #3541 A9: the store's horizon (reader clock, effective retention — see the reader) and how + many returned days fall before it. */ + retention_horizon = range.RetentionHorizon.ToString("yyyy-MM-dd"), + days_before_horizon = rows.Count(row => row.DataState is DailySummaryDataState.Purged or DailySummaryDataState.PastHorizon), + purged_day_count = rows.Count(row => row.DataState == DailySummaryDataState.Purged), + collected_day_count = rows.Count(row => row.DataState == DailySummaryDataState.Collected), days = rows.Select(row => new { summary_date = row.SummaryDate.ToString("yyyy-MM-dd"), overall_health = row.OverallHealth, health_band = row.HealthBand.ToString(), + data_state = DailySummaryRetention.Label(row.DataState), + data_note = DailySummaryRetention.Note(row.DataState, range.RetentionHorizon, row.SignalSourcesPresent), total_wait_time_sec = row.TotalWaitTimeSec, top_wait_type = row.TopWaitType, unique_queries = row.UniqueQueries, @@ -218,6 +268,9 @@ and that difference is the most useful thing on this payload. */ collection_errors = row.CollectionErrors, alert_count = row.AlertCount, max_block_duration_ms = row.MaxBlockDurationMs, + /* #3539 A2/A3, additive — see get_daily_summary's members of the same names. */ + collection_runs = row.CollectionRuns, + blocking_rate_per_hour = ServerHealthClassifier.BlockingRatePerHour(row.BlockingEvents, row.ToSignals().Window), }), }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs index 7b7506301..e57692fea 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpHostService.cs @@ -600,8 +600,9 @@ surface beside get_store_metrics. */ snapshots, no live hit). The Dashboard-only CASE enrichment (latch severity/description/ recommendation, spinlock description) and the #1410 client-side classifications (plan-cache bloat_level, cpu-scheduler pressure_level) are reproduced service-side so the full result shape - is served; Darling's delta collectors store no sample_interval_seconds, so per-second rates are - derived from the LAG interval. */ + is served. Per-second rates divide by each row's stored sample_interval_seconds (V127, #3540), + falling back to the LAG interval only for pre-V127 rows, and are null when the latest interval + was unknowable rather than 0. */ .WithGeminiCompatibleTools() /* get_pg_wait_stats — PostgreSQL wait events for an Aurora target, paired with the pg_wait_stats collector. A separate tool from get_wait_stats rather than a widened @@ -624,6 +625,15 @@ pg_statement_stats collector. Carries Aurora's I/O source split and per-statemen target can capture a plan at all, facet by facet with the remedy for each, which is the read somebody needs the moment the plans one comes back empty. */ .WithGeminiCompatibleTools() + /* get_pg_logging_audit (#3607) - the rest of the logging surface, in readiness's shape: + log_lock_waits, log_temp_files, log_autovacuum_min_duration, log_checkpoints, + log_connections / log_disconnections and log_min_duration_statement, each judged from + the stored pg_server_config snapshot with what it unlocks, the recommended value and its + cost, and the remedy in the hosting flavour's syntax. Registered beside the plan tools + because it is the other half of one onboarding question - is this target telling us + everything it could - and lists plan capture's own settings with a pointer to the + readiness read rather than judging them twice. */ + .WithGeminiCompatibleTools() /* get_pg_wraparound_risk — XID/MultiXact freeze headroom, the highest-consequence PostgreSQL signal and one with no SQL Server counterpart. Not Aurora-gated. */ .WithGeminiCompatibleTools() diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs index 062ac5b78..5c467a40f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInstructions.cs @@ -62,12 +62,16 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) ## Tool Reference - This server exposes 151 tools. 86 are the same names Performance Monitor Lite exposes, spanning diagnostic analysis, plan analysis, data reads at core and diagnostic depth, resource contention + jobs, trends, system-health parse-on-read, alerts + health overview, and the Default Trace. The remaining 65 are unique to Darling: thirty-three are the PostgreSQL reads (Aurora/PostgreSQL targets only Darling's central store can hold), eight are the Custom Views tools (seven manage the saved views — the one view-authoring write surface — and `describe_custom_view_catalog` returns the read-only compose vocabulary those authoring tools draw from), eight are the custom-alert-rule tools (`create_custom_alert_rule` / `update_custom_alert_rule` / `delete_custom_alert_rule` manage the user-authored alert rules, the one alert-authoring write surface, while `get_custom_alert_rule` / `list_custom_alert_rules` read them back, `validate_custom_alert_rule` checks a rule definition against the same compose catalog without saving it, `test_custom_alert_rule` evaluates a rule's metric NOW on each in-scope server and reports whether it would breach without delivering or persisting anything, and `list_custom_alert_templates` lists the built-in starter rule templates to browse and create from), five are alert-tuning write tools (`update_alert_settings` tunes the alert engine's thresholds; `create_mute_rule` / `update_mute_rule` / `delete_mute_rule` / `set_mute_rule_enabled` manage the mute rules, `update_mute_rule` editing a rule in place and `set_mute_rule_enabled` taking one out of force and putting it back — both without destroying it or resetting its creation date) that write only the shared alert configuration in the monitoring store, two are server-onboarding write tools (`add_servers` bulk-adds monitored servers; `remove_server` removes one) that add or remove rows in the monitoring store's monitored-server registry, `get_fleet_overview` and `get_ag_health` are the two cross-server reads of the fleet's CURRENT state only a central store can answer, `get_sweep_reports` is the cross-server read WITH MEMORY — the scheduled fleet sweep's persisted whole-fleet reports (the timeline with each sweep's document embedded, one sweep in full with its would-have-paged ledger, and the watch-item worklist), `get_store_metrics` reads the monitoring store's OWN hourly size/compression/growth series for capacity forecasting, `get_store_log` reads what the monitoring store's OWN PostgreSQL server log recorded as a per-class census with its capture denominator (the self-monitoring that shows the store's half of a client-side symptom), `get_collector_cost` reads the tool's OWN per-collector cost on the monitored servers (the self-monitoring that flags a collector regressing into a hog), `get_collector_stall_probes` reads the out-of-band server-wide wait samples this tool takes while one of its own collectors is stalled mid-read — the only surface here that reports what a monitored instance was doing inside the window the sequential sweep records nothing in — `get_oversized_plan_backlog` reads the backlog of cached plans the capture cap declined and what the out-of-band sweep has since done about each one (the self-monitoring that tells a sweep whose fetch half is working from one that has never once succeeded), and `get_blocking` is Darling's name for the blocked-process-report read that Lite exposes as `get_blocked_process_reports` — a naming difference, not a capability gap. Every data-read tool reads the data the collectors already captured into the store — a stored read, never a live query against the monitored server. + This server exposes 152 tools. 86 are the same names Performance Monitor Lite exposes, spanning diagnostic analysis, plan analysis, data reads at core and diagnostic depth, resource contention + jobs, trends, system-health parse-on-read, alerts + health overview, and the Default Trace. The remaining 66 are unique to Darling: thirty-four are the PostgreSQL reads (Aurora/PostgreSQL targets only Darling's central store can hold; two of them are the target-onboarding pair — `get_pg_plan_capture_readiness` judges plan capture's preconditions facet by facet and `get_pg_logging_audit` judges the rest of the logging surface setting by setting from the stored configuration, each with the remedy per finding in the hosting flavour's syntax — which together answer whether a target is telling us everything it could), eight are the Custom Views tools (seven manage the saved views — the one view-authoring write surface — and `describe_custom_view_catalog` returns the read-only compose vocabulary those authoring tools draw from), eight are the custom-alert-rule tools (`create_custom_alert_rule` / `update_custom_alert_rule` / `delete_custom_alert_rule` manage the user-authored alert rules, the one alert-authoring write surface, while `get_custom_alert_rule` / `list_custom_alert_rules` read them back, `validate_custom_alert_rule` checks a rule definition against the same compose catalog without saving it, `test_custom_alert_rule` evaluates a rule's metric NOW on each in-scope server and reports whether it would breach without delivering or persisting anything, and `list_custom_alert_templates` lists the built-in starter rule templates to browse and create from), five are alert-tuning write tools (`update_alert_settings` tunes the alert engine's thresholds; `create_mute_rule` / `update_mute_rule` / `delete_mute_rule` / `set_mute_rule_enabled` manage the mute rules, `update_mute_rule` editing a rule in place and `set_mute_rule_enabled` taking one out of force and putting it back — both without destroying it or resetting its creation date) that write only the shared alert configuration in the monitoring store, two are server-onboarding write tools (`add_servers` bulk-adds monitored servers; `remove_server` removes one) that add or remove rows in the monitoring store's monitored-server registry, `get_fleet_overview` and `get_ag_health` are the two cross-server reads of the fleet's CURRENT state only a central store can answer, `get_sweep_reports` is the cross-server read WITH MEMORY — the scheduled fleet sweep's persisted whole-fleet reports (the timeline with each sweep's document embedded, one sweep in full with its would-have-paged ledger, and the watch-item worklist), `get_store_metrics` reads the monitoring store's OWN hourly size/compression/growth series for capacity forecasting, `get_store_log` reads what the monitoring store's OWN PostgreSQL server log recorded as a per-class census with its capture denominator (the self-monitoring that shows the store's half of a client-side symptom), `get_collector_cost` reads the tool's OWN per-collector cost on the monitored servers (the self-monitoring that flags a collector regressing into a hog), `get_collector_stall_probes` reads the out-of-band server-wide wait samples this tool takes while one of its own collectors is stalled mid-read — the only surface here that reports what a monitored instance was doing inside the window the sequential sweep records nothing in — `get_oversized_plan_backlog` reads the backlog of cached plans the capture cap declined and what the out-of-band sweep has since done about each one (the self-monitoring that tells a sweep whose fetch half is working from one that has never once succeeded), and `get_blocking` is Darling's name for the blocked-process-report read that Lite exposes as `get_blocked_process_reports` — a naming difference, not a capability gap. Every data-read tool reads the data the collectors already captured into the store — a stored read, never a live query against the monitored server. ### Reading an empty result When a read comes back with no data, the `status` word says WHICH kind of nothing it is, and the four are not interchangeable. `empty` is a true negative: we looked and there was nothing to find. `unavailable` means this server could have that data and does not have it right now, so collection health is worth a look. `not_collected` means this server does not collect that at all — and when the reason is the ENGINE, the gap is PERMANENT: the collector serving that read does not run on this server's engine (an Azure SQL Database has no system_health session, no default trace and no SQL Agent; a PostgreSQL target collects none of the SQL Server signals at all, and the `get_pg_*` reads are the ones that answer there), so there is no session to start, no collector to enable, and nothing to check. The message names the engine and the collector. Do not send anyone to go and fix it. `precondition` is the one that IS worth acting on: this server could have that data, the collector is running, and a setup step on the monitored server is in the way — a Query Store that is off or has gone READ_ONLY, an Extended Events capture session that is not running, an extension that was never created, a grant the monitoring login was refused. The message names the precondition, quotes what the monitored server itself said, and gives the statement or grant that satisfies it. It is re-derived on EVERY read rather than decided when the connection was made, so once somebody does the thing it asked for the next call answers with data — usually with nothing to restart on the monitoring side. A few preconditions are the exception and SAY SO IN THEIR OWN MESSAGE: the fact that gates them is read once when the service connects to that server and cached for the connection's life, so satisfying them also needs the service to reconnect before collection resumes. Read the message rather than assuming the general case — it tells you which kind you have, and telling somebody to retry a connect-scoped one without reconnecting sends them round a loop that never terminates. + ### PostgreSQL waits come from one of three instruments + + A PostgreSQL target's wait history is fed by exactly ONE of three instruments, chosen once when the service connects to it, in this order of preference: Aurora native > the `pg_wait_sampling` extension > the service-side sampler. Aurora targets get `engine_cumulative` — the engine's own wait counters and measured wait TIME, read by `get_pg_wait_stats`. Stock and RDS targets are read by `get_pg_wait_sampling`, whose `instrument` field says which of the other two fed the rows: `extension_sampled` is the extension's in-engine 10 ms profiler; `service_sampled` means the extension is not installed and this service polled `pg_stat_activity` once a second for a 30-second window every five minutes instead. That last tier is a FLOOR, not parity — it under-counts waits shorter than a second and sees nothing between windows, and `instrument_note` says so in the answer — but it is what stops a first-install stock target from reading as "no waits at all". A count only means something beside its instrument: read `instrument` before comparing two servers' wait profiles, and treat `service_sampled` shares as trustworthy for steady fractions of the server's time and approximate for rare short events. Installing `pg_wait_sampling` (a `shared_preload_libraries` entry, then `CREATE EXTENSION` in the monitored database, then a reconnect) moves a target to the finer tier with no other change. + ### Asking about a PAST window Every tool below that takes `hours_back` also takes `as_of`: an optional ISO-8601 UTC instant that moves the END of the window off "now". `hours_back` stays the window's LENGTH. So the four hours around last Tuesday 03:00 is `as_of=2026-08-19T05:00:00Z, hours_back=4` — not `hours_back=170`. @@ -78,6 +82,7 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) - An unparseable `as_of`, or one in the future, is REFUSED with a message rather than quietly answered as "now" — a read that silently reverts to now is indistinguishable from a correct one. - An `as_of` older than anything the store still holds is NOT refused. It returns the read's normal `empty` / `unavailable` status, which means exactly what it says: we looked in the window you named and there was nothing in it. - Tools that take no window at all (latest-snapshot reads like `get_memory_stats`, `get_file_io_stats`, `get_index_usage`, and the configuration reads) do not take `as_of` — they read the newest row, and there is no window to move. + - **Latest is a time.** Every latest-snapshot read publishes `captured_at` — the snapshot's own collection instant — and the anchored ones publish `age_seconds` against the window's end. Read it before treating a "current" figure as current: the newest row a store holds is as old as its collector's last successful run, and the configuration family is captured on CONNECT, so a "current" setting can be days old. Where a latest read takes `hours_back`, its description says which of two things that means: the span SEARCHED for the newest snapshot (`get_latch_stats`, `get_cpu_scheduler_pressure`, `get_plan_cache_bloat` — a snapshot older than that is `unavailable`, not served as current), or a span READ beside the snapshot (`get_resource_semaphore` / `get_memory_grants` return `grants[]`, the newest snapshot, AND `window[]`, the peak / floor / summed-delta aggregate over every snapshot in the hours). `get_server_summary` carries three clocks by name — `cpu_captured_at`, `memory_captured_at`, and `last_collection` (the newest collection of ANY collector, which is the store's freshness and not the age of the two figures). - The analysis family DOES take it (#2506), and the anchor reaches the ENGINE rather than stopping at the tool: `get_analysis_facts` and `analyze_server` re-run fact collection and scoring over the anchored window, and `analyze_server`'s anomaly detection moves with it, so the window is compared against the hour-of-day x day-of-week baseline for the hours it actually covers instead of for the hours you happen to be asking in. `compare_analysis` hangs BOTH windows off the anchor, since `baseline_hours_back` has always been measured from the comparison window's end. `get_analysis_findings` is the odd one and worth reading twice: its window is on ANALYSIS TIME, so anchoring it asks what a scheduled analysis pass was SAYING then, which is a different question from re-analyzing that window now (that is `analyze_server` with the same anchor). - `analyze_server` with an `as_of` is EXPLORATORY and does NOT persist its findings; the result says so in `persisted` / `persistence_note`. A finding row is stamped with the time the analysis RAN, and `get_analysis_findings` and the viewer's Recommendations tab treat the newest `analysis_time` as the server's CURRENT state — so writing a backdated run would make last week's findings today's headline and would inflate the occurrence stats of any live incident sharing a story path. Run it without `as_of` when you want the present analyzed and recorded. - `get_pvs_stats` and `get_fleet_overview` do not take it. Each mixes a latest-snapshot measurement with a windowed one, so anchoring only the windowed half would return a result whose two halves describe different instants. `get_store_metrics` does not take it either: it windows in DAYS over the store's own growth series. @@ -86,12 +91,12 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `analyze_server` | Runs the inference engine: scores facts, traverses relationship graph, returns evidence-backed findings with severity and recommended next tools. A remediable finding also carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), with a two-sided risk-disclosure header on destructive changes; advisory only, never executed. Force-plan findings additionally carry `structured_remediation`: the verdict as machine-readable fields (eligible + named blockers), evidence, and split force/unforce/verify SQL. With `as_of` it analyzes a PAST window — anomaly baseline included — and is EXPLORATORY: the findings come back in full but are NOT persisted, which `persisted` / `persistence_note` state on every result | `server_name`, `hours_back` (default 4), `as_of` | - | `get_analysis_facts` | Exposes raw scored facts from the collect+score pipeline — every observation the engine sees with base severity, amplifiers, and metadata | `server_name`, `hours_back` (default 4), `source` (filter), `min_severity`, `as_of` | - | `compare_analysis` | Compares two time periods (e.g., peak vs off-peak, before vs after a change) showing severity deltas for each fact. When NEITHER window produced facts the result is `unavailable` rather than an all-zero comparison, because "nothing to compare" is not "nothing changed"; when only ONE window is empty the payload carries a `caveat` saying so, since every fact then counts as new or resolved by default. `baseline_hours_back` is measured from the comparison window's END, so `as_of` moves BOTH windows together | `server_name`, `hours_back` (default 4), `baseline_hours_back` (default 28), `as_of` | + | `analyze_server` | Runs the inference engine: scores facts, traverses relationship graph, returns evidence-backed findings with severity and recommended next tools. Each finding's `confidence` is an EVIDENCE score (0.20 for the symptom alone, more as the root fact's amplifier checks match and the chain deepens — a lone uncorroborated symptom is 0.20, never 1.0) and `confidence_basis` says in words what it rests on; rank by `severity` for impact and read `confidence` as how much of the engine's own corroboration showed up. A remediable finding also carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), with a two-sided risk-disclosure header on destructive changes; advisory only, never executed. Force-plan findings additionally carry `structured_remediation`: the verdict as machine-readable fields (eligible + named blockers), evidence, and split force/unforce/verify SQL. With `as_of` it analyzes a PAST window — anomaly baseline included — and is EXPLORATORY: the findings come back in full but are NOT persisted, which `persisted` / `persistence_note` state on every result | `server_name`, `hours_back` (default 4), `as_of` | + | `get_analysis_facts` | Exposes raw scored facts from the collect+score pipeline — every observation the engine sees with base severity, amplifiers, and metadata (an ANOMALY_* fact's `baseline_confidence` is the baseline's trustworthiness, not a finding's `confidence`) | `server_name`, `hours_back` (default 4), `source` (one of the engine's 15 registered sources — anomaly, bad_actor, blocking, config, coverage, cpu, database_config, disk, io, jobs, memory, queries, sessions, tempdb, waits; an unknown value is refused with the set), `min_severity`, `as_of` | + | `compare_analysis` | Compares two time periods (e.g., peak vs off-peak, yesterday vs today, the windows around a change), banding each fact worse / better / stable by how far its VALUE moved on the server's own scale — in the stored per-server baseline's robust sigma where one exists (`delta_sigma`, `band_source` `baseline`), otherwise only when the value moved at least a quarter of the larger side AND registers a quarter of the way up its own severity ladder (`band_source` `absolute`); `band_rules` states the rules on every payload. Rows are grouped into physical-cause `families` (one I/O stall is one family row), and `BAD_ACTOR_` appearances are `plan_cache_churn`, not new or resolved issues. A verdict is a DIFFERENCE, not an experiment: same-hour-yesterday at N=1 vs N=1 cannot show that a change caused anything. When NEITHER window produced facts the result is `unavailable` rather than an all-zero comparison, because "nothing to compare" is not "nothing changed"; when only ONE window is empty the payload carries a `caveat` saying so; a partly collected side flags every verdict row with `coverage_caveat`. `baseline_hours_back` is measured from the comparison window's END, so `as_of` moves BOTH windows together | `server_name`, `hours_back` (default 4), `baseline_hours_back` (default 28), `as_of` | | `audit_config` | Edition-aware configuration audit: evaluates CTFP, MAXDOP, max memory, and max worker threads against best practices | `server_name` | - | `get_analysis_findings` | Retrieves persisted findings from previous analysis runs (the service also analyzes on its own schedule, every 30 minutes per server), deduplicated to one entry per diagnostic chain (`story_path_hash` + `incident_id`): the latest occurrence plus `occurrences`/`first_seen`/`last_seen`/`peak_severity` spanning the window; each remediable finding carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the persisted action, advisory only and never executed; force-plan findings additionally carry `structured_remediation` (verdict + evidence + split artifacts, machine-readable). Its window is on ANALYSIS TIME, so `as_of` asks what analysis was saying then rather than re-analyzing that window now | `server_name`, `hours_back` (default 24), `as_of` | - | `mute_analysis_finding` | Mutes a finding pattern by story_path_hash so it won't appear in future runs | `story_path_hash` (required), `server_name`, `reason` | + | `get_analysis_findings` | Retrieves persisted findings from previous analysis runs (each with `confidence_basis`: rows persisted before `confidence` measured corroboration are labelled `path-shape (pre-#3538)` — under that formula a lone symptom read 1.0, so do not read those as corroborated) (the service also analyzes on its own schedule, every 30 minutes per server), deduplicated to one entry per diagnostic chain (`story_path_hash` + `incident_id`): the latest occurrence plus `occurrences`/`first_seen`/`last_seen`/`peak_severity` spanning the window; each remediable finding carries `remediation_command` — the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the persisted action, advisory only and never executed; force-plan findings additionally carry `structured_remediation` (verdict + evidence + split artifacts, machine-readable). Its window is on ANALYSIS TIME, so `as_of` asks what analysis was saying then rather than re-analyzing that window now | `server_name`, `hours_back` (default 24), `as_of` | + | `mute_analysis_finding` | Mutes a finding pattern by story_path_hash so it won't appear in future runs. Reports what the write did: `registered`, and `matched_now` — how many stored findings in scope carry the hash (status `muted_unmatched` when 0: the mute is kept, but check the hash) | `story_path_hash` (required), `server_name`, `reason` | ### Plan-analysis tools @@ -115,16 +120,16 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | `get_wait_stats` | Top wait types aggregated over the window (wait/signal/resource ms, signal %) | `server_name`, `hours_back` (default 24), `limit` (default 20), `as_of` | | `get_wait_trend` | A single wait type's per-second trend over time | `wait_type` (required), `server_name`, `hours_back` (default 24), `as_of` | | `get_wait_types` | The distinct wait types observed on the server (heaviest first) — pick a `wait_type` for get_wait_trend. An empty result distinguishes a quiet window (`empty`, widen `hours_back`) from a server no wait stats have ever been stored for (`unavailable`) | `server_name`, `hours_back` (default 24), `as_of` | - | `get_memory_stats` | Latest memory snapshot: physical / buffer pool / plan cache / utilization %, memory model | `server_name` | + | `get_memory_stats` | Latest memory snapshot: physical / buffer pool / plan cache / utilization %, memory model; `captured_at` | `server_name` | | `get_memory_clerks` | Latest top memory consumers by clerk type. An empty result is `unavailable`, never a quiet period — a live SQL Server always has clerks, so nothing retained means the collector has not run or its rows aged out | `server_name` | - | `get_file_io_stats` | Latest per-file I/O: reads/writes/bytes/stall and computed read/write latency | `server_name` | + | `get_file_io_stats` | Latest per-file I/O: reads/writes/bytes/stall and computed read/write latency; `captured_at` | `server_name` | | `get_tempdb_trend` | TempDB space over time (user / internal / version store / unallocated) + top consumer | `server_name`, `hours_back` (default 24), `as_of` | - | `get_perfmon_stats` | Latest perfmon counters (value + delta); filter by counter / instance | `server_name`, `counter_name`, `instance_name` | - | `get_top_queries_by_cpu` | Expensive queries from query stats (plan cache) with query_hash / sql_handle; `cpu_attribution.attributed_cpu_ratio` says how much of the box's measured CPU the returned rows explain | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `parallel_only`, `min_dop`, `as_of` | + | `get_perfmon_stats` | Latest perfmon counters (value + delta); filter by counter / instance; `captured_at` | `server_name`, `counter_name`, `instance_name` | + | `get_top_queries_by_cpu` | Expensive queries from query stats (plan cache) with query_hash / sql_handle; `cpu_attribution.attributed_cpu_ratio` says how much of the box's measured CPU the returned rows explain. `parallel_only` / `min_dop` are applied IN the query before the ranking and the cap (`filter_applied` names the floor), so the page is the top-N of the parallel population | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `parallel_only`, `min_dop`, `as_of` | | `get_top_procedures_by_cpu` | Most expensive stored procedures by total CPU, with the same `cpu_attribution` disclosure | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `as_of` | | `get_query_store_top` | Expensive queries from Query Store with query_id / plan_id (survives restarts) | `server_name`, `hours_back` (default 24), `top` (default 20), `database_name`, `as_of` | | `get_query_heatmap` | The desktop viewer's Query Heatmap as a TABLE: how many DISTINCT queries fell into each (time bin x log-magnitude bucket) cell, with the most-executed query in each cell. The only query read with a TIME axis — `get_top_queries_by_cpu` ranks a whole window and cannot show that the window had a quiet half and a bad half, which is the first question about an incident that has already ended. Bins are **5 minutes** wide by default because that is exactly what the desktop viewer uses, so both surfaces draw the same picture; raise `bucket_minutes` to cover a longer window in fewer cells (it is the lever to reach for before the cap). The seven magnitude buckets are the viewer's, in the metric's own unit, and the labels come back with the result. `limit` caps CELLS, and truncation drops the OLDEST bins rather than the least interesting cells — `first_time_bin` / `last_time_bin` say which slice came back. Zero cells is THREE states and the read says which: never collected (`unavailable`, nobody looked), nothing collected in the window (`empty`, widen it), or collected and genuinely idle — captures exist and every one recorded zero executions (`empty`) | `server_name`, `hours_back` (default 24), `metric` (default `duration`), `database_name`, `bucket_minutes` (default 5), `limit` (default 500), `as_of` | - | `get_query_store_regressions` | Queries whose Query Store performance got WORSE: each (database, query_id) group's averages inside the recent window vs its BASELINE — every capture collected BEFORE that window. Baseline vs recent duration / CPU / logical reads with a regression percent each, the execution-count-weighted `additional_duration_ms` (the ranking key: a 5 ms regression run a million times outranks a 5-second one run twice), the plan counts on both sides, and a severity band. `get_query_store_top` answers what is EXPENSIVE and the costliest query is usually the one that always was; this answers what CHANGED. Kept only where average CPU regressed > 25%. Zero rows is FOUR states and the read says which: never collected (`unavailable`), no BASELINE because all history falls inside the window (`unavailable`, and NOT a clean bill of health — shorten hours_back), nothing collected in the window (`empty`, widen it), or a genuine all-clear (`empty`) | `server_name`, `hours_back` (default 24), `database_name`, `limit` (default 50), `as_of` | + | `get_query_store_regressions` | Queries whose Query Store performance got WORSE: each (database, query_id) group's averages inside the recent window vs its BASELINE — every capture collected BEFORE that window. Baseline vs recent duration / CPU / logical reads with a regression percent each, the execution-count-weighted `additional_duration_ms` (the ranking key: a 5 ms regression run a million times outranks a 5-second one run twice), the plan counts on both sides, and a severity band. `get_query_store_top` answers what is EXPENSIVE and the costliest query is usually the one that always was; this answers what CHANGED. Kept only where average CPU regressed > 25%. Zero rows is FOUR states and the read says which: never collected (`unavailable`), no BASELINE because all history falls inside the window (`unavailable`, and NOT a clean bill of health — shorten hours_back), nothing collected in the window (`empty`, widen it), or a genuine all-clear (`empty`) A percent whose baseline side is 0 (e.g. no logical reads before the window, 50k inside it) has no denominator and is null with the reason under `undefined_percents` — never 0, which would read as no change; the ranking key is an absolute delta and exists for every row, and `severity` is null when the duration percent is. | `server_name`, `hours_back` (default 24), `database_name`, `limit` (default 50), `as_of` | | `list_servers` | All monitored servers with collection-freshness status and last collection time. Each row names its engine: `engine_kind` is the registry token (`sqlserver` / `postgres` / `aurora-postgres`, null before any connect stamps the row) and `engine_version` the engine-aware version label ("SQL Server 2022", "PostgreSQL 18"); `sql_version` is a DEPRECATED alias of `engine_version` kept for existing consumers — do not read an engine from its name. Plus `peer_fleets` — the declared SIBLING Darling stores and what each covers (disclosure only; this server cannot read them) and `peer_note`, which says what an EMPTY `peer_fleets` does and does not prove | none | | `get_collection_health` | Per-collector health (running / failing / stale) over the last 7 days, plus the server's sweep_pressure block: a `verdict` for SUSTAINED demand (a SATURATED body collects at a multiple of its configured cadence with every collector healthy) and a separate `peak_cycle_risk` for a SINGLE sweep (BODY_OVERRUN means one scheduled body cannot fit the budget even when the verdict reads OK, the signature of one infrequent heavy collector; `peak_collector` names it). Per-collector rows carry `avg_duration_ms`, `p95_duration_ms` and `max_duration_ms`: a mean far below the p95 means the collector's runs come in two sizes and the mean describes neither | `server_name` | | `get_collection_log` | The RAW per-run collection log behind that rollup: one row per collector run with total duration split into time on the monitored server and time on the store, rows collected, status and any error. Reach for it when the rollup reads HEALTHY and collection still looks wrong, or to see what a collector was doing during a specific window. Newest first, or SLOWEST first when `min_duration_ms` is supplied — a duration floor under newest-first ordering cannot reach the tail, so the two are one decision and `order` names which you got. Both filters are applied in SQL BEFORE the cap, so `run_count` and `truncated` describe the MATCHING rows. `hours_back` is the span you asked for; `oldest_returned_collection_time` and `newest_returned_collection_time` bound the PAGE you got (under the default ordering that is also the reach; under a duration floor the page is a cost-ranked sample and its oldest row says nothing about reach), and the cap can make those differ by orders of magnitude — read them before concluding anything from the rows. An empty result distinguishes THREE states: filters that matched nothing (`empty`, and it says nothing about the window as a whole), a quiet window (`empty`, widen it), and a server that has never collected (`unavailable`, collection is not running) | `server_name`, `hours_back`, `limit`, `as_of`, `collector_name`, `min_duration_ms` | @@ -138,43 +143,43 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `get_blocking` | Recent blocked/blocking pairs from the blocked-process-report XE + the always-on DMV fallback | `server_name`, `hours_back` (default 24), `limit` (default 30), `dedup_key` (optional), `as_of` | - | `get_deadlocks` | Recent deadlocks: victim process/SQL + a process summary | `server_name`, `hours_back` (default 24), `limit` (default 20), `dedup_key` (optional), `as_of` | - | `get_deadlock_detail` | The raw deadlock graph XML for the recent deadlocks | `server_name`, `hours_back` (default 24), `limit` (default 5), `dedup_key` (optional), `as_of` | - | `get_blocked_process_xml` | The raw blocked-process-report XML | `server_name`, `hours_back` (default 24), `limit` (default 5), `as_of` | - | `get_long_query_completions` | Longest completed queries (rpc/batch over the trace threshold) + attentions/cancels from the opt-in long-query trace, duration DESC (empty until the collector is enabled) | `server_name`, `hours_back` (default 24), `limit` (default 30), `as_of` | + | `get_blocking` | Recent blocked/blocking pairs from the blocked-process-report XE + the always-on DMV fallback, newest first. The page is bounded by `limit`, not `hours_back`: read `truncated` and `oldest_returned_event_time` before treating the page as the window | `server_name`, `hours_back` (default 24), `limit` (default 30), `dedup_key` (optional), `as_of` | + | `get_deadlocks` | Recent deadlocks: victim process/SQL + a process summary, newest first. Page bounded by `limit`; `truncated` and `oldest_returned_deadlock_time` say what it reached | `server_name`, `hours_back` (default 24), `limit` (default 20), `dedup_key` (optional), `as_of` | + | `get_deadlock_detail` | The raw deadlock graph XML for the recent deadlocks, newest first; `limit` counts graphs, `truncated` says the window held more | `server_name`, `hours_back` (default 24), `limit` (default 5), `dedup_key` (optional), `as_of` | + | `get_blocked_process_xml` | The raw blocked-process-report XML, newest first; `limit` counts reports with XML, `truncated` says the window held more | `server_name`, `hours_back` (default 24), `limit` (default 5), `as_of` | + | `get_long_query_completions` | The window's SLOWEST completed queries (rpc/batch over the trace threshold) + attentions/cancels from the opt-in long-query trace, duration DESC (empty until the collector is enabled). `truncated` says the window held more than `limit`; the page's time stamps bound the slowest runs, not the reach | `server_name`, `hours_back` (default 24), `limit` (default 30), `as_of` | | `get_blocking_trend` | Per-minute blocking-incident counts over time (XE, DMV-snapshot fallback). An empty result distinguishes a genuine all-clear (`empty`, with the collector run counts in `hints` so you can see how many captures the window actually holds) from a window no collector covered (`unavailable`), which is NOT an all-clear | `server_name`, `hours_back` (default 24), `as_of` | | `get_deadlock_trend` | Per-minute deadlock counts over time. An empty result distinguishes a genuine all-clear (`empty`, with the collector run counts in `hints` so you can see how many captures the window actually holds) from a window no collector covered (`unavailable`), which is NOT an all-clear | `server_name`, `hours_back` (default 24), `as_of` | | `get_lock_wait_trend` | Every LCK% wait type's wait milliseconds per SECOND at each collection — the aggregate lock-wait lane. The two trends above count incidents and `get_wait_trend` charts ONE named wait type; this is the whole lock family as a rate, which is what shows lock pressure rising when no single type dominates. Rate rather than raw delta, so it compares across servers on different cadences. An empty result distinguishes a genuinely quiet window (`empty`, widen `hours_back`) from a server no wait stats have ever been stored for (`unavailable`), which is NOT a report of a server without lock contention | `server_name`, `hours_back` (default 24), `as_of` | | `get_session_stats` | Latest per-application connection/session counts (running/sleeping/dormant) + resource totals | `server_name` | - | `get_active_queries` | Captured running-query snapshots over the window (waits, CPU, blocking, grants) | `server_name`, `hours_back` (default 1), `database_name`, `blocking_only`, `limit` (default 50), `as_of` | - | `get_waiting_tasks` | Individual waiting tasks captured at collection time | `server_name`, `hours_back` (default 1), `limit` (default 30), `as_of` | + | `get_active_queries` | Captured running-query snapshots over the window (waits, CPU, blocking, grants). `database_name` / `blocking_only` are applied IN the query; `total_snapshots` counts the filtered population, `snapshots_returned` the page, `truncated` says the population held more. Head blockers a victim names are never stripped (`is_head_blocker`); a victim whose blocker is absent says why in `blocker_not_shown` (`not_captured` / `filtered` / `past_page`) | `server_name`, `hours_back` (default 1), `database_name`, `blocking_only`, `limit` (default 50), `as_of` | + | `get_waiting_tasks` | Individual waiting tasks captured at collection time, newest capture first. Page bounded by `limit`; `truncated` and `oldest_returned_collection_time` say what it reached | `server_name`, `hours_back` (default 1), `limit` (default 30), `as_of` | | `get_server_config_changes` | sp_configure changes, diffed from config snapshots | `server_name`, `hours_back` (default 168), `as_of` | | `get_database_config_changes` | sys.databases setting changes, diffed from config snapshots | `server_name`, `hours_back` (default 168), `as_of` | | `get_trace_flag_changes` | Trace flags enabled/disabled/modified, diffed from config snapshots | `server_name`, `hours_back` (default 168), `as_of` | - | `get_database_scoped_config` | Latest database-scoped configuration (MAXDOP, legacy CE, ...) | `server_name`, `database_name` | + | `get_database_scoped_config` | Latest database-scoped configuration (MAXDOP, legacy CE, ...) as of `captured_at` (captured on connect) | `server_name`, `database_name` | | `get_query_store_health` | Per-database Query Store health (latest hourly snapshot) — actual vs desired state, readonly_reason decoded, storage vs cap, cleanup thresholds | `server_name`, `database_name` | - | `get_server_config` | CURRENT sys.configurations (latest snapshot) — what CTFP / MAXDOP / max memory are set to now | `server_name` | - | `get_database_config` | CURRENT per-database settings (latest snapshot) — recovery model, RCSI, Query Store, ... | `server_name`, `database_name` | - | `get_trace_flags` | CURRENT active trace flags (latest snapshot) — flag number, enabled, global/session | `server_name` | - | `get_table_index_sizes` | Largest tables with size + growth (7d/30d/daily) from the latest daily snapshot | `server_name` | + | `get_server_config` | CURRENT sys.configurations (latest snapshot) — what CTFP / MAXDOP / max memory are set to as of `captured_at` (captured on connect — can be days old) | `server_name` | + | `get_database_config` | CURRENT per-database settings (latest snapshot) — recovery model, RCSI, Query Store, ... as of `captured_at` (captured on connect) | `server_name`, `database_name` | + | `get_trace_flags` | CURRENT active trace flags (latest snapshot) — flag number, enabled, global/session, as of `captured_at` (captured on connect) | `server_name` | + | `get_table_index_sizes` | The 100 largest tables with size + growth from the latest daily snapshot. Growth spans only history the store holds: `history` says how many days exist and whether the 7d/30d baselines are reachable; `growth_7d_mb` / `growth_30d_mb` / `growth_pct_30d` are null (reason in `growth_note`) when their baseline does not exist — never re-measured over a shorter span under the same name — and `growth_over_available_history_*` spans exactly `growth_window_days` | `server_name` | | `get_index_usage` | Per-index usage classified Unused / Write-only / Active | `server_name` | | `get_object_locking` | Per-index lock/latch contention, most contended first | `server_name` | | `get_database_sizes` | Per-file database sizes, space usage, and volume free space | `server_name` | ### Resource-contention + jobs data-read tools - Deeper reads for an internal-contention / worker-thread / memory-grant / plan-cache / SQL Agent investigation. Same names + parameters Lite and the Dashboard expose. The Dashboard's per-class latch `severity` / `description` / `recommendation`, spinlock `description`, plan-cache `bloat_level`, and CPU-scheduler `pressure_level` / `recommendation` are the Dashboard / reporting-view CASE derivations (not collected columns) — reproduced here so the full result shape is served. Per-second latch/spinlock rates are derived from the collection interval (Darling's delta collectors store no `sample_interval_seconds`). + Deeper reads for an internal-contention / worker-thread / memory-grant / plan-cache / SQL Agent investigation. Same names + parameters Lite and the Dashboard expose. The Dashboard's per-class latch `severity` / `description` / `recommendation`, spinlock `description`, plan-cache `bloat_level`, and CPU-scheduler `pressure_level` / `recommendation` are the Dashboard / reporting-view CASE derivations (not collected columns) — reproduced here so the full result shape is served. Per-second latch/spinlock rates divide by the row's stored `sample_interval_seconds` (the measured seconds the deltas accrued over); they are `null`, never 0, when that interval was unknowable — the collector's first sighting, a counter reset, or a gap past the delta policy, typically a restart. | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `get_latch_stats` | Top latch classes by wait time, with per-second rates + severity / description / recommendation | `server_name`, `hours_back` (default 24), `top` (default 10), `as_of` | + | `get_latch_stats` | Top latch classes by wait time (window totals), with per-second rates + severity / description / recommendation; `severity` is banded from the LATEST interval only and `severity_banded_from` names that interval (delta, seconds, `captured_at`) | `server_name`, `hours_back` (default 24), `top` (default 10), `as_of` | | `get_spinlock_stats` | Top spinlocks by collisions, with per-second rates + description | `server_name`, `hours_back` (default 24), `top` (default 10), `as_of` | - | `get_resource_semaphore` | Latest workspace-memory semaphores: target / max-target ceiling vs granted / used, waiter / timeout / forced | `server_name`, `hours_back` (default 24), `as_of` | - | `get_memory_grants` | Latest per-pool grant detail: available / granted / used + waiter / timeout / forced deltas | `server_name`, `hours_back` (default 1), `as_of` | + | `get_resource_semaphore` | Per-semaphore workspace memory vs target/max ceiling: `grants[]` = newest snapshot in the window (`captured_at` / `age_seconds`) AND `window[]` = peak waiters (+ when), peak grant, available floor, summed timeout / forced deltas over EVERY snapshot in the window, per (semaphore, pool) | `server_name`, `hours_back` (default 24), `as_of` | + | `get_memory_grants` | Per-pool grant pressure: `grants[]` = newest snapshot in the window (`captured_at` / `age_seconds`) AND `window[]` = the same peak / floor / summed-delta aggregate per pool | `server_name`, `hours_back` (default 1), `as_of` | | `get_memory_pressure_events` | RING_BUFFER_RESOURCE_MONITOR memory-pressure notifications (process/system indicator scale 0-3+); not on Azure SQL DB | `server_name`, `hours_back` (default 24), `as_of` | - | `get_plan_cache_bloat` | Plan cache single-use vs multi-use composition + bloat_level classification | `server_name`, `hours_back` (default 24), `as_of` | - | `get_cpu_scheduler_pressure` | Latest scheduler snapshot: runnable queue, worker utilization, pressure_level + warnings | `server_name` | + | `get_plan_cache_bloat` | Plan cache single-use vs multi-use composition + bloat_level classification; `captured_at` / `age_seconds` | `server_name`, `hours_back` (default 24; the span SEARCHED for the newest snapshot), `as_of` | + | `get_cpu_scheduler_pressure` | Latest scheduler snapshot within `hours_back` of `as_of`: runnable queue, worker utilization, pressure_level + warnings; `captured_at` / `age_seconds` | `server_name`, `hours_back` (default 24; the span SEARCHED for the newest snapshot), `as_of` | | `get_running_jobs` | Currently running SQL Agent jobs with duration vs historical average / p95 | `server_name` | ### Trend data-read tools @@ -187,14 +192,16 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | `get_perfmon_trend` | A single performance counter's value + delta over time (summed across instances) | `counter_name` (required), `server_name`, `hours_back` (default 24), `as_of` | | `get_file_io_trend` | Per-database file I/O read/write latency over time (top-10 busiest files). An empty result distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) | `server_name`, `hours_back` (default 24), `as_of` | | `get_query_trend` | One query's per-collection history (deltas, avg cpu/elapsed, DOP) by query_hash | `query_hash` (required), `database_name` (required), `server_name`, `hours_back` (default 24), `as_of` | - | `get_query_duration_trend` | Overall query elapsed-ms/sec + executions/sec across all queries over time, from the PLAN CACHE. Each point carries `value` (ms/sec), `execution_count` and `executions_per_second` — the last two are the same quantity, and `execution_count` is truncated to an integer, so read `executions_per_second` on a quiet server where the rate is below 1. An empty result distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) | `server_name`, `hours_back` (default 24), `as_of` | - | `get_procedure_duration_trend` | The same series over `procedure_stats`. NOT a duplicate of the above: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs — this charges the whole call to the procedure. Read the two together to tell an ad-hoc regression from a procedure regression | `server_name`, `hours_back` (default 24), `as_of` | - | `get_query_store_duration_trend` | The same series over Query Store. The plan-cache trends lose everything an eviction or a restart takes with them; Query Store persists per interval, so this is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work RAN. Its `unavailable` names the cause the other two do not have: Query Store may simply be OFF on every database | `server_name`, `hours_back` (default 24), `as_of` | + | `get_query_duration_trend` | Overall query elapsed-ms/sec + executions/sec across all queries over time, from the PLAN CACHE. Each point carries `value` (ms/sec), `execution_count` and `executions_per_second` — the last two are the same quantity, and `execution_count` is truncated to an integer, so read `executions_per_second` on a quiet server where the rate is below 1. Each point also carries `elapsed_ms_per_second`, the same quantity as `value` with its unit in the name — read that one. Raw `query_stats` rows are dropped at 4 days on a TimescaleDB store, so a window reaching past that is served from the hourly rollup: `source` says which tier answered (`raw` or `hourly`), `bucket` its grain (`per-collection` or `1 hour`), `effective_start` / `effective_hours_back` where the served series actually begins, `truncated` whether that head sits later than asked, and `aggregate_note` what the hourly tier trades (each point is the hour's work over 3,600 s, so a partly-collected hour reads low; the rollup trails the clock by up to two hours). An empty result carries the same block and distinguishes a quiet window (`empty`, widen `hours_back`) from a server nothing has ever been collected for (`unavailable`, collection is not running) — and from a window whose head the tier does not hold (`empty`, but the message says the rows were dropped or not yet materialized and that widening cannot help; `--backfill-rollups` is the remedy) On the raw route each point is a rate over the gap since the PREVIOUS collection, so the window's first collection carries null rates (`unrated_points` / `unrated_note`): unknowable, never reported as 0 (the hourly route divides by the bucket width and has no such point). | `server_name`, `hours_back` (default 24), `as_of` | + | `get_procedure_duration_trend` | The same series over `procedure_stats`. NOT a duplicate of the above: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs — this charges the whole call to the procedure. Read the two together to tell an ad-hoc regression from a procedure regression. Same tier routing and the same `source` / `effective_start` / `truncated` / `bucket` / `aggregate_note` block as `get_query_duration_trend`, over `procedure_stats` / `procedure_stats_hourly` | `server_name`, `hours_back` (default 24), `as_of` | + | `get_query_store_duration_trend` | The same series over Query Store. The plan-cache trends lose everything an eviction or a restart takes with them; Query Store persists per interval, so this is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work RAN. Its `unavailable` names the cause the other two do not have: Query Store may simply be OFF on every database. Carries the same `source` / `effective_start` / `truncated` / `bucket` / `aggregate_note` block as its siblings; where the store has the corrected Query Store rollup, `source` is `rollup+raw` and a `routing` block names the rollup, `raw_from` (the seam: 1-hour buckets before it, per-interval points from it) and, when the window reaches below what the rollup has materialized, `unserved_before` — points before that instant are missing, not zero | `server_name`, `hours_back` (default 24), `as_of` | ### System-health parse-on-read tools The `get_health_parser_*` family the Dashboard exposes, over Darling's raw `system_health_events`. Where the Dashboard reads its server-side-parsed `collect.HealthParser_*` tables, these shred the raw extended-event XML ON READ with the shared SystemHealthParser and return the same SIGNIFICANT warning set (sp_HealthParser at `@warnings_only = 1`) — the exception is `get_health_parser_system_health`, whose corruption/contention counter series is UNGATED (every snapshot). Each returns the full sp_HealthParser column set per row keyed on the event's `event_time`; the tools window on `event_time` (the event's real time), so "last 24 hours" means events that happened in the last 24 hours. + Every one of the nine carries a SOURCE WITNESS: `source_observed` (whether this server's system_health session has EVER been read into the store — any event of any type) and `last_captured_at` (the collector's newest capture). Zero rows is four different answers and the read says which: captured in the window and gated out (`empty`, `events_in_window` > 0 — the healthy one), captured before but not in this window (`empty`, `last_captured_of_type_at` says when — widen), never captured for THIS category while the session is being read (`empty`, `source_observed` true — for a rare category such as a memory-node OOM this is the measurement, not a gap), and nothing of any type ever (`unavailable`, `source_observed` false — a dead session or a collector that never ran, and NOT an all-clear). Zero is a measurement only when the source was observed. + | Tool | Purpose | Key Parameters | |------|---------|----------------| | `get_health_parser_system_health` | SYSTEM-component snapshots: corruption (bad pages / dumps / access violations) + contention (non-yielding / latch / sick spinlock / CPU) counters | `server_name`, `hours_back` (default 24), `limit` (default 50), `as_of` | @@ -213,12 +220,12 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `get_alert_history` | Alerts that fired (metric, value vs threshold, delivery success/failure, muted); omit server_name for the whole fleet, each row names its server | `server_name` (optional — all servers if omitted), `hours_back` (default 24), `limit` (default 50), `as_of` | + | `get_alert_history` | Alerts that fired (metric, value vs threshold, delivery success/failure, muted), newest first; omit server_name for the whole fleet, each row names its server. EXCLUDES operator-dismissed alerts by default and says so (`dismissed_excluded`, `dismissed_excluded_count`); pass `include_dismissed` when reconstructing an incident. Page bounded by `limit`; `truncated` and `oldest_returned_alert_time` say what it reached | `server_name` (optional — all servers if omitted), `hours_back` (default 24), `limit` (default 50), `as_of`, `include_dismissed` (default false) | | `get_alert_settings` | The current alert config the service uses: per-alert enable + thresholds, cooldown, excluded databases, delivery mode, and the scheduled-analysis cadence | none | | `get_mute_rules` | The alert mute rules in force, so a suppressed server is distinguishable from a healthy-quiet one. An empty result distinguishes no rule ever written from rules that exist but have all lapsed, with the configured count in `hints` | `enabled_only` (default true) | - | `get_server_summary` | One-shot per-server health: current CPU %, memory, recent blocking count, recent deadlock count | `server_name` | - | `get_daily_summary` | A day's composite health band (Healthy / Warning / Critical) plus the signals behind it (waits, deadlocks, blocking, high CPU, memory pressure, alerts) | `server_name`, `summary_date` (yyyy-MM-dd, default today) | - | `get_daily_summary_range` | The SAME rollup across a span of days — one row per collected day, which is the desktop viewer's Performance Calendar month grid. Use it when the question is WHICH day rather than how one day went: scan the bands, then call `get_daily_summary` for the day that stands out. A day with ANY collection appears even when every signal was quiet (Healthy, not missing), so a day absent from the result is a gap in COLLECTION. `as_of` anchors the LAST day of the range, so a past month is `as_of` its last day with `days_back` its length. An empty result distinguishes a range outside this server's history (`empty`) from a server nothing has ever been collected for (`unavailable`) | `server_name`, `days_back` (default 30, max 366), `as_of` | + | `get_server_summary` | One-shot per-server health: current CPU %, memory, recent blocking count, recent deadlock count; three clocks named (`cpu_captured_at`, `memory_captured_at`, `last_collection` = newest collection of ANY collector) | `server_name` | + | `get_daily_summary` | A day's composite health band (Healthy / Warning / Critical) plus the signals behind it (waits, deadlocks, blocking, high CPU, memory pressure, alerts). A day before the store's `retention_horizon` is `unavailable` with `data_state: purged` — never a band, because its zeros are absences | `server_name`, `summary_date` (yyyy-MM-dd ONLY, refused otherwise; default today) | + | `get_daily_summary_range` | The SAME rollup across a span of days — one row per collected day, which is the desktop viewer's Performance Calendar month grid. Use it when the question is WHICH day rather than how one day went: scan the bands, then call `get_daily_summary` for the day that stands out. A day with ANY collection appears even when every signal was quiet (Healthy, not missing), so a day absent from the result is a gap in COLLECTION — inside retention. The payload publishes `retention_horizon` (the oldest day every signal table still holds) and `days_before_horizon`; a returned day before the horizon is `data_state: purged` (no signal table holds it) or `past_horizon` (some still do) and `NoData`, NEVER Healthy, because a purged day's per-signal zeros are absences left by the purge while the longer-lived collection log still names the day. Purged and past-horizon rows carry no verdict; `no_run_record` (inside retention, no collector run recorded) keeps its band with a caveat. `as_of` anchors the LAST day of the range, so a past month is `as_of` its last day with `days_back` its length. An empty result distinguishes a range outside this server's history (`empty`) from a server nothing has ever been collected for (`unavailable`) | `server_name`, `days_back` (default 30, max 366), `as_of` | **Tuning the alerting (write).** Five Darling-only tools change the shared alert configuration the service delivers on — the SAME config `get_alert_settings` / `get_mute_rules` read, and the same the Viewer's Settings window writes. They are the only alert writes here; none touches a monitored server or the collected data, and a change hot-reloads into the running service within one collection sweep. @@ -250,11 +257,11 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) ### Store self-metrics - `get_store_metrics` reads the MONITORING STORE's own growth series — not a monitored SQL Server's. The service records an hourly self-metrics snapshot into the store: per-hypertable total size, pre/post-compression bytes and chunk count; the query-text / query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; and the whole store's size with the enabled-server count. The tool returns the latest snapshot per object plus a daily series, including the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth ÷ enabled servers) — the number to multiply when onboarding N servers. Each daily point is that day's LAST snapshot rather than its maximum, so a maximum question needs the per-run route the tool's own description names. Use it for capacity forecasting: what is driving store growth and how fast. 400 days of history; no `server_name` parameter, because the store is the subject. + `get_store_metrics` reads the MONITORING STORE's own growth series — not a monitored SQL Server's. The service records an hourly self-metrics snapshot into the store: per-hypertable total size, pre/post-compression bytes and chunk count; the same three for every continuous aggregate, sized through its materialization and named by its view (`object_kind` `continuous_aggregate` — on one production store the twenty aggregates were 57% of the database and the inventory used to show none of them); the query-text / query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; the product's named plain tables (`table`: `collect.query_store_text`, `collect.query_store_plan_map`, `config.config_alert_log`); two catch-all rows, `other` (every user-schema relation no named row accounts for, with its relation count) and `system` (the PostgreSQL catalogs and TimescaleDB's bookkeeping); and the whole store's size with the enabled-server count. The response's `inventory` block RECONCILES the newest sweep against its own `pg_database_size` and states coverage in words: `enumerated_percent` is the share under named objects, `attributed_percent` the share any row explains, `residual_bytes` what none does, and `reconciled` is false — a finding — when that residual exceeds the larger of 1% and 64 MiB; `bytes_by_kind` answers "what is driving growth" in one glance and `largest_unenumerated` names, live, the biggest relations inside `other`. Each `continuous_aggregate` object also carries `compression_enabled` and its refresh / compression / retention policy job ids, read live from the catalog. The tool returns the latest snapshot per object plus a daily series, including the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth ÷ enabled servers) — the number to multiply when onboarding N servers. Each daily point is that day's LAST snapshot rather than its maximum, so a maximum question needs the per-run route the tool's own description names — `timescaledb_information.job_history`, which records successful runs only while its GUC is on (failed runs are written regardless) AND shows rows only to members of the job's owner role or the database owner (the `jobs` view is not filtered, which is the trap); the response's `job_history` block reports the GUC's state, which role it read as, whether the view lets that role see anything, and the rows it observed over the last 24 hours — and, because in managed mode that role is the least-privilege `mcp` role the view shows nothing, the OWNER's count as well: the service's hourly sweep runs as the owner and persists what it saw over the same window, reported as `owner_rows_observed` / `owner_newest_row_at` / `owner_observed_at` with the note saying the number is the sweep's, not this connection's. Read it before treating an empty history as an answer. Use it for capacity forecasting: what is driving store growth and how fast, and how much of the store the inventory can actually see. 400 days of history; no `server_name` parameter, because the store is the subject. | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `get_store_metrics` | Latest size/compression snapshot per store object (hypertables, payload dimensions, the whole store) plus a daily growth series with the per-server ingest rate | `days_back` (default 30, max 400) | + | `get_store_metrics` | Latest size/compression snapshot per store object (hypertables, continuous aggregates, payload dimensions, named tables, the two catch-all rows, the whole store) reconciled against `pg_database_size` with a stated coverage, plus a daily growth series with the per-server ingest rate | `days_back` (default 30, max 400) | | `get_store_log` | Per-class census of the monitoring store's OWN server log, with the capture denominator, the per-hour median beside each class's count, the retained-message detail, and the classes this build has that the window did NOT see | `hours_back` (default 24, max 168), `limit` (default 50), `as_of` | | `get_collector_stall_probes` | The out-of-band server-wide wait samples taken while one of OUR collectors was stalled mid-read: the client-side trigger evidence beside the instance's whole wait list and scheduler load, with the outcome census (including the probes that could not connect) always beside them. Unbanded. | `server_name` (optional; omit for the fleet), `days_back` (default 7, max 60), `limit` (default 50) | | `get_oversized_plan_backlog` | The cached plans measured as too large to capture inline, and what the out-of-band sweep has done about each: per server the three verdict buckets (pending/captured/expired, a strict partition of the total), the attempt figures on still-pending rows, the newest capture and expiry instants, and `observed_bytes` min/median/max, with the per-collector census beside them. `last_captured_at` is how a working fetch half is told from one that has never succeeded. No time window — a worklist updated in place, not a series. | `server_name` (optional; omit for the fleet), `include_rows` (default false; needs `server_name`), `limit` (default 50) | @@ -284,7 +291,7 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | `get_custom_view` | Gets one view in full, including its definition JSON and current version | `view_id` (required) | | `validate_custom_view` | Dry-run: validates a definition against the catalog + composer rules WITHOUT saving; returns `{valid, error}` | `definition` (required) | | `create_custom_view` | Validates then saves a new view (returns the stored view at version 1); conflict on a duplicate name | `name` (required), `definition` (required), `description` | - | `update_custom_view` | Validates then updates a view in place under optimistic concurrency (pass the `version` you read); conflict on a stale version or duplicate name | `view_id` (required), `name` (required), `definition` (required), `version` (required), `description` | + | `update_custom_view` | Validates then updates a view in place under optimistic concurrency (pass the `version` you read); conflict on a stale version or duplicate name. `description` follows the shared write vocabulary: omitted = unchanged, empty string = cleared | `view_id` (required), `name` (required), `definition` (required), `version` (required), `description` | | `delete_custom_view` | Deletes a view by id (permanent) | `view_id` (required) | | `run_custom_view_panel` | Compiles + runs a single composed panel and returns `{sql, rows, annotations}` — the composer's live preview, for checking a panel's data before saving | `spec` (required — a JSON object `{panel, variables?, values?, server?, hours?}`) | @@ -294,8 +301,8 @@ public static string Build(DarlingPeerDirectory.Snapshot peers) | Tool | Purpose | Key Parameters | |------|---------|----------------| - | `add_servers` | BULK-adds monitored servers: pass a JSON ARRAY of server objects and each is validated, connection-tested IN the service, and (if new and reachable) saved. Per object: `host` (required), `display_name`, `database` (one DB only, e.g. an Azure SQL Database), `auth` (`Windows`/`SQL`, default `Windows`), `username`+`password` (required for `SQL`), `encrypt_mode` (`Optional`/`Mandatory`/`Strict`, default `Mandatory`), `trust_server_certificate` (default false), `read_only_intent`, `multi_subnet_failover`. Servers are processed in order; a duplicate (case-folded, vs existing or an earlier entry) is `duplicate`, an unreachable server is `connection_failed` (the batch continues), Entra/MFA/Service-Principal/Managed-Identity auth is `invalid` (Windows/SQL only). Returns `{added, skipped, failed, results:[{server, status, detail}]}` | `servers_json` (required — a JSON array of server objects) | - | `remove_server` | Removes a monitored server by name (resolved like every `server_name`). Returns `{status:"removed", server}` or `{status:"not_found"}`. Already-collected history is NOT deleted | `server_name` (required) | + | `add_servers` | BULK-adds monitored servers: pass a JSON ARRAY of server objects and each is validated, connection-tested IN the service, and (if new and reachable) saved. Per object: `host` (required), `display_name`, `database` (one DB only, e.g. an Azure SQL Database), `engine` (`sqlserver` default, or `postgres`), `auth` (`Windows`/`SQL`, or the two NON-INTERACTIVE Microsoft Entra modes `ServicePrincipal` / `ManagedIdentity` — default `Windows`; a PostgreSQL target requires `SQL`), `username`+`password` (required for `SQL`; for `ServicePrincipal` they are the Entra application/client id + client secret; for `ManagedIdentity` `username` is an optional user-assigned identity's client id and there is no password), `port` (PostgreSQL only), `encrypt_mode` (`Optional`/`Mandatory`/`Strict`, default `Mandatory`), `trust_server_certificate` (default false), `read_only_intent`, `multi_subnet_failover`. Servers are processed in order; a duplicate (case-folded, vs existing or an earlier entry) is `duplicate`, an unreachable server is `connection_failed` (the batch continues), a server whose probed connection lands in a database another registration already covers is `collides` (not added), and the INTERACTIVE Entra modes (MFA / device-code / default-credential) are `invalid` — they cannot run headless. Returns `{requested, added, skipped, collided, failed, results:[{server, status, detail}]}`; the four counters SUM to `requested` (`added`→added, `duplicate`→skipped, `collides`→collided, `connection_failed`/`invalid`→failed), and only `added` servers are monitored | `servers_json` (required — a JSON array of server objects) | + | `remove_server` | Removes a monitored server by name — exact match on the storage or display name, or a partial match ONLY when it is unique. An ambiguous name (a fragment several servers contain, or a display name two registrations share) deletes NOTHING and returns `{status:"ambiguous", candidates:[{server, display_name}]}`; re-issue with one candidate's full name. Returns `{status:"removed", server, matched_by}`, `{status:"ambiguous", ...}` or `{status:"not_found"}`. Already-collected history is NOT deleted | `server_name` (required) | A SQL password is encrypted at rest (DPAPI, the service identity) and is NEVER returned by a read tool. It DOES travel to this endpoint inside `add_servers`' request JSON, so on a LAN deployment reach the endpoint only through the documented TLS reverse proxy. @@ -309,7 +316,10 @@ carried onto downstream tickets. Pass it to get exactly that incident instead of identifier. Two things to know when it matches nothing: the fingerprint is scoped to the server's DISPLAY name and to the incident's involved objects, so a server renamed since the alert fired has different keys now; and `hours_back` still bounds the search, so widen it before concluding the incident is gone. The no-match response - says how many rows it examined, which distinguishes those cases. + says how many rows it examined, which distinguishes those cases. The fingerprint scan runs over the window + BEFORE `limit` (so the cap can never discard the incident the key names), up to a stated ceiling: a match + payload carries `rows_examined` and `scan_truncated`, and a no-match answer on a window the scan could not + finish says so and asks for `as_of` anchored at the alert time with a narrow `hours_back`. Note on `next_tools`: analyze_server findings include `next_tools` recommendations. Most are hosted on this server — the plan-analysis tools (`analyze_query_plan`, `analyze_query_store_plan`) and the data-read tools listed above (`get_wait_stats`, `get_top_queries_by_cpu`, `get_cpu_utilization`, `get_memory_stats`, `get_file_io_stats`, `get_tempdb_trend`, `get_blocking`, `get_deadlocks`, `get_waiting_tasks`, `get_active_queries`, ...) — so follow those here. `get_top_queries_by_cpu` / `get_top_procedures_by_cpu` / `get_query_store_top` are where the `query_hash` / `sql_handle` / `query_id` + `plan_id` keys for the plan-analysis tools come from. The resource-contention + jobs tools (`get_latch_stats`, `get_spinlock_stats`, `get_resource_semaphore`, `get_memory_grants`, `get_plan_cache_bloat`, `get_cpu_scheduler_pressure`, `get_running_jobs`), the trend siblings (`get_memory_trend`, `get_perfmon_trend`, `get_file_io_trend`, `get_query_trend`, `get_query_duration_trend`), the `get_health_parser_*` system-health family, and the blocking/deadlock trend + memory-pressure reads (`get_blocking_trend`, `get_deadlock_trend`, `get_memory_pressure_events`) are all hosted here too — follow those `next_tools` on this server. Two `next_tools` names differ from what this edition hosts: `get_blocked_process_reports` (a Lite name) is served here as `get_blocked_process_xml` (with `get_blocking` for a quick overview), and `get_blocking_deadlock_stats` (the Dashboard's blocking/deadlock rollup) is not hosted at all — use `get_blocking` / `get_deadlocks` instead. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLatchSpinlockTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLatchSpinlockTools.cs index 843650aa1..f0a3a55cb 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLatchSpinlockTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLatchSpinlockTools.cs @@ -22,8 +22,9 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// The latch / spinlock contention MCP tools — get_latch_stats, get_spinlock_stats — served over Darling's /// Postgres store. Each tool body mirrors the Dashboard's McpLatchSpinlockTools field-for-field (Lite -/// exposes neither tool, so the Dashboard is the only reference; there is no divergent Lite shape to follow). -/// Reads flow through — STORED reads (no live monitored-server hit), +/// has since ported both names as LATEST-SNAPSHOT reads over its own store — McpLatchSpinlockTools in +/// Lite/Mcp — so the two SKUs share the names but not the shape: Darling aggregates the window, Lite +/// serves the newest snapshot in it). Reads flow through — STORED reads (no live monitored-server hit), /// windowed on hours_back. /// /// @@ -37,7 +38,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpLatchSpinlockTools { - [McpServerTool(Name = "get_latch_stats"), Description("Gets top latch contention by class. Shows latch waits, wait time, and per-second rates. High LATCH_EX on ACCESS_METHODS_DATASET_PARENT or FGCB_ADD_REMOVE indicates TempDB allocation contention.")] + [McpServerTool(Name = "get_latch_stats"), Description("Gets top latch contention by class. Shows latch waits, wait time, and per-second rates. High LATCH_EX on ACCESS_METHODS_DATASET_PARENT or FGCB_ADD_REMOVE indicates TempDB allocation contention. TWO CLOCKS PER ROW, NAMED: total_delta_* SUM every collection in the window; severity, waits_per_second and wait_ms_per_second are banded/derived from the LATEST interval only - the severity_banded_from block names that interval (its delta wait, the seconds it accrued over, and the collection it ended at, which is also latest_collection_time). A LOW severity beside a large window total is a class that was hot earlier in the window and is quiet now, not a contradiction.")] public static async Task GetLatchStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -70,9 +71,19 @@ public static async Task GetLatchStats( avg_wait_ms_per_request = r.TotalDeltaWaitingRequests > 0 ? Math.Round((double)r.TotalDeltaWaitTimeMs / r.TotalDeltaWaitingRequests, 2) : (double?)null, - waits_per_second = Math.Round(r.WaitsPerSecond, 2), - wait_ms_per_second = Math.Round(r.WaitMsPerSecond, 2), + /* null when the latest interval was unknowable (#3540) — a restart, not a quiet latch. */ + waits_per_second = r.WaitsPerSecond is double waits ? Math.Round(waits, 2) : (double?)null, + wait_ms_per_second = r.WaitMsPerSecond is double waitMs ? Math.Round(waitMs, 2) : (double?)null, severity = DarlingLatchSpinlockReader.LatchSeverity(r.LatestDeltaWaitTimeMs), + /* #3541 A10: the band above is a function of ONE interval's delta, published beside window + totals it is not a function of. Naming the interval — its delta, its length, its end — is + what lets a reader tell "LOW now, 40 s of waits over the day" from "LOW all day". */ + severity_banded_from = new + { + delta_wait_time_ms = r.LatestDeltaWaitTimeMs, + interval_seconds = r.LatestIntervalSeconds is double seconds ? Math.Round(seconds, 0) : (double?)null, + captured_at = r.LatestCollectionTime.ToString("o") + }, description = DarlingLatchSpinlockReader.LatchDescription(r.LatchClass), recommendation = DarlingLatchSpinlockReader.LatchRecommendation(r.LatchClass), latest_collection_time = r.LatestCollectionTime.ToString("o") @@ -126,8 +137,9 @@ public static async Task GetSpinlockStats( spins_per_collision = r.TotalDeltaCollisions > 0 ? Math.Round((double)r.TotalDeltaSpins / r.TotalDeltaCollisions, 1) : (double?)null, - collisions_per_second = Math.Round(r.CollisionsPerSecond, 2), - spins_per_second = Math.Round(r.SpinsPerSecond, 2), + /* null when the latest interval was unknowable (#3540) — a restart, not a quiet spinlock. */ + collisions_per_second = r.CollisionsPerSecond is double collisions ? Math.Round(collisions, 2) : (double?)null, + spins_per_second = r.SpinsPerSecond is double spins ? Math.Round(spins, 2) : (double?)null, description = DarlingLatchSpinlockReader.SpinlockDescription(r.SpinlockName), latest_collection_time = r.LatestCollectionTime.ToString("o") }); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLongQueryTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLongQueryTools.cs index 709456b31..541fb44aa 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLongQueryTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpLongQueryTools.cs @@ -29,12 +29,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpLongQueryTools { - [McpServerTool(Name = "get_long_query_completions"), Description("Gets long-running query completions captured by the opt-in long-query trace: rpc/batch completions whose duration exceeded the trace threshold, plus attentions (client cancels / query timeouts). Shows duration, CPU, reads/writes, row count, result (OK/Error/Abort — Abort means the long query was cancelled), the statement text, and the calling session/app/login. The collector is OFF by default; if it returns empty, enable the 'long_query_completions' collector in the schedule.")] + [McpServerTool(Name = "get_long_query_completions"), Description("Gets the SLOWEST long-running query completions in the window, captured by the opt-in long-query trace: rpc/batch completions whose duration exceeded the trace threshold, plus attentions (client cancels / query timeouts), ranked by duration DESC with attentions (no duration) last. Shows duration, CPU, reads/writes, row count, result (OK/Error/Abort — Abort means the long query was cancelled), the statement text, and the calling session/app/login. THE PAGE IS THE window's limit SLOWEST, NOT ITS NEWEST: completions_returned is how many rows you got and truncated says the window held more than limit. Because the page is duration-RANKED, oldest_returned_event_time / newest_returned_event_time tell you how old the slowest runs are and say NOTHING about how far back the read reached — every row in the window was a candidate, so a truncated page still holds the window's slowest. Both SKUs keep the same population. The collector is OFF by default; if it returns empty, enable the 'long_query_completions' collector in the schedule.")] public static async Task GetLongQueryCompletions( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum rows. Default 30.")] int limit = 30, + [Description("Maximum rows to return, slowest first. Default 30. This is what bounds the page — read truncated to know whether the window held more.")] int limit = 30, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -48,8 +48,12 @@ public static async Task GetLongQueryCompletions( try { var now = windowEnd; + + /* #3541 A3: the caller's limit + 1 as the fetch, the extra row as the observed truncation signal. + The reader's own LIMIT 200 was invisible to the caller, and `total_completions` published it as + the window's count. */ var rows = await DarlingLongQueryReader.GetRecentLongQueryCompletionsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "long_query_completions") /* #2546: this collector is opt-in, so the fall-through below already sends the reader to @@ -59,7 +63,10 @@ that is already switched on. */ ?? await DarlingRuntimePrecondition.StatusAsync(postgres, resolved.ServerId, resolved.ServerName, "long_query_completions") ?? McpHelpers.Status("empty", "No long-running query completions found in the specified time range. The long_query_completions collector is opt-in (default OFF) — enable it in the collector schedule to capture data."); - var result = rows.Take(limit).Select(r => new + var truncated = rows.Count > limit; + var page = truncated ? rows.Take(limit).ToList() : rows; + + var result = page.Select(r => new { event_time = r.EventTime?.ToString("o"), event_type = r.EventType, @@ -85,7 +92,14 @@ that is already switched on. */ { server = resolved.ServerName, hours_back, - total_completions = rows.Count, + /* #3541 A3: the page described as a page. Under a duration RANKING the two stamps bound the + slowest runs, not the reach — the description says so, and QueryStoreTopWindowTests states + the general trap for cost-ranked pages. */ + completions_returned = page.Count, + truncated, + oldest_returned_event_time = page.Min(r => r.EventTime)?.ToString("o"), + newest_returned_event_time = page.Max(r => r.EventTime)?.ToString("o"), + order = "duration_ms_desc", completions = result }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs index f40694756..6c7d1ae0e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpMemoryGrantTools.cs @@ -21,20 +21,30 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// The memory-grant MCP tools — get_resource_semaphore, get_memory_grants — served over Darling's Postgres -/// store, both reading the LATEST memory_grant_stats snapshot through -/// (STORED reads, no live monitored-server hit). The two names are two -/// lenses on the one collector table, so Darling hosts BOTH: get_resource_semaphore is the Dashboard's -/// semaphore/ceiling shape (per resource semaphore, with the workspace-memory target/max-target ceiling); -/// get_memory_grants is Lite's per-pool grant-detail shape. A client familiar with either SKU finds its tool. +/// store through (STORED reads, no live monitored-server hit). The two +/// names are two lenses on the one collector table, so Darling hosts BOTH: get_resource_semaphore is the +/// Dashboard's semaphore/ceiling shape (per resource semaphore, with the workspace-memory target/max-target +/// ceiling); get_memory_grants is Lite's per-pool grant-detail shape. A client familiar with either SKU finds +/// its tool. +/// +/// #3541 A10 — the window is read, and the snapshot says when it was taken. Each tool serves two +/// things under one hours_back / as_of window: grants, the NEWEST snapshot in the window +/// (stamped once as captured_at, with age_seconds against the window's end), and window, +/// the per-semaphore / per-pool aggregate over EVERY snapshot in it — peak waiters and when, peak grant, the +/// available-workspace floor, and the summed timeout / forced-grant deltas. Before this the tools accepted +/// hours_back and read only the latest row, so a three-hour-old grant storm was invisible behind a +/// calm snapshot while the parameter read as a window. Dropping the parameter was the other honest shape; +/// reading the window was chosen because "was there grant pressure in the last N hours" is the question an +/// agent brings to this tool, and the store answers it in one bounded scan. /// [McpServerToolType] public sealed class DarlingMcpMemoryGrantTools { - [McpServerTool(Name = "get_resource_semaphore"), Description("Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance.")] + [McpServerTool(Name = "get_resource_semaphore"), Description("Gets resource semaphore statistics showing granted vs available workspace memory against the target/max-target ceiling, waiter counts, and timeout/forced grant pressure indicators. High waiter counts or rising timeout/forced deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per resource semaphore and pool), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per (resource_semaphore_id, pool_id): peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting for a grant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). A calm grants[] beside a window[] with waiters or timeouts is a grant storm that has passed; read window[] first for 'was there pressure', grants[] for 'is there pressure now'. Each grants[] row also carries sample_interval_seconds, the measured seconds its two deltas accrued over; it is null with interval_known false when the row is a restart marker (no delta was knowable, so the zero deltas beside it are not 'no timeouts') or predates the column.")] public static async Task GetResourceSemaphore( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Hours of history. Default 24.")] int hours_back = 24, + [Description("Hours of history. Default 24. window[] aggregates every snapshot in these hours; grants[] is the newest snapshot in them.")] int hours_back = 24, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -46,12 +56,18 @@ public static async Task GetResourceSemaphore( try { var now = windowEnd; + var windowStart = now.AddHours(-hours_back); var rows = await DarlingMemoryGrantReader.GetResourceSemaphoreLatestAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, windowStart, now); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "memory_grant_stats") ?? McpHelpers.Status("unavailable", "No memory grant data available."); + /* Same window bounds as the latest read, so the window's last_snapshot_at IS captured_at and the + two halves describe one span of the same rows (#3541 A10). */ + var window = await DarlingMemoryGrantReader.GetResourceSemaphoreWindowAsync( + postgres, resolved.ServerId, windowStart, now); + var grants = rows.Select(r => new { collection_time = r.CollectionTime.ToString("o"), @@ -68,13 +84,28 @@ public static async Task GetResourceSemaphore( timeout_error_count = r.TimeoutErrorCount, forced_grant_count = r.ForcedGrantCount, timeout_error_count_delta = r.TimeoutErrorCountDelta, - forced_grant_count_delta = r.ForcedGrantCountDelta + forced_grant_count_delta = r.ForcedGrantCountDelta, + /* #3540 (V128): the interval the deltas accrued over, the way the perfmon and file-I/O tools + hand it over. A stored 0 is the calculator's no-delta-knowable marker (a restart, not a + quiet semaphore) and is reported as null rather than 0 — 0 seconds is not a measurement; + a pre-V128 row that never recorded one is null too. interval_known states the one thing + both nulls have in common: the two *_delta zeros beside them are not "none this interval". */ + sample_interval_seconds = r.SampleIntervalSeconds is > 0 ? r.SampleIntervalSeconds : null, + interval_known = r.SampleIntervalSeconds is > 0 }); return JsonSerializer.Serialize(new { server = resolved.ServerName, - grants + hours_back, + window_start = windowStart.ToString("o"), + window_end = now.ToString("o"), + /* Every grants[] row shares this stamp by construction (the read is WHERE collection_time = + MAX(...) in the window); it is published once, as the snapshot's own clock. */ + captured_at = rows[0].CollectionTime.ToString("o"), + age_seconds = LatestSnapshotStamp.AgeSeconds(rows[0].CollectionTime, now), + grants, + window = window.Select(WindowShape) }, McpHelpers.JsonOptions); } catch (Exception ex) @@ -83,11 +114,11 @@ public static async Task GetResourceSemaphore( } } - [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance.")] + [McpServerTool(Name = "get_memory_grants"), Description("Gets resource semaphore statistics showing granted vs available workspace memory per resource pool, waiter counts, and timeout/forced grant deltas. High waiter counts or rising timeout deltas indicate memory grant pressure affecting query performance. TWO READS UNDER ONE WINDOW: grants[] is the NEWEST snapshot in the window (one row per pool, summed across its semaphores), stamped once as captured_at with age_seconds against the window's end - it is a moment, not the window. window[] aggregates EVERY snapshot in the window per pool: peak_waiter_count and peak_waiters_at (the most sessions ever seen waiting on the pool at one instant and when), peak_granted_memory_mb, min_available_memory_mb, and timeout_errors_in_window / forced_grants_in_window (the SUM of the per-interval deltas across the window). A calm grants[] beside a window[] with waiters or timeouts is a grant storm that has passed; read window[] first for 'was there pressure', grants[] for 'is there pressure now'.")] public static async Task GetMemoryGrants( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Hours of history. Default 1.")] int hours_back = 1, + [Description("Hours of history. Default 1. window[] aggregates every snapshot in these hours; grants[] is the newest snapshot in them.")] int hours_back = 1, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -99,12 +130,16 @@ public static async Task GetMemoryGrants( try { var now = windowEnd; + var windowStart = now.AddHours(-hours_back); var rows = await DarlingMemoryGrantReader.GetMemoryGrantsLatestAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, windowStart, now); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "memory_grant_stats") ?? McpHelpers.Status("unavailable", "No memory grant data available."); + var window = await DarlingMemoryGrantReader.GetMemoryGrantsWindowAsync( + postgres, resolved.ServerId, windowStart, now); + var grants = rows.Select(r => new { collection_time = r.CollectionTime.ToString("o"), @@ -121,7 +156,13 @@ public static async Task GetMemoryGrants( return JsonSerializer.Serialize(new { server = resolved.ServerName, - grants + hours_back, + window_start = windowStart.ToString("o"), + window_end = now.ToString("o"), + captured_at = rows[0].CollectionTime.ToString("o"), + age_seconds = LatestSnapshotStamp.AgeSeconds(rows[0].CollectionTime, now), + grants, + window = window.Select(WindowShape) }, McpHelpers.JsonOptions); } catch (Exception ex) @@ -130,6 +171,27 @@ public static async Task GetMemoryGrants( } } + /// + /// The window half's payload shape, shared by both lenses so the same key set describes a semaphore's + /// window and a pool's window (the pool lens carries a null resource_semaphore_id, which + /// writes rather than drops — the key is present on both so a + /// caller can read one shape). + /// + private static object WindowShape(DarlingMemoryGrantReader.MemoryGrantWindowRow w) => new + { + resource_semaphore_id = w.ResourceSemaphoreId, + pool_id = w.PoolId, + snapshots_in_window = w.SnapshotsInWindow, + first_snapshot_at = w.FirstSnapshotAt.ToString("o"), + last_snapshot_at = w.LastSnapshotAt.ToString("o"), + peak_waiter_count = w.PeakWaiterCount, + peak_waiters_at = w.PeakWaitersAt.ToString("o"), + peak_granted_memory_mb = Math.Round(w.PeakGrantedMemoryMb, 2), + min_available_memory_mb = Math.Round(w.MinAvailableMemoryMb, 2), + timeout_errors_in_window = w.TimeoutErrorsInWindow, + forced_grants_in_window = w.ForcedGrantsInWindow + }; + [McpServerTool(Name = "get_memory_pressure_events"), Description(@"Gets memory pressure notifications from the RING_BUFFER_RESOURCE_MONITOR ring buffer (same source as sp_pressuredetector). Returns RESOURCE_MEMPHYSICAL_LOW, RESOURCE_MEMVIRTUAL_LOW, RESOURCE_MEMPHYSICAL_HIGH, and RESOURCE_MEM_STEADY notifications with indicator values. Indicator scale (applies to both memory_indicators_process and memory_indicators_system): diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpObjectStatsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpObjectStatsTools.cs index b8737f097..5a95bc997 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpObjectStatsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpObjectStatsTools.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Text.Json; @@ -35,7 +36,7 @@ public sealed class DarlingMcpObjectStatsTools private const int IndexUsageTop = 200; private const int ObjectLockingTop = 200; - [McpServerTool(Name = "get_table_index_sizes"), Description("Gets the largest tables with per-table size, growth (7d/30d/daily rate), and row counts from the latest daily snapshot. Indexes are rolled up per table. Use to find storage hot-spots and fast-growing tables for capacity planning.")] + [McpServerTool(Name = "get_table_index_sizes"), Description("Gets the 100 largest tables with per-table size, growth (7d/30d/daily rate), and row counts from the latest daily snapshot. Indexes are rolled up per table. Use to find storage hot-spots and fast-growing tables for capacity planning. Growth is measured only over history the store actually holds: the history block says how many days of snapshots exist and whether the 7-day and 30-day baselines are reachable; growth_7d_mb / growth_30d_mb / growth_pct_30d are null (with the reason in growth_note) when their baseline does not exist, never re-labelled from a nearer one, and growth_over_available_history_* always spans exactly growth_window_days. A table absent from a baseline snapshot (created since) reports null growth for that window, not 0. tables_returned and truncated bound the page.")] public static async Task GetTableIndexSizes( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null) @@ -46,13 +47,22 @@ public static async Task GetTableIndexSizes( try { var now = DateTime.UtcNow; + /* Over-fetch by one so truncation is observed, not inferred from a full page (#3541 A3's rule). */ var rows = await DarlingObjectStatsReader.GetObjectSizeGrowthAsync( - postgres, resolved.ServerId, now.AddDays(-7), now.AddDays(-30), TableSizesTop); + postgres, resolved.ServerId, now.AddDays(-7), now.AddDays(-30), TableSizesTop + 1); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "index_object_stats") ?? McpHelpers.Status("unavailable", "No object size data available. Index/object stats are collected daily."); - var result = rows.Select(r => new + var truncated = rows.Count > TableSizesTop; + var page = rows.Take(TableSizesTop).ToList(); + + /* The store's span is one fact for every row (the boundaries CTE), so it is published once. */ + var span = page[0]; + var covers7d = span.Snapshot7dTime is not null; + var covers30d = span.Snapshot30dTime is not null; + + var result = page.Select(r => new { database_name = r.DatabaseName, schema_name = r.SchemaName, @@ -61,15 +71,39 @@ public static async Task GetTableIndexSizes( used_mb = r.CurrentUsedMb, total_rows = r.TotalRows, index_count = r.IndexCount, + /* Each nominal-window figure comes from exactly the baseline it names, or is null (#3541 + A12). The SQL this replaced folded a missing 30-day baseline onto the 7-day one and a + missing 7-day one onto the oldest, and labelled the result with the window asked for. */ growth_7d_mb = r.Growth7dMb, growth_30d_mb = r.Growth30dMb, + growth_pct_30d = r.GrowthPct30d, + /* The figure that is always honest: growth from the store's earliest snapshot of this table + to its latest, over exactly growth_window_days. Null only when there is no span at all. */ + growth_over_available_history_mb = r.GrowthOverAvailableHistoryMb, + growth_over_available_history_pct = r.GrowthOverAvailableHistoryPct, + growth_window_days = r.DaysOfData, daily_growth_rate_mb = r.DailyGrowthRateMb, - growth_pct_30d = r.GrowthPct30d + growth_note = GrowthNote(r), }); return JsonSerializer.Serialize(new { server = resolved.ServerName, + history = new + { + earliest_snapshot = span.EarliestSnapshotTime.ToString("o"), + latest_snapshot = span.LatestSnapshotTime.ToString("o"), + history_days_available = span.DaysOfData, + covers_7d = covers7d, + covers_30d = covers30d, + note = covers30d + ? null + : $"The store holds {span.DaysOfData} day(s) of index snapshots for this server, so the " + + (covers7d ? "30-day baseline does not exist: growth_30d_mb and growth_pct_30d are null" : "7-day and 30-day baselines do not exist: growth_7d_mb, growth_30d_mb and growth_pct_30d are null") + + " rather than re-measured over a shorter span under the same name. Read growth_over_available_history_* — it spans exactly growth_window_days.", + }, + tables_returned = page.Count, + truncated, tables = result }, McpHelpers.JsonOptions); } @@ -79,6 +113,34 @@ public static async Task GetTableIndexSizes( } } + /// + /// Why a row's growth figures are null, when they are (#3541 A12): the store has no snapshot old enough + /// for the window, or the snapshot exists but this table was not in it (created since), or there is no + /// span at all. Null when every figure is defined, so the common row carries no note. Lite's twin words + /// it identically. + /// + internal static string? GrowthNote(DarlingObjectStatsReader.ObjectSizeGrowthRow r) + { + var notes = new List(); + if (r.DaysOfData < 1) + notes.Add("the store holds a single day of snapshots for this server, so no growth is knowable yet — every growth figure is null, not 0"); + if (r.Snapshot7dTime is null) + notes.Add("no snapshot 7+ days old exists, so growth_7d_mb is null"); + else if (r.ReservedMb7dAgo is null) + notes.Add($"this table was not in the {r.Snapshot7dTime:o} snapshot (created since), so growth_7d_mb is null — its whole current size is newer than 7 days"); + if (r.Snapshot30dTime is null) + notes.Add("no snapshot 30+ days old exists, so growth_30d_mb and growth_pct_30d are null"); + else if (r.ReservedMb30dAgo is null) + notes.Add($"this table was not in the {r.Snapshot30dTime:o} snapshot (created since), so growth_30d_mb and growth_pct_30d are null"); + else if (r.ReservedMb30dAgo <= 0) + notes.Add("the table was empty 30 days ago, so growth_pct_30d has no denominator and is null (growth_30d_mb carries the absolute)"); + if (r.DaysOfData >= 1 && r.ReservedMbOldest is null) + notes.Add($"this table was not in the earliest snapshot ({r.EarliestSnapshotTime:o}), so growth_over_available_history_* and daily_growth_rate_mb are null"); + else if (r.DaysOfData >= 1 && r.ReservedMbOldest <= 0) + notes.Add("the table was empty at the earliest snapshot, so growth_over_available_history_pct has no denominator and is null"); + return notes.Count == 0 ? null : string.Join("; ", notes) + "."; + } + [McpServerTool(Name = "get_index_usage"), Description("Gets per-index usage (seeks, scans, lookups, updates) from the latest daily snapshot, classifying each index as Unused, Write-only, or Active. Unused and write-only indexes are listed FIRST because they are drop candidates - which means that on a server with many unused indexes the row limit can be filled entirely by one database's unused indexes, hiding every Active index elsewhere. Pass database_name to ask about one database, which is almost always what you want; the response carries matching_index_count and truncated so a short answer is never mistaken for an absent one. Counters are cumulative since the last instance restart. last_user_access is UTC - the underlying sys.dm_db_index_usage_stats columns are in the monitored server's local clock and this read de-skews them - so it compares directly against get_collection_log and list_servers.")] public static async Task GetIndexUsage( NpgsqlDataSource postgres, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs index 5a092a3c6..5236f71ee 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgAutovacuumTools.cs @@ -29,8 +29,13 @@ public sealed class DarlingMcpPgAutovacuumTools /// count is routine on a large table and urgent on a small one, so the ratio is the only comparable /// figure — and whether the pile is growing separates autovacuum losing a race from autovacuum not /// running at all. + /// The ratio handed in is the WORSE of the dead-tuple and insert-only ratios — the same GREATEST + /// the read ranks by. Classifying from the dead ratio alone let the #1-ranked table, an append-only + /// one far past its INSERT threshold, carry severity "ok" (#3534). + /// Growth is nullable because a one-sample window cannot measure it: null never escalates, and + /// never reads as "flat" either. /// - internal static string Classify(bool autovacuumDisabled, double? thresholdRatio, bool deadTuplesGrowing) + internal static string Classify(bool autovacuumDisabled, double? thresholdRatio, bool? deadTuplesGrowing) { /* A configuration finding, and the one case where the count is beside the point: this table will never be vacuumed by autovacuum no matter how bad it gets, and it holds back the whole @@ -50,7 +55,7 @@ internal static string Classify(bool autovacuumDisabled, double? thresholdRatio, /* Ten times past the line is not a busy table, it is a stuck one — most often autovacuum being cancelled repeatedly by conflicting locks, or starved of workers. */ >= 10 => "critical_far_past_threshold", - >= 2 when deadTuplesGrowing => "warning_past_threshold_and_growing", + >= 2 when deadTuplesGrowing == true => "warning_past_threshold_and_growing", >= 2 => "warning_past_threshold", >= 1 => "info_at_threshold", _ => "ok", @@ -113,19 +118,37 @@ table we know nothing about above tables we have measured. */ double? analyzeRatio = r.AnalyzeThreshold > 0 ? Math.Round((double)r.ModsSinceAnalyze / r.AnalyzeThreshold, 2) : null; - var growing = r.DeadTuples > r.FirstDeadTuples; + /* The insert-side twin, mirroring the read's ORDER BY CASE: the -1 sentinel (a major + without autovacuum_vacuum_insert_threshold, or an unreadable inserts figure) produces + no ratio rather than a negative one. */ + double? insertRatio = r.InsertVacuumThreshold > 0 && r.InsertsSinceVacuum >= 0 + ? Math.Round((double)r.InsertsSinceVacuum / r.InsertVacuumThreshold, 2) + : null; + /* Severity comes from the axis the ranking already uses — GREATEST(dead, insert), NULLs + ignored, exactly as the read orders. Classifying from the dead ratio alone let an + append-only worst_table read "ok" (#3534). */ + double? worstRatio = ratio is null ? insertRatio + : insertRatio is null ? ratio + : Math.Max(ratio.Value, insertRatio.Value); + /* One sample cannot measure growth: first == latest is the same reading, and a false + there converts into "autovacuum blocked or not running" territory the window cannot + support. Null, not false — and the change figure goes with it. */ + bool? growing = r.FirstSeenAt == r.MeasuredAt ? null : r.DeadTuples > r.FirstDeadTuples; return new { database_name = r.DatabaseName, table_name = $"{r.SchemaName}.{r.TableName}", - severity = Classify(r.AutovacuumDisabled, ratio, growing), + severity = Classify(r.AutovacuumDisabled, worstRatio, growing), dead_tuples = r.DeadTuples, vacuum_threshold = r.VacuumThreshold >= 0 ? r.VacuumThreshold : (long?)null, /* The headline number: 1.0 means autovacuum should be triggering right now. */ threshold_ratio = ratio, dead_tuples_growing = growing, - dead_tuple_change = r.DeadTuples - r.FirstDeadTuples, + dead_tuple_change = growing is null ? null : (long?)(r.DeadTuples - r.FirstDeadTuples), + /* The window the growth claim is measured over, so a caller can see how much history + stands behind it — and that null growth means one sample, not missing data. */ + first_seen_at = r.FirstSeenAt, live_tuples = r.LiveTuples, /* The analyze half. Stale statistics produce bad row estimates and bad plans, which is a different symptom from bloat and gets missed because both come from one process. */ @@ -136,6 +159,9 @@ a different symptom from bloat and gets missed because both come from one proces rule, and a table never vacuumed is never frozen either. */ inserts_since_vacuum = r.InsertsSinceVacuum >= 0 ? r.InsertsSinceVacuum : (long?)null, insert_vacuum_threshold = r.InsertVacuumThreshold >= 0 ? r.InsertVacuumThreshold : (long?)null, + /* threshold_ratio's insert-side sibling, so the figure severity ranks on is visible + when the dead-tuple ratio is not the one that put the table here. */ + insert_threshold_ratio = insertRatio, autovacuum_disabled = r.AutovacuumDisabled, total_bytes = r.TotalBytes >= 0 ? r.TotalBytes : (long?)null, total_gb = r.TotalBytes >= 0 ? Math.Round(r.TotalBytes / 1024.0 / 1024.0 / 1024.0, 2) : (double?)null, @@ -158,9 +184,14 @@ a different symptom from bloat and gets missed because both come from one proces hours_back, status = "tables_with_pending_maintenance", table_count = tables.Count, + /* Page-scoped counts: each is computed over the rows the read's LIMIT let through, not + the server. limit_reached is the discriminator that makes that caveat actionable — when + it bit, the caller knows these are top-N figures and can raise the limit (the + get_pg_database_stats pattern). */ autovacuum_disabled_count = tables.Count(t => t.autovacuum_disabled), - past_threshold_count = tables.Count(t => t.threshold_ratio >= 1), - growing_count = tables.Count(t => t.dead_tuples_growing), + past_threshold_count = tables.Count(t => t.threshold_ratio >= 1 || t.insert_threshold_ratio >= 1), + growing_count = tables.Count(t => t.dead_tuples_growing == true), + limit_reached = tables.Count >= limit, worst_table = tables[0].table_name, worst_severity = tables[0].severity, tables, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs index 66e8dd3d2..199065a75 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgIoTools.cs @@ -33,12 +33,12 @@ public sealed class DarlingMcpPgIoTools /// internal static string ContextMeaning(string? context) => DarlingPgIoReader.ContextMeaning(context); - [McpServerTool(Name = "get_pg_io_stats"), Description("Gets PostgreSQL I/O attributed to WHO did it, to WHAT, and WHY - the (backend_type, object, context) breakdown from pg_stat_io, differenced across the requested window. Richer than SQL Server's file-level dm_io_virtual_file_stats: instead of 'this file is busy' you get 'autovacuum workers are reading relations in the vacuum context', which names the cause. The context dimension is the one with no SQL Server equivalent and the one that changes the remedy - it separates ordinary buffer-pool misses (where more shared_buffers or a better index helps) from sequential scans that deliberately bypass the pool via a ring buffer (where it will not help at all), from vacuum's ring buffer, from a standby applying WAL. Reports whether write counters are TRACKED at all, because on Amazon Aurora they are always null - backends there do not write data files, the storage layer does - and a zero would otherwise read as 'no writes happened'. Requires PostgreSQL 16 or later; valid on a standby.")] + [McpServerTool(Name = "get_pg_io_stats"), Description("Gets PostgreSQL I/O attributed to WHO did it, to WHAT, and WHY - the (backend_type, object, context) breakdown from pg_stat_io, differenced across the requested window. Richer than SQL Server's file-level dm_io_virtual_file_stats: instead of 'this file is busy' you get 'autovacuum workers are reading relations in the vacuum context', which names the cause. The context dimension is the one with no SQL Server equivalent and the one that changes the remedy - it separates ordinary buffer-pool misses (where more shared_buffers or a better index helps) from sequential scans that deliberately bypass the pool via a ring buffer (where it will not help at all), from vacuum's ring buffer, from a standby applying WAL. Reports whether the server tracks I/O TIMING at all: track_io_timing is OFF by default in PostgreSQL, and its zero read_time would otherwise divide out to a latency of 0.000 ms that reads as an impossibly fast disk rather than an unmeasured one - the time fields are null when untracked, and busiest_basis says what the ranking actually used. Also reports whether write counters are TRACKED at all, because on Amazon Aurora they are always null - backends there do not write data files, the storage layer does - and a zero would otherwise read as 'no writes happened'. Requires PostgreSQL 16 or later; valid on a standby. THE PAGE IS BOUNDED BY limit: combination_count is how many combinations you got, truncated says the window held more, and the rows are the busiest so the ones past the cap did less. SHARES ARE OF THE WINDOW, NOT OF THE PAGE: pct_of_total_reads' denominator is total_reads and pct_of_total_read_time's is total_read_time_ms, each the WHOLE window's figure across every combination that moved, computed in the same statement as the rows - so a three-row page does not sum to 100%, and the gap between returned_reads / returned_read_time_ms (what the page adds up to) and the totals is the I/O the cap left out; returned_pct_of_total_reads and returned_pct_of_total_read_time are those ratios stated once.")] public static async Task GetPgIoStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum (backend_type, object, context) combinations to return, busiest first. Default 20.")] int limit = 20, + [Description("Maximum (backend_type, object, context) combinations to return, busiest first. Default 20. This is what bounds the page - read truncated to know whether the window held more; the shares stay of the whole window whatever this is set to.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -52,10 +52,13 @@ public static async Task GetPgIoStats( try { var now = windowEnd; - var rows = await DarlingPgIoReader.GetPgIoAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit); + /* #3541 A7: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal - and the window's reads and read time ride on the same statement, so the shares + below have denominators the cap cannot shrink. */ + var page = await DarlingPgIoReader.GetPgIoPageAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); - if (rows.Count == 0) + if (page.Rows.Count == 0) { /* Ask the engine BEFORE offering the idle-server reading (#2532). "No combination recorded activity" is a statement about a PostgreSQL instance; said about a SQL Server target it @@ -79,108 +82,202 @@ public static async Task GetPgIoStats( }, McpHelpers.JsonOptions); } - var totalReads = rows.Sum(r => r.Reads); - var totalReadTime = rows.Sum(r => r.ReadTimeMs); + /* Asked of the server's OWN configuration rather than inferred from the zeros, exactly as the + trend sibling asks it (#3536): the two readings a zero latency permits — "the disk is + instant" and "nobody is timing it" — are not distinguishable in the counters, and + track_io_timing is OFF by default, so the zeros are the ordinary case rather than a fault. */ + var timingSetting = await DarlingPgTrendReader.GetIoTimingTrackedAsync( + postgres, resolved.ServerId, windowEnd); - var combinations = rows.Select(r => - { - var accesses = r.Reads + r.Hits; - return new - { - backend_type = r.BackendType, - object_type = r.ObjectType, - context = r.Context, - context_meaning = ContextMeaning(r.Context), - reads = r.Reads, - read_time_ms = Math.Round(r.ReadTimeMs, 1), - /* Per-read latency is the figure that separates "a lot of I/O" from "slow I/O", and - they have completely different remedies. */ - avg_read_ms = r.Reads > 0 ? Math.Round(r.ReadTimeMs / r.Reads, 3) : (double?)null, - hits = r.Hits, - /* A hit ratio scoped to this combination, which is the only scope where it means - anything: a server-wide ratio averages bulkread's deliberate misses together with - normal-context misses and understates both. */ - hit_pct = accesses > 0 ? Math.Round((double)r.Hits / accesses * 100, 1) : (double?)null, - pct_of_total_reads = totalReads > 0 ? Math.Round((double)r.Reads / totalReads * 100, 1) : 0, - pct_of_total_read_time = totalReadTime > 0 ? Math.Round(r.ReadTimeMs / totalReadTime * 100, 1) : 0, - extends = r.Extends, - extend_time_ms = Math.Round(r.ExtendTimeMs, 1), - evictions = r.Evictions, - /* Ring-buffer reuse, NOT eviction pressure. Conflating the two is the standard - misreading of this view: reuses are a bulk operation recycling its OWN buffers. */ - reuses = r.Reuses, - writes = r.WriteCountersTracked ? r.Writes : (long?)null, - write_time_ms = r.WriteCountersTracked ? Math.Round(r.WriteTimeMs, 1) : (double?)null, - write_counters_tracked = r.WriteCountersTracked, - /* The block size an operation moves. Gone from 18, where a read is no longer one - block, so it is null there and read_bytes below is measured instead of derived. */ - block_bytes = r.OpBytes > 0 ? r.OpBytes : (long?)null, - /* One name for the volume answer, and bytes_source says how it was arrived at. From 18 - these are measured totals; below 18 they are reads x block size. Never both, and - never silently swapped: the two are different quantities, and on 18 the old estimate - would UNDERCOUNT because a vectored read covers several blocks. */ - read_bytes = r.ByteCountersTracked - ? r.ReadBytes - : (r.OpBytes > 0 ? r.Reads * r.OpBytes : (decimal?)null), - write_bytes = r.ByteCountersTracked - ? r.WriteBytes - : (r.OpBytes > 0 && r.WriteCountersTracked ? r.Writes * r.OpBytes : (decimal?)null), - extend_bytes = r.ByteCountersTracked ? r.ExtendBytes : (decimal?)null, - bytes_source = r.ByteCountersTracked - ? "measured" - : (r.OpBytes > 0 ? "estimated_from_block_size" : "unavailable"), - stats_reset = r.StatsReset, - }; - }) - .ToList(); - - var anyWritesTracked = rows.Any(r => r.WriteCountersTracked); - /* #2655: PostgreSQL 18 replaced op_bytes with measured byte totals. Said once at the top for - the same reason the write flag is: a caller has to know which quantity it is reading before - it compares two servers, and the two are not comparable. */ - var bytesMeasured = rows.Any(r => r.ByteCountersTracked); - var bytesEstimated = !bytesMeasured && rows.Any(r => r.OpBytes > 0); - - return JsonSerializer.Serialize(new - { - server = resolved.ServerName, - hours_back, - status = "io_activity", - combination_count = combinations.Count, - total_reads = totalReads, - total_read_time_ms = Math.Round(totalReadTime, 1), - busiest_by_read_time = $"{rows[0].BackendType}/{rows[0].ObjectType}/{rows[0].Context}", - /* Said once at the top rather than repeated per row: on Aurora this is false everywhere, - and a caller needs to know the write side is unmeasured before it concludes anything - from the absence of writes. */ - write_counters_tracked_anywhere = anyWritesTracked, - bytes_source = bytesMeasured - ? "measured" - : (bytesEstimated ? "estimated_from_block_size" : "unavailable"), - note = anyWritesTracked - ? "All counters are windowed differences, clamped per interval so a stats reset cannot " - + "produce a negative figure." - : "All counters are windowed differences. This server tracks NO write counters — the " - + "signature of Amazon Aurora, where backends do not write data files and the storage " - + "layer does. Absent writes here mean unmeasured, not zero.", - bytes_note = bytesMeasured - ? "Byte totals are MEASURED, from PostgreSQL 18's read_bytes/write_bytes/extend_bytes. " - + "They are not comparable with the figures a pre-18 server reports, which are " - + "reads x block size - 18 reads several blocks per operation, so the older estimate " - + "undercounts." - : (bytesEstimated - ? "Byte totals are ESTIMATED as count x block_bytes, which is exact below " - + "PostgreSQL 18 because one operation moves one block. PostgreSQL 18 measures " - + "them directly instead." - : "This server reports no byte figures at all: op_bytes is absent and the measured " - + "columns PostgreSQL 18 replaced it with are not being collected. The counts and " - + "times above are unaffected."), - combinations, - }, McpHelpers.JsonOptions); + return BuildIoJson(resolved.ServerName, hours_back, page, limit, timingSetting); } catch (Exception ex) { return McpHelpers.Status("error", $"Reading PostgreSQL I/O stats failed: {ex.Message}"); } } + + /// + /// The response body, split out so the WIRE SHAPE can be asserted without a live store — the same + /// reason the plan tools' BuildPlansJson is separate. + /// + /// timingSetting is the target's own track_io_timing from pg_server_config, + /// or null when the configuration has not been collected — in which case whether any non-zero time + /// appears in the window is the only evidence available and is used, stated as inference. + /// + /// The denominators are the window's, not the page's (#3541 A7). + /// carries WindowTotalReads and WindowTotalReadTimeMs from the same statement as its rows, and + /// pct_of_total_reads / pct_of_total_read_time divide by THOSE. The previous shape divided by + /// the sums of the rows fetched, so at limit = 3 the three shares summed to 100% and read as "these + /// three combinations are all the I/O". The page's own sums still travel as returned_reads / + /// returned_read_time_ms, and the ratios of page to window are stated once each. + /// + /// holds up to limit + 1 rows; the extra one is the truncation signal + /// and is cut before the projection. The timing inference below runs over the CUT rows: the sentinel is + /// not on the page, so it must not decide anything the page reports. + /// + internal static string BuildIoJson( + string serverName, + int hoursBack, + DarlingPgIoReader.PgIoPage page, + int limit, + bool? timingSetting) + { + var truncated = page.Rows.Count > limit; + var rows = truncated ? page.Rows.Take(limit).ToList() : page.Rows; + + var timingObserved = rows.Any(r => r.ReadTimeMs > 0 || r.WriteTimeMs > 0); + var timingTracked = timingSetting ?? timingObserved; + + var totalReads = page.WindowTotalReads; + var totalReadTime = page.WindowTotalReadTimeMs; + var returnedReads = rows.Sum(r => r.Reads); + var returnedReadTime = rows.Sum(r => r.ReadTimeMs); + + var combinations = rows.Select(r => + { + var accesses = r.Reads + r.Hits; + return new + { + backend_type = r.BackendType, + object_type = r.ObjectType, + context = r.Context, + context_meaning = ContextMeaning(r.Context), + reads = r.Reads, + /* Every time figure is null when the server does not measure I/O time (#3536), rather + than the 0.0 the arithmetic produces: that zero is a fact about the configuration, and + printed as a time it is the most reassuring wrong number here. */ + read_time_ms = timingTracked ? Math.Round(r.ReadTimeMs, 1) : (double?)null, + /* Per-read latency is the figure that separates "a lot of I/O" from "slow I/O", and + they have completely different remedies. */ + avg_read_ms = timingTracked && r.Reads > 0 ? Math.Round(r.ReadTimeMs / r.Reads, 3) : (double?)null, + hits = r.Hits, + /* A hit ratio scoped to this combination, which is the only scope where it means + anything: a server-wide ratio averages bulkread's deliberate misses together with + normal-context misses and understates both. */ + hit_pct = accesses > 0 ? Math.Round((double)r.Hits / accesses * 100, 1) : (double?)null, + /* Of the WINDOW's totals, never of the page's — see the remarks. */ + pct_of_total_reads = totalReads > 0 ? Math.Round((double)r.Reads / totalReads * 100, 1) : 0, + pct_of_total_read_time = timingTracked + ? (totalReadTime > 0 ? Math.Round(r.ReadTimeMs / totalReadTime * 100, 1) : 0) + : (double?)null, + extends = r.Extends, + extend_time_ms = timingTracked ? Math.Round(r.ExtendTimeMs, 1) : (double?)null, + evictions = r.Evictions, + /* Ring-buffer reuse, NOT eviction pressure. Conflating the two is the standard + misreading of this view: reuses are a bulk operation recycling its OWN buffers. */ + reuses = r.Reuses, + writes = r.WriteCountersTracked ? r.Writes : (long?)null, + write_time_ms = timingTracked && r.WriteCountersTracked ? Math.Round(r.WriteTimeMs, 1) : (double?)null, + write_counters_tracked = r.WriteCountersTracked, + /* The block size an operation moves. Gone from 18, where a read is no longer one + block, so it is null there and read_bytes below is measured instead of derived. */ + block_bytes = r.OpBytes > 0 ? r.OpBytes : (long?)null, + /* One name for the volume answer, and bytes_source says how it was arrived at. From 18 + these are measured totals; below 18 they are reads x block size. Never both, and + never silently swapped: the two are different quantities, and on 18 the old estimate + would UNDERCOUNT because a vectored read covers several blocks. */ + read_bytes = r.ByteCountersTracked + ? r.ReadBytes + : (r.OpBytes > 0 ? r.Reads * r.OpBytes : (decimal?)null), + write_bytes = r.ByteCountersTracked + ? r.WriteBytes + : (r.OpBytes > 0 && r.WriteCountersTracked ? r.Writes * r.OpBytes : (decimal?)null), + extend_bytes = r.ByteCountersTracked ? r.ExtendBytes : (decimal?)null, + bytes_source = r.ByteCountersTracked + ? "measured" + : (r.OpBytes > 0 ? "estimated_from_block_size" : "unavailable"), + stats_reset = r.StatsReset, + }; + }) + .ToList(); + + var anyWritesTracked = rows.Any(r => r.WriteCountersTracked); + /* #2655: PostgreSQL 18 replaced op_bytes with measured byte totals. Said once at the top for + the same reason the write flag is: a caller has to know which quantity it is reading before + it compares two servers, and the two are not comparable. */ + var bytesMeasured = rows.Any(r => r.ByteCountersTracked); + var bytesEstimated = !bytesMeasured && rows.Any(r => r.OpBytes > 0); + + return JsonSerializer.Serialize(new + { + server = serverName, + hours_back = hoursBack, + status = "io_activity", + /* #3541 A3 dialect: the page described as a page. combination_count keeps its name — the web + I/O tile reads it by key — and it IS a page count, which the description says; truncated + beside it says whether the window held more. No time bounds: each row is one combination + differenced across the whole window, so there is no page reach to report, only a cap. */ + combination_count = combinations.Count, + truncated, + order = "read_time_ms_desc_then_reads_desc", + /* The WINDOW's reads and read time, across every combination that moved — the denominators of + every pct_of_total_reads / pct_of_total_read_time above. NOT the sums of the rows; those are + returned_reads / returned_read_time_ms. The read-time total is null under untracked timing + for the same reason every other time figure is: it is a sum of stored zeros, not a measurement. */ + total_reads = totalReads, + total_read_time_ms = timingTracked ? Math.Round(totalReadTime, 1) : (double?)null, + returned_reads = returnedReads, + returned_read_time_ms = timingTracked ? Math.Round(returnedReadTime, 1) : (double?)null, + returned_pct_of_total_reads = totalReads > 0 ? Math.Round((double)returnedReads / totalReads * 100, 1) : 0, + returned_pct_of_total_read_time = timingTracked + ? (totalReadTime > 0 ? Math.Round(returnedReadTime / totalReadTime * 100, 1) : 0) + : (double?)null, + /* The key survives untracked timing for the web tile's sake; busiest_basis beside it says + what the ranking actually used. The reader orders by read time and then by read count, so + over a store of zero times the count IS the ordering rather than a tiebreak. */ + busiest_by_read_time = $"{rows[0].BackendType}/{rows[0].ObjectType}/{rows[0].Context}", + busiest_basis = timingTracked + ? "total read time (read_time_ms), then read count" + : "read count (reads) — this server does not measure I/O time, so every read-time figure " + + "is zero in the store and cannot rank anything; the ordering falls back to the counter " + + "that exists", + /* Said once at the top rather than repeated per row: on Aurora this is false everywhere, + and a caller needs to know the write side is unmeasured before it concludes anything + from the absence of writes. */ + write_counters_tracked_anywhere = anyWritesTracked, + io_timing_tracked = timingTracked, + io_timing_source = timingSetting is null + ? "inferred from the data - this server's configuration has not been collected, so " + + "track_io_timing is unknown and the answer here is simply whether any non-zero I/O " + + "time appears in the window" + : "the target's own track_io_timing, as collected into pg_server_config", + bytes_source = bytesMeasured + ? "measured" + : (bytesEstimated ? "estimated_from_block_size" : "unavailable"), + note = (anyWritesTracked + ? "All counters are windowed differences, clamped per interval so a stats reset cannot " + + "produce a negative figure." + : "All counters are windowed differences. This server tracks NO write counters — the " + + "signature of Amazon Aurora, where backends do not write data files and the storage " + + "layer does. Absent writes here mean unmeasured, not zero.") + + " total_reads and total_read_time_ms are the WHOLE window's figures across every " + + "combination that moved, computed in the same statement as the rows; each row's " + + "pct_of_total_reads and pct_of_total_read_time divide by them, so the shares on a page do " + + "not sum to 100 unless the page is the whole window (truncated = false). returned_reads " + + "and returned_read_time_ms are what the rows returned add up to.", + timing_note = timingTracked + ? "read_time_ms, avg_read_ms, write_time_ms and extend_time_ms are measured I/O times: " + + "this server has track_io_timing on." + : "read_time_ms, avg_read_ms, write_time_ms, extend_time_ms and the read-time shares are " + + "NULL throughout because this server does not measure I/O time. track_io_timing is off " + + "by DEFAULT in PostgreSQL, so this is the ordinary configuration rather than a fault - " + + "but it means the operation counts are the only I/O evidence here, and nothing in this " + + "store can say whether the storage is slow. Turning it on costs a clock read per " + + "operation; measure that on the platform before enabling it fleet-wide.", + bytes_note = bytesMeasured + ? "Byte totals are MEASURED, from PostgreSQL 18's read_bytes/write_bytes/extend_bytes. " + + "They are not comparable with the figures a pre-18 server reports, which are " + + "reads x block size - 18 reads several blocks per operation, so the older estimate " + + "undercounts." + : (bytesEstimated + ? "Byte totals are ESTIMATED as count x block_bytes, which is exact below " + + "PostgreSQL 18 because one operation moves one block. PostgreSQL 18 measures " + + "them directly instead." + : "This server reports no byte figures at all: op_bytes is absent and the measured " + + "columns PostgreSQL 18 replaced it with are not being collected. The counts and " + + "times above are unaffected."), + combinations, + }, McpHelpers.JsonOptions); + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgKernelStatsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgKernelStatsTools.cs index 82a634e22..7c87f4c28 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgKernelStatsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgKernelStatsTools.cs @@ -42,12 +42,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpPgKernelStatsTools { - [McpServerTool(Name = "get_pg_kernel_stats"), Description("Gets per-query-shape OPERATING SYSTEM resource usage from the pg_stat_kcache extension: CPU split into user and system time, bytes that reached the storage device, and major page faults. Use this together with get_pg_top_queries, which reports ELAPSED time: elapsed is CPU plus waiting, so comparing the two tells you whether a slow statement is burning CPU or waiting on something, which is the first split any tuning question needs. The byte counters are DEVICE reads and writes, so a zero means the operating system page cache served the request rather than that no data was read - do not read them as logical I/O. queryid joins get_pg_top_queries and get_pg_wait_sampling. Rows are ranked by total CPU.")] + [McpServerTool(Name = "get_pg_kernel_stats"), Description("Gets per-query-shape OPERATING SYSTEM resource usage from the pg_stat_kcache extension: CPU split into user and system time, bytes that reached the storage device, and major page faults. Use this together with get_pg_top_queries, which reports ELAPSED time: elapsed is CPU plus waiting, so comparing the two tells you whether a slow statement is burning CPU or waiting on something, which is the first split any tuning question needs. The byte counters are DEVICE reads and writes, so a zero means the operating system page cache served the request rather than that no data was read - do not read them as logical I/O. queryid joins get_pg_top_queries and get_pg_wait_sampling. Rows are ranked by total CPU. THE PAGE IS BOUNDED BY limit: queries_returned is how many shapes you got, truncated says the window held more, and the rows are the heaviest so the ones past the cap burned less. SHARES ARE OF THE WINDOW, NOT OF THE PAGE: pct_of_total_cpu's denominator is total_cpu_ms, the WHOLE window's user + system CPU across every (database, query) series, computed in the same statement as the rows - so a three-row page does not sum to 100%, and the gap between returned_cpu_ms (what the page adds up to) and total_cpu_ms is the CPU the cap left out; returned_pct_of_total is that ratio stated once.")] public static async Task GetPgKernelStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum rows to return. Default 20.")] int limit = 20, + [Description("Maximum query shapes to return, most CPU first. Default 20. This is what bounds the page - read truncated to know whether the window held more; the shares stay of the whole window whatever this is set to.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -60,10 +60,13 @@ public static async Task GetPgKernelStats( try { - var rows = await DarlingPgKernelStatsReader.GetPgKernelStatsAsync( - postgres, resolved.ServerId, windowEnd.AddHours(-hours_back), windowEnd, limit); + /* #3541 A7: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal - and the window's CPU rides on the same statement, so the shares below have a + denominator the cap cannot shrink. */ + var page = await DarlingPgKernelStatsReader.GetPgKernelStatsPageAsync( + postgres, resolved.ServerId, windowEnd.AddHours(-hours_back), windowEnd, limit + 1); - if (rows.Count == 0) + if (page.Rows.Count == 0) { /* pg_stat_kcache needs shared_preload_libraries and a restart, so "not installed" is the likely answer and the precondition vocabulary names the fix. Ordered after the @@ -79,52 +82,89 @@ capability check so a wrong-engine target is never told to install an extension. + "has nothing to difference against and the window fills on the second one."); } - var totalCpuMs = rows.Sum(r => r.TotalCpuMs); - var resetInWindow = rows.Any(r => r.CounterReset); - - var queries = rows.Select(r => new - { - /* String, like every other queryid on this surface — a signed 64-bit value would round in - a double-decoding JSON parser and produce an id that joins to nothing. */ - queryid = r.QueryId.ToString(CultureInfo.InvariantCulture), - database_name = r.DatabaseName, - cpu_ms = Math.Round(r.TotalCpuMs, 1), - /* Kept split rather than only summed: system time dominated by kernel work is a different - finding from user time dominated by the planner or by expression evaluation. */ - user_cpu_ms = Math.Round(r.ExecUserTimeMs, 1), - system_cpu_ms = Math.Round(r.ExecSystemTimeMs, 1), - pct_of_total_cpu = totalCpuMs > 0 ? Math.Round(r.TotalCpuMs / totalCpuMs * 100, 1) : 0, - device_read_bytes = r.ExecReadBytes, - device_write_bytes = r.ExecWriteBytes, - /* Bytes AND megabytes, the convention get_pg_index_usage already follows: an agent wants - the exact figure, a grid wants something readable, and deriving one from the other at - the display layer is where rounding disagreements start. */ - device_read_mb = Math.Round(r.ExecReadBytes / 1024.0 / 1024.0, 1), - device_write_mb = Math.Round(r.ExecWriteBytes / 1024.0 / 1024.0, 1), - /* A major fault is a page read from disk to satisfy a memory access — the signal that the - host is short of memory, which no PostgreSQL-side counter reports at all. */ - major_faults = r.MajorFaults, - counter_reset = r.CounterReset, - }); - - return JsonSerializer.Serialize(new - { - server = resolved.ServerName, - hours_back, - total_cpu_ms = Math.Round(totalCpuMs, 1), - note = "CPU is measured by the OPERATING SYSTEM, not by PostgreSQL. device_read_bytes and " - + "device_write_bytes count bytes that reached the device, so zero means the page " - + "cache served it rather than that nothing was read." - + (resetInWindow - ? " At least one series was RESET inside this window, so its figures cover only " - + "the time since the reset." - : string.Empty), - queries, - }, McpHelpers.JsonOptions); + return BuildKernelStatsJson(resolved.ServerName, hours_back, page, limit); } catch (Exception ex) { return McpHelpers.Status("error", $"Reading PostgreSQL kernel stats failed: {ex.Message}"); } } + + /// + /// The response body, split out so the WIRE SHAPE can be asserted without a live store. + /// + /// The denominator is the window's, not the page's (#3541 A7). carries + /// WindowTotalCpuMs from the same statement as its rows, and every pct_of_total_cpu divides by + /// THAT. The previous shape divided by the sum of the rows fetched, so at limit = 3 the three shares + /// summed to 100% and read as "these three burned all the CPU". The page's own sum still travels as + /// returned_cpu_ms; the ratio of the two is returned_pct_of_total. + /// + internal static string BuildKernelStatsJson( + string serverName, + int hoursBack, + DarlingPgKernelStatsReader.PgKernelStatsPage page, + int limit) + { + var truncated = page.Rows.Count > limit; + var rows = truncated ? page.Rows.Take(limit).ToList() : page.Rows; + + var windowTotalCpuMs = page.WindowTotalCpuMs; + var returnedCpuMs = rows.Sum(r => r.TotalCpuMs); + var resetInWindow = rows.Any(r => r.CounterReset); + + var queries = rows.Select(r => new + { + /* String, like every other queryid on this surface — a signed 64-bit value would round in + a double-decoding JSON parser and produce an id that joins to nothing. */ + queryid = r.QueryId.ToString(CultureInfo.InvariantCulture), + database_name = r.DatabaseName, + cpu_ms = Math.Round(r.TotalCpuMs, 1), + /* Kept split rather than only summed: system time dominated by kernel work is a different + finding from user time dominated by the planner or by expression evaluation. */ + user_cpu_ms = Math.Round(r.ExecUserTimeMs, 1), + system_cpu_ms = Math.Round(r.ExecSystemTimeMs, 1), + /* Of the WINDOW's CPU, never of the page's — see the remarks. */ + pct_of_total_cpu = windowTotalCpuMs > 0 ? Math.Round(r.TotalCpuMs / windowTotalCpuMs * 100, 1) : 0, + device_read_bytes = r.ExecReadBytes, + device_write_bytes = r.ExecWriteBytes, + /* Bytes AND megabytes, the convention get_pg_index_usage already follows: an agent wants + the exact figure, a grid wants something readable, and deriving one from the other at + the display layer is where rounding disagreements start. */ + device_read_mb = Math.Round(r.ExecReadBytes / 1024.0 / 1024.0, 1), + device_write_mb = Math.Round(r.ExecWriteBytes / 1024.0 / 1024.0, 1), + /* A major fault is a page read from disk to satisfy a memory access — the signal that the + host is short of memory, which no PostgreSQL-side counter reports at all. */ + major_faults = r.MajorFaults, + counter_reset = r.CounterReset, + }) + .ToList(); + + return JsonSerializer.Serialize(new + { + server = serverName, + hours_back = hoursBack, + /* #3541 A3 dialect: the page described as a page. No time bounds — each row is one series + differenced across the whole window, so there is no page reach to report, only a cap. */ + queries_returned = queries.Count, + truncated, + order = "cpu_ms_desc", + /* The WINDOW's CPU, across every series — the denominator of every pct_of_total_cpu above. + NOT the sum of the rows; that is returned_cpu_ms. */ + total_cpu_ms = Math.Round(windowTotalCpuMs, 1), + returned_cpu_ms = Math.Round(returnedCpuMs, 1), + returned_pct_of_total = windowTotalCpuMs > 0 ? Math.Round(returnedCpuMs / windowTotalCpuMs * 100, 1) : 0, + note = "CPU is measured by the OPERATING SYSTEM, not by PostgreSQL. device_read_bytes and " + + "device_write_bytes count bytes that reached the device, so zero means the page " + + "cache served it rather than that nothing was read. total_cpu_ms is the WHOLE window's " + + "user + system CPU across every series, computed in the same statement as the rows; " + + "each row's pct_of_total_cpu divides by it, so the shares on a page do not sum to 100 " + + "unless the page is the whole window (truncated = false). returned_cpu_ms is what the " + + "rows returned add up to." + + (resetInWindow + ? " At least one series was RESET inside this window, so its figures cover only " + + "the time since the reset." + : string.Empty), + queries, + }, McpHelpers.JsonOptions); + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgLoggingAuditTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgLoggingAuditTools.cs new file mode 100644 index 000000000..ccceadba5 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgLoggingAuditTools.cs @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.ComponentModel; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using ModelContextProtocol.Server; +using Npgsql; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Storage; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// get_pg_logging_audit (#3607): whether a PostgreSQL target's logging settings are producing the lines +/// they could, setting by setting, with the remedy for each in the hosting flavour's own syntax. +/// +/// The judgment is 's and the rows are +/// 's; this class is the wire. It sits beside +/// get_pg_plan_capture_readiness deliberately: that read judges plan capture's own preconditions +/// and this one judges the rest of the logging surface, and between them target onboarding has one "is this +/// target telling us everything it could" answer, which is the issue's ask. The two do not overlap — +/// the plan-capture settings appear here as observed values with a pointer, never as a second verdict. +/// +/// No hours_back and no as_of PARAMETER, and that is the Stamped latest-read shape +/// (#3541 A10) rather than an omission. Configuration is a state, not a window: the audit is of the +/// NEWEST snapshot, and the response carries that snapshot's collection time as captured_at — the +/// one stamp every row shares, selected on the row statement — so the reader knows how old the state is. A +/// windowed form would answer "no logging configuration" about a server whose hourly collector last ran +/// just outside the window. McpLatestSnapshotStampTests holds the dialect; this tool is its +/// Darling-only allowance because the roster's other half is a Lite file and Lite has no PostgreSQL. +/// +[McpServerToolType] +public sealed class DarlingMcpPgLoggingAuditTools +{ + [McpServerTool(Name = "get_pg_logging_audit"), Description("Audits a PostgreSQL target's LOGGING settings - log_min_duration_statement, log_lock_waits, log_temp_files, log_autovacuum_min_duration, log_checkpoints, log_connections, log_disconnections - and says, per setting, whether it is producing the lines it could, what telemetry those lines unlock, the recommended value WITH its cost, and the remedy in the syntax this server's hosting needs (ALTER SYSTEM plus a reload where the server is yours to administer; a parameter group on RDS/Aurora, decided from rds.* parameters in the stored snapshot rather than guessed). It reads the STORED configuration snapshot pg_server_config already collects hourly, never the live server. LATEST IS A TIME: captured_at is the instant that snapshot was taken, the collector runs hourly, so every value here is 'as of' that stamp and a change made since is not reflected until the next collection. Read it at onboarding and whenever a target-side log read comes back empty: a target with every one of these off looks identical to a fully instrumented one from every counter-based read, and the difference shows up at incident time when the log somebody reaches for holds nothing. Verdicts are instrumented, partial, off or unknown - partial means a THRESHOLD is filtering (statements faster than N ms, temp files under N kB) and the row says what falls below it; for log_min_duration_statement the threshold IS the recommended posture, and its cost_note says so, because 0 logs every statement the server runs. unknown means the setting is not in the snapshot and nothing is inferred. Every facet names the Darling family that would consume its lines and says PLANNED where that consumer does not ship yet (#3601/#3602/#3603) - the counter reads that exist today are named beside it with what they cannot see. Plan capture's own settings (auto_explain, log_line_prefix %Q, lc_messages) are LISTED as observed for completeness but judged by get_pg_plan_capture_readiness, which owns their traps; lc_messages decides whether any of these lines are written in the English the parsers match. PostgreSQL-only.")] + public static async Task GetPgLoggingAudit( + NpgsqlDataSource postgres, + [Description("Server name or display name.")] string? server_name = null) + { + var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); + if (error != null) return error; + + try + { + var snapshot = await DarlingPgLoggingAuditReader.GetNewestSnapshotAsync(postgres, resolved.ServerId); + + if (snapshot.Count == 0) + { + /* The same miss vocabulary as get_pg_server_config, because it is the same absence: the + collector this reads never ran here (engine gate), or has not run YET. Neither is an audit + result, and an audit of an empty snapshot would say 'unknown' seven times and look like one. */ + return await DarlingEngineCapability.NotCollectedStatusAsync( + postgres, resolved.ServerId, resolved.ServerName, "pg_server_config") + ?? McpHelpers.Status( + "empty", + $"No configuration snapshot has been collected for {resolved.ServerName} yet, so there " + + "is nothing to audit. pg_server_config runs hourly; a server registered in the last " + + "hour has not reached its first collection. This is not a verdict about the " + + "server's logging - it is the absence of the evidence."); + } + + return BuildAuditJson(resolved.ServerName, DarlingPgLoggingAudit.Audit(snapshot)); + } + catch (Exception ex) + { + /* The engine gate again, inside the catch, the way the plan tools do it: a read that throws on a + store where this collector never runs should still answer not_collected rather than a raw + error, because the gate is the more specific fact and the exception is its symptom. */ + var gated = await DarlingEngineCapability.NotCollectedStatusAsync( + postgres, resolved.ServerId, resolved.ServerName, "pg_server_config"); + if (gated != null) + { + return gated; + } + + return McpHelpers.Status("error", $"Reading the PostgreSQL logging audit failed: {ex.Message}"); + } + } + + /// + /// The response body, split out so the WIRE SHAPE can be asserted without a live store — the reason + /// BuildReadinessJson is separate on the plan tools. + /// + internal static string BuildAuditJson(string serverName, DarlingPgLoggingAudit.Result audit) + { + var facets = audit.Facets; + + /* The counts are over ALL facets, never a page - there is no limit on this read, the facet list is + fixed and small, so the #2629 cap-versus-window trap does not arise and the summary is a fact about + the server. off and unknown are NAMED because they are the actionable ones; partial is not + listed as a to-do because for two settings it is the recommendation. */ + var off = facets.Where(f => f.Verdict == DarlingPgLoggingAudit.Off).Select(f => f.Setting).ToArray(); + var unknown = facets.Where(f => f.Verdict == DarlingPgLoggingAudit.Unknown).Select(f => f.Setting).ToArray(); + /* Named at the top for the reason get_pg_server_config names them: a judged value that the next + restart will change is the one row whose remedy should not be acted on from this response alone. */ + var pendingRestart = facets.Where(f => f.PendingRestart).Select(f => f.Setting).ToArray(); + + return JsonSerializer.Serialize(new + { + server = serverName, + status = "logging_audit", + /* The snapshot's collection time, under the #3541 A10 name every stamped latest read uses. It + is how old this state is, not a window. */ + captured_at = audit.CapturedAt.ToString("o"), + source = "pg_server_config, newest snapshot - stored configuration, not the live server", + hosting = audit.Managed ? "managed (RDS/Aurora)" : "self-hosted", + hosting_evidence = audit.HostingEvidence, + total = facets.Count, + instrumented_count = facets.Count(f => f.Verdict == DarlingPgLoggingAudit.Instrumented), + partial_count = facets.Count(f => f.Verdict == DarlingPgLoggingAudit.Partial), + off_count = off.Length, + unknown_count = unknown.Length, + off_settings = off, + unknown_settings = unknown, + pending_restart_count = pendingRestart.Length, + pending_restart_settings = pendingRestart, + note = "One facet per logging setting, in the order an operator reaches for them - statements, " + + "locks, spills, maintenance, checkpoints, connections - not a causal order; nothing here " + + "gates anything else. verdict describes the LINES: instrumented writes every line the " + + "setting can, partial has a threshold filtering and the row says what falls below it, off " + + "writes nothing, unknown is not in the snapshot and nothing is inferred. partial is the " + + "recommended posture for log_min_duration_statement and can be for log_temp_files - read " + + "cost_note before changing a partial row. A row with pending_restart true is judged on the " + + "RUNNING value while the file already holds another - read restart_note before acting on its " + + "remedy. consumer names the Darling family that reads the " + + "lines and says PLANNED where it does not ship yet; the setting is still worth turning on " + + "first, because the log it fills is the one somebody opens at incident time. remedy is " + + "worded for this server's hosting (see hosting_evidence). judged_by_readiness lists plan " + + "capture's own settings as observed in the same snapshot; " + DarlingPgLoggingAudit.ReadinessTool + + " judges them, and its message_locale facet decides whether ANY of these lines are written " + + "in the English the parsers match.", + facets = facets.Select(f => new + { + setting = f.Setting, + verdict = f.Verdict, + /* Verbatim, so a reader sees what the server said rather than this tool's reading of it. */ + value = f.Value, + unit = f.Unit, + default_value = f.DefaultValue, + source = f.Source, + change_needs = f.ChangeNeeds, + unlocks = f.Unlocks, + consumer = f.Consumer, + recommended = f.Recommended, + cost_note = f.CostNote, + remedy = f.Remedy, + scope_note = f.ScopeNote, + /* The file and the running server disagree: the value above is the running one and the + remedy is written against it. restart_note says what that means and where to look. */ + pending_restart = f.PendingRestart, + restart_note = f.RestartNote, + }), + judged_by_readiness = new + { + tool = DarlingPgLoggingAudit.ReadinessTool, + note = "Shown as observed in this snapshot for completeness and NOT judged here: " + + DarlingPgLoggingAudit.ReadinessTool + " owns each of these with its own trap - a loaded " + + "auto_explain at -1 captures nothing, an auto_explain.* value on a server that never " + + "loaded the module is a placeholder, a log_line_prefix without %Q orphans every plan, " + + "and a translated lc_messages blinds every log read here. A null value means the " + + "setting is not in the snapshot, which for auto_explain.log_min_duration usually means " + + "the library is not loaded.", + settings = audit.JudgedByReadiness.Select(s => new + { + setting = s.Setting, + value = s.Value, + source = s.Source, + readiness_facet = s.ReadinessFacet, + }), + }, + }, McpHelpers.JsonOptions); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgPlanTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgPlanTools.cs index f34a708b4..1e17300c7 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgPlanTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgPlanTools.cs @@ -53,7 +53,7 @@ public static async Task GetPgPlans( [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, [Description("Maximum plan shapes to return. Default 10.")] int limit = 10, - [Description("Only return plans for this queryid, as a string. Optional.")] string? query_id = null, + [Description("Only return plans for this queryid, as a string. Optional. The filter is applied in the store over EVERY capture in the window, not over the top-duration page, so a statement ranked far below the busiest shapes is still found - and an empty answer with this set genuinely means no plan for it was captured in the window.")] string? query_id = null, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -87,17 +87,16 @@ public static async Task GetPgPlans( var now = windowEnd; var start = now.AddHours(-hours_back); + /* The queryid pin travels INTO the SQL (#3533). It used to be applied here, over a fetched + top-duration page, which made every plan ranked below the page unfindable and then reported + the miss as "not captured" — the predicate has to run where the rows are. */ var rows = await DarlingPgPlanCaptureReader.GetPgPlanCaptureAsync( - postgres, resolved.ServerId, start, now, wantedQueryId is null ? limit : limit * 10); - - if (wantedQueryId is not null) - { - rows = rows.Where(r => r.QueryId == wantedQueryId.Value).Take(limit).ToList(); - } + postgres, resolved.ServerId, start, now, limit, wantedQueryId); if (rows.Count == 0) { - return await NoPlansStatusAsync(postgres, resolved.ServerId, resolved.ServerName, wantedQueryId); + return await NoPlansStatusAsync( + postgres, resolved.ServerId, resolved.ServerName, wantedQueryId, hours_back); } return BuildPlansJson(resolved.ServerName, hours_back, rows, limit); @@ -259,7 +258,7 @@ to avoid. The unsatisfied facets are NAMED instead. */ /// the threshold" become a true statement rather than a guess. /// private static async Task NoPlansStatusAsync( - NpgsqlDataSource postgres, int serverId, string serverName, long? wantedQueryId) + NpgsqlDataSource postgres, int serverId, string serverName, long? wantedQueryId, int hoursBack) { var gated = await DarlingEngineCapability.NotCollectedStatusAsync( postgres, serverId, serverName, "pg_plan_capture"); @@ -291,18 +290,38 @@ private static async Task NoPlansStatusAsync( + "satisfied or not, in the order they have to be fixed in."); } - var subject = wantedQueryId is null ? "any statement" : "this statement"; - - return McpHelpers.Status( - "empty", - $"Capture is configured on this server and no plan was captured for {subject} in this window. " - + "Two things produce that and they are different: the statement never ran longer than " - + "auto_explain.log_min_duration, which is the healthy answer and means it is not the query to " - + "look at; or a plan was captured earlier and has aged out, which plan_content_retention_days " - + "governs — widen hours_back to tell those apart, because a plan that exists further back will " - + "reappear and one that never existed will not."); + return McpHelpers.Status("empty", NoPlanCapturedMessage(wantedQueryId, hoursBack)); } + /// + /// The empty answer once capture is known to be configured, split by whether a queryid was asked for. + /// + /// The queryid arm changed with #3533 and its honesty depends on the reader: the filter now runs + /// in the SQL over every capture in the window, so "no plan for this statement was captured" is a fact + /// rather than a statement about a fetched page. The text this replaced said the query was "not the + /// query to look at" — over a top-N page that was an invented verdict, and even over the whole window + /// it collapses three different causes into the most reassuring one. + /// + internal static string NoPlanCapturedMessage(long? wantedQueryId, int hoursBack) => wantedQueryId is null + ? $"Capture is configured on this server and nothing was captured in the last {hoursBack} hour(s) — " + + "this read has no filter, so that is a fact about every statement, not about a page of them. " + + "Two things produce it and they are different: no statement ran longer than " + + "auto_explain.log_min_duration, which is the healthy answer on a server whose statements are " + + "all fast; or plans were captured earlier and have aged out, which plan_content_retention_days " + + "governs — widen hours_back to tell those apart, because a plan that exists further back will " + + "reappear and one that never existed will not." + : $"Every capture in the last {hoursBack} hour(s) was searched for this query_id — the whole " + + "window, in the store, not a top-N page — and none matches, so no plan for this statement was " + + "captured in this window. That has three different causes with three different remedies: the " + + "statement did not run in this window at all, which get_pg_top_queries can confirm from its " + + "call counts; it ran but never crossed auto_explain.log_min_duration, so auto_explain never " + + "wrote a plan — the healthy reading for a statement that is fast HERE, not a verdict about it " + + "at other times; or capture was not working when it ran — get_pg_plan_capture_readiness " + + "reports every precondition with the remedy beside it, and its facets are the latest reading " + + "rather than the window's history, so a server that is ready now can still have missed an " + + "earlier run. A plan captured before this window has aged out under " + + "plan_content_retention_days; widen hours_back to reach further back."; + /// /// The unsatisfied readiness facets, newest reading per facet. Read directly rather than through the /// readiness tool so this stays a fact lookup rather than one MCP tool narrating another's prose. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs index af0eb207a..b3499c09e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgSlotTools.cs @@ -28,8 +28,12 @@ public sealed class DarlingMcpPgSlotTools /// Severity from slot state, not from the retained figure alone. The size of the hole matters far /// less than whether it is still being dug: a slot holding 45 GB steadily is a consumer keeping pace, /// while one that grew from 2 GB to 45 GB in an hour is a volume filling in front of you. + /// Growth is nullable because it is not always measurable — the collector's -1 sentinel on + /// either endpoint, or a one-sample window. Unknown never escalates to the growing verdict, and never + /// reads as the flat one either: "flat" is a measured claim, and it is the claim that separates a + /// consumer between polls from a volume filling (#3535). /// - internal static string Classify(string? walStatus, bool isActive, bool retainedWalGrowing) => + internal static string Classify(string? walStatus, bool isActive, bool? retainedWalGrowing) => walStatus switch { /* The slot is already unusable — its consumer cannot resume and needs recreating. */ @@ -37,13 +41,37 @@ internal static string Classify(string? walStatus, bool isActive, bool retainedW /* Required WAL has been removed; the consumer is about to find that out. */ "unreserved" => "critical_wal_already_removed", /* WAL is being retained BECAUSE of this slot. Inactive and still growing is the disk bomb. */ - "extended" when !isActive && retainedWalGrowing => "critical_orphan_filling_disk", + "extended" when !isActive && retainedWalGrowing == true => "critical_orphan_filling_disk", + "extended" when !isActive && retainedWalGrowing is null => "warning_retaining_wal_growth_unknown", "extended" => "warning_retaining_wal", - _ when !isActive && retainedWalGrowing => "warning_inactive_and_growing", + _ when !isActive && retainedWalGrowing == true => "warning_inactive_and_growing", + _ when !isActive && retainedWalGrowing is null => "info_inactive_growth_unknown", _ when !isActive => "info_inactive", _ => "ok", }; + /// + /// Rank by how bad the label is, so the headline slot is the worst-CLASSIFIED one rather than the + /// fattest one (the get_pg_wraparound_risk pattern). Picking worst_slot by retained size contradicted + /// this type's own design note: an active 45 GB keeping-pace slot ("ok") outranked an inactive 2→8 GB + /// grower ("critical_orphan_filling_disk") — the one slot the caller needed to see first (#3535). + /// Size still breaks ties within a label. + /// + internal static int Rank(string severity) => severity switch + { + "critical_slot_lost" => 8, + "critical_wal_already_removed" => 7, + "critical_orphan_filling_disk" => 6, + "warning_inactive_and_growing" => 5, + /* Above the measured-flat warning: this is the orphan shape with its discriminator unmeasured, + which deserves the look before a slot known to be holding steady. */ + "warning_retaining_wal_growth_unknown" => 4, + "warning_retaining_wal" => 3, + "info_inactive_growth_unknown" => 2, + "info_inactive" => 1, + _ => 0, + }; + [McpServerTool(Name = "get_pg_replication_slots"), Description("Gets PostgreSQL replication slot health, including whether any slot is retaining WAL without bound. An abandoned slot is one of the few PostgreSQL conditions that can take a server down by itself, and it does so two independent ways: it retains every WAL segment its consumer has not confirmed - unbounded by default, so it will fill the volume and stop the server - and it simultaneously pins the vacuum horizon so nothing gets reclaimed cluster-wide. Reports whether retained WAL is still GROWING across the window, which is the difference between a consumer that is merely behind and a volume filling in front of you. Common orphan sources are a removed CDC task, a finished blue/green deployment, a decommissioned Debezium consumer, or a failed major-version upgrade. Works on any PostgreSQL target.")] public static async Task GetPgReplicationSlots( NpgsqlDataSource postgres, @@ -94,11 +122,16 @@ collector has never run and never will. */ var slots = rows.Select(r => { - var growthBytes = r.RetainedWalBytes >= 0 && r.FirstRetainedWalBytes >= 0 - ? r.RetainedWalBytes - r.FirstRetainedWalBytes - : 0; + /* Growth exists only when BOTH endpoints were measured across a window that spans time. + The -1 sentinel used to become a measured 0 here — "no growth" — feeding growing=false + into Classify, so an unmeasurable slot read as stable (#3535); and a one-sample window + is the same fabrication one step milder, "flat" from a single point. Null, both. */ + long? growthBytes = + r.MeasuredAt != r.FirstSeenAt && r.RetainedWalBytes >= 0 && r.FirstRetainedWalBytes >= 0 + ? r.RetainedWalBytes - r.FirstRetainedWalBytes + : null; var hours = Math.Max((r.MeasuredAt - r.FirstSeenAt).TotalHours, 0); - var growing = growthBytes > 0; + bool? growing = growthBytes is null ? null : growthBytes > 0; return new { @@ -109,15 +142,17 @@ collector has never run and never will. */ database_name = r.DatabaseName, is_active = r.IsActive, wal_status = r.WalStatus, - retained_wal_bytes = r.RetainedWalBytes, + /* -1 is the collector's sentinel, not a size — null it here as the _gb twin below + always has, rather than serializing a figure no slot can hold. */ + retained_wal_bytes = r.RetainedWalBytes >= 0 ? r.RetainedWalBytes : (long?)null, retained_wal_gb = r.RetainedWalBytes >= 0 ? Math.Round(r.RetainedWalBytes / 1024.0 / 1024.0 / 1024.0, 2) : (double?)null, /* Growth is the actionable half. Rate is only reported when the window actually spans time, so a single-sample window cannot produce a fabricated per-hour figure. */ retained_wal_growth_bytes = growthBytes, - retained_wal_growth_gb_per_hour = hours >= 0.05 - ? Math.Round(growthBytes / 1024.0 / 1024.0 / 1024.0 / hours, 3) + retained_wal_growth_gb_per_hour = growthBytes is not null && hours >= 0.05 + ? Math.Round(growthBytes.Value / 1024.0 / 1024.0 / 1024.0 / hours, 3) : (double?)null, /* -1 is the collector's not-applicable sentinel: safe_wal_size is NULL whenever max_slot_wal_keep_size is -1, which is the default, so on a stock server there is @@ -132,7 +167,11 @@ no configured ceiling at all. */ conflicting = r.Conflicting, }; }) - .OrderByDescending(s => s.retained_wal_bytes) + /* Worst-CLASSIFIED first; size only breaks ties. Ordering by size alone put the fattest slot + in worst_slot regardless of verdict (#3535). Unknown sizes sort after measured ones within + a label. */ + .OrderByDescending(s => Rank(s.severity)) + .ThenByDescending(s => s.retained_wal_bytes ?? -1) .ToList(); var worst = slots[0]; @@ -147,7 +186,7 @@ no configured ceiling at all. */ worst_slot = worst.slot_name, worst_severity = worst.severity, total_retained_wal_gb = Math.Round( - slots.Where(s => s.retained_wal_bytes > 0).Sum(s => s.retained_wal_bytes) / 1024.0 / 1024.0 / 1024.0, 2), + slots.Where(s => s.retained_wal_bytes > 0).Sum(s => s.retained_wal_bytes ?? 0) / 1024.0 / 1024.0 / 1024.0, 2), slots, }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs index 734b7aef1..836eb20e8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgStatementTools.cs @@ -7,7 +7,6 @@ */ using System; -using System.Collections.Generic; using System.ComponentModel; using System.Globalization; using System.Linq; @@ -27,12 +26,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpPgStatementTools { - [McpServerTool(Name = "get_pg_top_queries"), Description("Gets the top PostgreSQL query shapes by total execution time over a time period, wherever pg_stat_statements is installed: on Amazon Aurora the collector reads aurora_stat_statements(), and on any other PostgreSQL the vanilla view, with the Aurora-only columns null rather than zero. On Aurora that includes the I/O source breakdown, which stock PostgreSQL cannot provide: a block 'read' may have come from the distributed storage volume or from the local NVMe Optimized Reads cache, and the two have very different costs. Also reports peak memory per statement (Aurora only), the closest PostgreSQL equivalent of a memory grant, and WAL bytes generated, which has no SQL Server DMV counterpart. Returns query_text for each statement, captured hourly and keyed on queryid, or null when none has been captured yet (a statement first seen minutes ago, or a queryid minted by a major-version upgrade). queryid itself is stable within a major version but changes across a major upgrade — which is exactly why the text is STORED rather than fetched live: after an upgrade the live view no longer holds the old ids, so their text would otherwise be unrecoverable and the history would read as a list of integers. queryid is returned as a STRING, not a number: it is a signed 64-bit value spread over the whole int8 range, so most ids exceed what a JSON number survives and a numeric wire form would be silently rounded by any parser that decodes numbers as IEEE-754 doubles — after which it matches nothing. Compare it and join on it as text. This is a separate tool from get_top_queries_by_cpu, which covers SQL Server.")] + [McpServerTool(Name = "get_pg_top_queries"), Description("Gets the top PostgreSQL query shapes by total execution time over a time period, wherever pg_stat_statements is installed: on Amazon Aurora the collector reads aurora_stat_statements(), and on any other PostgreSQL the vanilla view, with the Aurora-only columns null rather than zero. On Aurora that includes the I/O source breakdown, which stock PostgreSQL cannot provide: a block 'read' may have come from the distributed storage volume or from the local NVMe Optimized Reads cache, and the two have very different costs. Also reports peak memory per statement (Aurora only), the closest PostgreSQL equivalent of a memory grant, and WAL bytes generated, which has no SQL Server DMV counterpart. Returns query_text for each statement, captured hourly and keyed on queryid, or null when none has been captured yet (a statement first seen minutes ago, or a queryid minted by a major-version upgrade). queryid itself is stable within a major version but changes across a major upgrade — which is exactly why the text is STORED rather than fetched live: after an upgrade the live view no longer holds the old ids, so their text would otherwise be unrecoverable and the history would read as a list of integers. queryid is returned as a STRING, not a number: it is a signed 64-bit value spread over the whole int8 range, so most ids exceed what a JSON number survives and a numeric wire form would be silently rounded by any parser that decodes numbers as IEEE-754 doubles — after which it matches nothing. Compare it and join on it as text. This is a separate tool from get_top_queries_by_cpu, which covers SQL Server. THE PAGE IS BOUNDED BY limit: queries_returned is how many shapes you got, truncated says the window held more, and the rows are the heaviest so the ones past the cap are lighter. SHARES ARE OF THE WINDOW, NOT OF THE PAGE: pct_of_total_time's denominator is total_exec_time_ms, the WHOLE window's execution time across every query shape, computed in the same statement as the rows - so a three-row page does not sum to 100%, and the gap between returned_exec_time_ms (what the page adds up to) and total_exec_time_ms is the work the cap left out; returned_pct_of_total is that ratio stated once.")] public static async Task GetPgTopQueries( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum rows to return. Default 20.")] int limit = 20, + [Description("Maximum query shapes to return, heaviest by total execution time first. Default 20. This is what bounds the page - read truncated to know whether the window held more; the shares stay of the whole window whatever this is set to.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -46,10 +45,14 @@ public static async Task GetPgTopQueries( try { var now = windowEnd; - var rows = await DarlingPgStatementReader.GetPgTopQueriesAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + /* #3541 A7 (and the A3 shape it sat on): the caller's limit + 1 as the fetch, the extra row as the + observed truncation signal. The reader carried LIMIT 50 as a literal under a limit the tool + accepts up to 1,000, and the tool's Take(limit) on top of it meant a share was computed over + whichever of the two caps bit - never over the window. */ + var page = await DarlingPgStatementReader.GetPgTopQueriesPageAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); - if (rows.Count == 0) + if (page.Rows.Count == 0) { /* The capability answer settles the dialect branch and the stock-PostgreSQL branch whenever the store knows the engine (#2532), so this sentence is reached in exactly two @@ -79,7 +82,7 @@ reporting it when somebody acts on the advice. */ + "not be a PostgreSQL one at all — check list_servers."); } - return BuildTopQueriesJson(resolved.ServerName, hours_back, rows, limit); + return BuildTopQueriesJson(resolved.ServerName, hours_back, page, limit); } catch (Exception ex) { @@ -118,16 +121,31 @@ an Aurora target still surfaces as an error. */ /// tool itself needs a live store and a resolved server, neither of which a serialization guard has any /// business standing up — and a guard that re-implements the projection instead would keep passing while /// the shipped one drifted underneath it. + /// + /// The denominator is the window's, not the page's (#3541 A7). carries + /// WindowTotalExecTimeMs from the same statement as its rows, and every pct_of_total_time divides + /// by THAT. The previous shape divided by the sum of the rows fetched, so at limit = 3 the three + /// shares summed to 100% and read as "these three are everything" to any agent holding only the JSON. + /// The page's own sum still travels, under a name that says what it is (returned_exec_time_ms), and + /// the ratio of the two is stated once as returned_pct_of_total — the honest form of "the top N + /// account for X%". + /// + /// holds up to limit + 1 rows; the extra one is the truncation signal + /// and is cut before the projection. The window total is unaffected by the cut by construction. /// internal static string BuildTopQueriesJson( string serverName, int hoursBack, - IReadOnlyList rows, + DarlingPgStatementReader.PgTopQueriesPage page, int limit) { - var totalTimeMs = rows.Sum(r => r.TotalExecTimeMs); + var truncated = page.Rows.Count > limit; + var rows = truncated ? page.Rows.Take(limit).ToList() : page.Rows; - var result = rows.Take(limit).Select(r => + var windowTotalMs = page.WindowTotalExecTimeMs; + var returnedMs = rows.Sum(r => r.TotalExecTimeMs); + + var result = rows.Select(r => { /* Null when the source did not report the split at all (self-hosted PostgreSQL, #2625), which is a different answer from zero and must not become a 0% cache-hit ratio. */ @@ -153,7 +171,8 @@ database_id stays a number beside it because an oid is unsigned 32-bit and so ca avg_exec_time_ms = r.Calls > 0 ? Math.Round((double)r.TotalExecTimeMs / r.Calls, 3) : 0, max_exec_time_ms = Math.Round(r.MaxExecTimeMs, 3), rows_returned = r.RowsReturned, - pct_of_total_time = totalTimeMs > 0 ? Math.Round((double)r.TotalExecTimeMs / totalTimeMs * 100, 1) : 0, + /* Of the WINDOW's total, never of the page's — see the remarks. */ + pct_of_total_time = windowTotalMs > 0 ? Math.Round((double)r.TotalExecTimeMs / windowTotalMs * 100, 1) : 0, /* Aurora's I/O split, which is the point of using aurora_stat_statements over the vanilla view. A high orcache share means the reads were cheap local NVMe hits; a high storage share means network round trips to the cluster volume. The community @@ -178,20 +197,34 @@ the local_blks_* family and a different problem. */ // have none until the next refresh. An empty string would read as "the query is blank". query_text = r.QueryText, }; - }); + }) + .ToList(); return JsonSerializer.Serialize(new { server = serverName, hours_back = hoursBack, - total_exec_time_ms = totalTimeMs, + /* #3541 A3 dialect: the page described as a page. No time bounds — the rows are per-shape + aggregates over the whole window, so there is no page reach to report, only a cap. */ + queries_returned = result.Count, + truncated, + order = "total_exec_time_ms_desc", + /* The WINDOW's execution time, across every shape that ran — the denominator of every + pct_of_total_time below. It is NOT the sum of the rows; that is returned_exec_time_ms. */ + total_exec_time_ms = windowTotalMs, + returned_exec_time_ms = returnedMs, + returned_pct_of_total = windowTotalMs > 0 ? Math.Round((double)returnedMs / windowTotalMs * 100, 1) : 0, /* Every counter here covers the window, so a caller can safely divide one by another. Only the two high-water marks are not counters, and saying which is cheaper than letting someone assume max_exec_time_ms is a windowed total. */ note = "All counters cover the requested window: calls, total_exec_time_ms and " + "rows_returned from stored per-interval deltas, and the block and WAL figures " + "differenced across the window's snapshots. max_exec_time_ms and " - + "max_exec_peakmem_bytes are high-water marks, not windowed totals.", + + "max_exec_peakmem_bytes are high-water marks, not windowed totals. The top-level " + + "total_exec_time_ms is the WHOLE window's execution time across every query shape, " + + "computed in the same statement as the rows; each row's pct_of_total_time divides by it, " + + "so the shares on a page do not sum to 100 unless the page is the whole window " + + "(truncated = false). returned_exec_time_ms is what the rows returned add up to.", queries = result, }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitSamplingTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitSamplingTools.cs index 1384c3fbd..4d522c2a1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitSamplingTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitSamplingTools.cs @@ -14,6 +14,7 @@ using System.Threading.Tasks; using ModelContextProtocol.Server; using Npgsql; +using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Storage; @@ -45,16 +46,27 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// mostly CPU and a profile that is mostly IO call for opposite next steps, and a wait-only view cannot /// tell them apart. /// +/// +/// +/// It discloses its INSTRUMENT (#3604). Since the collector grew a service-side sampler arm for +/// targets without the extension, the same table holds two grains: the extension's 10 ms in-engine samples +/// and this service's one-second pg_stat_activity polls over a 30 s window each cycle. A count of 300 +/// is three seconds of waiting under one and five minutes under the other, and estimated_wait_ms is +/// right on both only because each arm stores its own period. So every non-empty answer carries +/// instrument (extension_sampled or service_sampled, read from the collector's own +/// per-server state) and, on the service tier, the floor caveat in +/// — the same disclose-the-instrument pattern get_pg_plan_capture_readiness established for plans. +/// /// [McpServerToolType] public sealed class DarlingMcpPgWaitSamplingTools { - [McpServerTool(Name = "get_pg_wait_sampling"), Description("Gets sampled PostgreSQL wait events attributed to query shapes, from the pg_wait_sampling extension. This is the stock-PostgreSQL counterpart of get_pg_wait_stats, which reads an Aurora-only source: use this tool on any self-hosted or non-Aurora PostgreSQL target. A sampling profiler periodically records what each backend is doing, so results are sample COUNTS, and estimated_wait_ms is samples multiplied by the sampling period rather than a measured duration - treat a rare event's estimate as approximate. Rows with event_type CPU mean the backend was running, not waiting, so the profile answers 'waiting or working' as well as 'waiting on what'. queryid joins get_pg_top_queries; queryid 0 is work belonging to no statement, such as a background worker. The profile is cluster-wide and carries no database attribution by design.")] + [McpServerTool(Name = "get_pg_wait_sampling"), Description("Gets sampled PostgreSQL wait events attributed to query shapes, for any non-Aurora PostgreSQL target. This is the stock-PostgreSQL counterpart of get_pg_wait_stats, which reads an Aurora-only source. PostgreSQL wait history comes from one of THREE instruments, chosen once per target when the service connects (Aurora native > pg_wait_sampling extension > service sampler), and the answer's instrument field says which one fed these rows: extension_sampled means the pg_wait_sampling extension's in-engine 10 ms profiler; service_sampled means this service polled pg_stat_activity once a second for a 30-second window every five minutes because the extension is not installed - a FLOOR, not parity, that under-counts waits shorter than a second and cannot see between windows (instrument_note spells out the limit and the one-step upgrade). Either way a sampler periodically records what each backend is doing, so results are sample COUNTS, and estimated_wait_ms is samples multiplied by that instrument's own period rather than a measured duration - treat a rare event's estimate as approximate. Rows with event_type CPU mean the backend was running, not waiting, so the profile answers 'waiting or working' as well as 'waiting on what'. queryid joins get_pg_top_queries; queryid 0 is work belonging to no statement, such as a background worker. The profile is cluster-wide and carries no database attribution by design. THE PAGE IS BOUNDED BY limit: waits_returned is how many rows you got, truncated says the window held more, and the rows are the most-sampled so the ones past the cap are rarer. SHARES ARE OF THE WINDOW, NOT OF THE PAGE: pct_of_samples' denominator is total_samples, the WHOLE window's differenced sample count across every (event type, event, query) series, computed in the same statement as the rows - so a three-row page does not sum to 100%, and the gap between returned_samples (what the page adds up to) and total_samples is the activity the cap left out; returned_pct_of_total is that ratio stated once.")] public static async Task GetPgWaitSampling( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum rows to return. Default 20.")] int limit = 20, + [Description("Maximum rows to return, most samples first. Default 20. This is what bounds the page - read truncated to know whether the window held more; the shares stay of the whole window whatever this is set to.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -67,10 +79,18 @@ public static async Task GetPgWaitSampling( try { - var rows = await DarlingPgWaitSamplingReader.GetPgWaitSamplingAsync( - postgres, resolved.ServerId, windowEnd.AddHours(-hours_back), windowEnd, limit); + /* #3541 A7: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal - and the window's sample count rides on the same statement, so the shares below + have a denominator the cap cannot shrink. */ + var page = await DarlingPgWaitSamplingReader.GetPgWaitSamplingPageAsync( + postgres, resolved.ServerId, windowEnd.AddHours(-hours_back), windowEnd, limit + 1); - if (rows.Count == 0) + /* #3604: which arm fed these rows, off the collector's own state. Read on the empty path too, so + a service-tier target that is genuinely idle is told it is being sampled at the floor grain + rather than left to wonder whether the extension tier was ever in play. */ + var instrument = await DarlingPgWaitSamplingReader.GetWaitInstrumentAsync(postgres, resolved.ServerId); + + if (page.Rows.Count == 0) { /* Three distinct empty states, and only the last of them is "nothing happened". The capability answer rules out a wrong-engine target; the precondition answer names an @@ -87,52 +107,145 @@ public static async Task GetPgWaitSampling( + "figures here are per-interval deltas, so a single collection has nothing to " + "difference against and the window fills on the second one. On a genuinely idle " + "server this is the healthy state: the profiler samples backends, and an idle " - + "server has none to sample."); + + "server has none to sample." + + DescribeInstrumentForEmpty(instrument)); } - /* The denominator is SAMPLES, not the estimate. Every row's estimate is the same count times - the same period, so the two shares are arithmetically identical — and taking it from the - count keeps the percentage anchored to what was actually observed. */ - var totalSamples = rows.Sum(r => r.SampleCount); - var resetInWindow = rows.Any(r => r.CounterReset); - - var waits = rows.Select(r => new - { - event_type = r.EventType, - wait_event = r.Event, - /* String, like every other queryid on this surface: it is a signed 64-bit value and a - JSON number would round it in any double-decoding parser, silently producing an id - that joins to nothing. */ - queryid = r.QueryId.ToString(CultureInfo.InvariantCulture), - samples = r.SampleCount, - estimated_wait_ms = r.EstimatedWaitMs, - backends = r.BackendCount, - pct_of_samples = totalSamples > 0 - ? Math.Round((double)r.SampleCount / totalSamples * 100, 1) - : 0, - /* Per row, because a reset is per series: one query's profile can be reset while another - accumulated normally, and a single window-level flag would libel both. */ - counter_reset = r.CounterReset, - }); - - return JsonSerializer.Serialize(new - { - server = resolved.ServerName, - hours_back, - total_samples = totalSamples, - note = "Sample COUNTS from a periodic profiler, not measured durations. estimated_wait_ms " - + "is samples multiplied by the sampling period and is approximate — most so for rare " - + "events. event_type CPU means the backend was running rather than waiting." - + (resetInWindow - ? " At least one series was RESET inside this window, so its figures cover only " - + "the time since the reset." - : string.Empty), - waits, - }, McpHelpers.JsonOptions); + return BuildWaitSamplingJson(resolved.ServerName, hours_back, page, limit, instrument); } catch (Exception ex) { return McpHelpers.Status("error", $"Reading PostgreSQL sampled waits failed: {ex.Message}"); } } + + /// + /// The sentence the empty arm appends about the instrument (#3604): the service tier's floor caveat when + /// that is what is sampling, nothing when the extension is (its idle answer needs no qualification), and a + /// plain "not yet recorded" when no cycle has stated an arm. + /// + internal static string DescribeInstrumentForEmpty(DarlingPgWaitSamplingReader.WaitInstrumentState? instrument) + { + if (instrument is null) + { + return " No collection cycle has recorded which wait instrument this server is on yet, so the " + + "first answer here will also say whether it is the extension's profiler or the service sampler."; + } + + return string.Equals(instrument.Instrument, PgWaitInstrument.ServiceSampled, StringComparison.Ordinal) + ? " This server is on the service_sampled tier (the pg_wait_sampling extension is not installed), " + + "so \"none to sample\" was measured over one-second polls in a 30-second window each cycle. " + + PgWaitInstrument.ServiceSampledCaveat + : string.Empty; + } + + /// + /// The response body, split out so the WIRE SHAPE can be asserted without a live store. + /// + /// The denominator is the window's, not the page's (#3541 A7). carries + /// WindowTotalSamples from the same statement as its rows, and every pct_of_samples divides by + /// THAT. The previous shape divided by the sum of the rows fetched, so at limit = 3 the three shares + /// summed to 100% and "waiting or working?" was answered over three series rather than over the + /// profile. The page's own sum still travels as returned_samples; the ratio of the two is + /// returned_pct_of_total. + /// + /// The denominator is SAMPLES, not the estimate. Every row's estimate is the same count times the + /// same period, so the two shares are arithmetically identical — and taking it from the count keeps + /// the percentage anchored to what was actually observed. + /// + internal static string BuildWaitSamplingJson( + string serverName, + int hoursBack, + DarlingPgWaitSamplingReader.PgWaitSamplingPage page, + int limit, + DarlingPgWaitSamplingReader.WaitInstrumentState? instrument = null) + { + /* #3604: an unrecognised token is not echoed as an instrument - a future arm this build does not know + reads as unknown, which is the honest word, rather than as a grain the caller might act on. */ + var instrumentToken = instrument is not null && PgWaitInstrument.IsKnown(instrument.Instrument) + ? instrument.Instrument + : "unknown"; + var serviceTier = string.Equals(instrumentToken, PgWaitInstrument.ServiceSampled, StringComparison.Ordinal); + var truncated = page.Rows.Count > limit; + var rows = truncated ? page.Rows.Take(limit).ToList() : page.Rows; + + var windowTotalSamples = page.WindowTotalSamples; + var returnedSamples = rows.Sum(r => r.SampleCount); + var resetInWindow = rows.Any(r => r.CounterReset); + + var waits = rows.Select(r => new + { + event_type = r.EventType, + wait_event = r.Event, + /* String, like every other queryid on this surface: it is a signed 64-bit value and a + JSON number would round it in any double-decoding parser, silently producing an id + that joins to nothing. */ + queryid = r.QueryId.ToString(CultureInfo.InvariantCulture), + samples = r.SampleCount, + estimated_wait_ms = r.EstimatedWaitMs, + backends = r.BackendCount, + /* Of the WINDOW's samples, never of the page's — see the remarks. */ + pct_of_samples = windowTotalSamples > 0 + ? Math.Round((double)r.SampleCount / windowTotalSamples * 100, 1) + : 0, + /* Per row, because a reset is per series: one query's profile can be reset while another + accumulated normally, and a single window-level flag would libel both. */ + counter_reset = r.CounterReset, + }) + .ToList(); + + return JsonSerializer.Serialize(new + { + server = serverName, + hours_back = hoursBack, + /* #3541 A3 dialect: the page described as a page. No time bounds — each row is one series + differenced across the whole window, so there is no page reach to report, only a cap. */ + waits_returned = waits.Count, + truncated, + order = "samples_desc", + /* #3604: the grain. Beside the page facts rather than buried in the note, because it changes what + every number below means and a reader comparing two servers must see it first. */ + instrument = instrumentToken, + instrument_recorded_at = instrument?.RecordedAtUtc, + instrument_note = instrumentToken switch + { + PgWaitInstrument.ExtensionSampled => + "The pg_wait_sampling extension's in-engine profiler: every backend sampled every " + + "profile_period (10 ms by default) and attributed to its queryid. The finest grain " + + "available on stock PostgreSQL.", + PgWaitInstrument.ServiceSampled => PgWaitInstrument.ServiceSampledCaveat, + _ => "No collection cycle has recorded which arm fed these rows (a store written before the " + + "service sampler existed, or a collector that has not completed a cycle since). Read " + + "profile_period_ms per row with that in mind: 10 is the extension's default, 1000 is " + + "the service sampler's.", + }, + /* The WINDOW's samples, across every series — the denominator of every pct_of_samples above. + NOT the sum of the rows; that is returned_samples. */ + total_samples = windowTotalSamples, + returned_samples = returnedSamples, + returned_pct_of_total = windowTotalSamples > 0 + ? Math.Round((double)returnedSamples / windowTotalSamples * 100, 1) + : 0, + note = "Sample COUNTS from a periodic profiler, not measured durations. estimated_wait_ms " + + "is samples multiplied by the sampling period and is approximate — most so for rare " + + "events. event_type CPU means the backend was running rather than waiting. " + + "total_samples is the WHOLE window's sample count across every series, computed in the " + + "same statement as the rows; each row's pct_of_samples divides by it, so the shares on a " + + "page do not sum to 100 unless the page is the whole window (truncated = false). " + + "returned_samples is what the rows returned add up to." + + (serviceTier + ? " backend_count on this tier is the distinct backends seen in the LAST window, not " + + "since the profile started. The tally behind these counts is kept in the monitoring " + + "store's own collector_state and reloaded every cycle, so it SURVIVES a service restart; " + + "a series starts over only when the tally's 500-key cap evicted it as least-sampled and " + + "it was seen again, or when the target moved to the extension tier and back — and " + + "counter_reset reports either the same way it reports an extension reset." + : string.Empty) + + (resetInWindow + ? " At least one series was RESET inside this window, so its figures cover only " + + "the time since the reset." + : string.Empty), + waits, + }, McpHelpers.JsonOptions); + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs index 89e2e36ea..9756d2660 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgWaitTools.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using ModelContextProtocol.Server; using Npgsql; +using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; using PerformanceMonitor.Darling.Storage; @@ -24,12 +25,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpPgWaitTools { - [McpServerTool(Name = "get_pg_wait_stats"), Description("Gets the top PostgreSQL wait events aggregated over a time period, for Amazon Aurora PostgreSQL targets. Waits reveal what the database spends time waiting on: IO events point at storage or cache misses, Lock events at blocking between sessions, LWLock at internal contention, and LSN is Aurora's storage-durability wait. Background-worker and client-idle waits are already excluded by the collector, so every row here is real work. Note this is a separate tool from get_wait_stats, which covers SQL Server: PostgreSQL has a two-level type/event taxonomy, no signal-wait concept, and reports in microseconds, so the two cannot share one result shape.")] + [McpServerTool(Name = "get_pg_wait_stats"), Description("Gets the top PostgreSQL wait events aggregated over a time period, for Amazon Aurora PostgreSQL targets - the engine_cumulative tier, the finest of the three PostgreSQL wait instruments (Aurora native > pg_wait_sampling extension > service sampler; stock targets take one of the other two and get_pg_wait_sampling serves them, disclosing which). Waits reveal what the database spends time waiting on: IO events point at storage or cache misses, Lock events at blocking between sessions, LWLock at internal contention, and LSN is Aurora's storage-durability wait. Background-worker and client-idle waits are already excluded by the collector, so every row here is real work. Note this is a separate tool from get_wait_stats, which covers SQL Server: PostgreSQL has a two-level type/event taxonomy, no signal-wait concept, and reports in microseconds, so the two cannot share one result shape. THE PAGE IS BOUNDED BY limit: wait_events_returned is how many events you got, truncated says the window held more, and the rows are the heaviest so the ones past the cap are lighter. SHARES ARE OF THE WINDOW, NOT OF THE PAGE: pct_of_total_wait's denominator is total_wait_time_ms, the WHOLE window's wait time across every event, computed in the same statement as the rows - so a three-row page does not sum to 100%, and the gap between returned_wait_time_ms (what the page adds up to) and total_wait_time_ms is the waiting the cap left out; returned_pct_of_total is that ratio stated once.")] public static async Task GetPgWaitStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history to analyze. Default 24.")] int hours_back = 24, - [Description("Maximum rows to return. Default 20.")] int limit = 20, + [Description("Maximum wait events to return, heaviest total wait first. Default 20. This is what bounds the page - read truncated to know whether the window held more; the shares stay of the whole window whatever this is set to.")] int limit = 20, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -43,13 +44,16 @@ public static async Task GetPgWaitStats( try { var now = windowEnd; - var rows = await DarlingPgWaitReader.GetPgWaitStatsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit); + /* #3541 A7: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal - and the window total rides on the same statement, so the shares below have a + denominator the cap cannot shrink. */ + var page = await DarlingPgWaitReader.GetPgWaitStatsPageAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); /* An empty result is genuinely ambiguous here in a way it is not for SQL Server: it means either no data in the window, or that this server is not a PostgreSQL target at all. Say so, rather than letting a caller read "no waits" as "no waiting". */ - if (rows.Count == 0) + if (page.Rows.Count == 0) { /* Both halves of the old ambiguity are answerable when the store knows the engine (#2532): a SQL Server target gets the dialect answer, a stock PostgreSQL one gets the Aurora-only @@ -67,32 +71,82 @@ capability answer has already ruled out. */ + "PostgreSQL one at all — check list_servers."); } - var totalWaitMs = rows.Sum(r => r.TotalWaitTimeMs); - - /* No Take(limit) — the SQL now applies the cap, so the rows returned ARE the rows asked for. */ - var result = rows.Select(r => new - { - wait_type = r.WaitType, - wait_event = r.WaitEvent, - total_wait_time_ms = Math.Round(r.TotalWaitTimeMs, 1), - waits = r.TotalWaits, - avg_wait_time_ms = Math.Round(r.AvgWaitTimeMs, 3), - /* Share of the window's total wait time. The absolute figure alone does not say whether - an event is the story or a rounding error. */ - pct_of_total_wait = totalWaitMs > 0 ? Math.Round(r.TotalWaitTimeMs / totalWaitMs * 100, 1) : 0, - }); - - return JsonSerializer.Serialize(new - { - server = resolved.ServerName, - hours_back, - total_wait_time_ms = Math.Round(totalWaitMs, 1), - waits = result, - }, McpHelpers.JsonOptions); + return BuildWaitStatsJson(resolved.ServerName, hours_back, page, limit); } catch (Exception ex) { return McpHelpers.Status("error", $"Reading PostgreSQL wait stats failed: {ex.Message}"); } } + + /// + /// The response body, split out so the WIRE SHAPE can be asserted without a live store — the same reason + /// the statement tool's BuildTopQueriesJson is separate. + /// + /// The denominator is the window's, not the page's (#3541 A7). carries + /// WindowTotalWaitTimeMs from the same statement as its rows, and every pct_of_total_wait + /// divides by THAT. The previous shape divided by the sum of the rows fetched, so at limit = 3 the + /// three shares summed to 100% and read as "these three are everything". The page's own sum still + /// travels as returned_wait_time_ms, and the ratio of the two is returned_pct_of_total. + /// + /// holds up to limit + 1 rows; the extra one is the truncation signal + /// and is cut before the projection. + /// + internal static string BuildWaitStatsJson( + string serverName, + int hoursBack, + DarlingPgWaitReader.PgWaitStatsPage page, + int limit) + { + var truncated = page.Rows.Count > limit; + var rows = truncated ? page.Rows.Take(limit).ToList() : page.Rows; + + var windowTotalMs = page.WindowTotalWaitTimeMs; + var returnedMs = rows.Sum(r => r.TotalWaitTimeMs); + + var result = rows.Select(r => new + { + wait_type = r.WaitType, + wait_event = r.WaitEvent, + total_wait_time_ms = Math.Round(r.TotalWaitTimeMs, 1), + waits = r.TotalWaits, + avg_wait_time_ms = Math.Round(r.AvgWaitTimeMs, 3), + /* Share of the WINDOW's total wait time - never of the page's. The absolute figure alone does + not say whether an event is the story or a rounding error; a share of the page could not say + it either, because a page always sums to itself. */ + pct_of_total_wait = windowTotalMs > 0 ? Math.Round(r.TotalWaitTimeMs / windowTotalMs * 100, 1) : 0, + }) + .ToList(); + + return JsonSerializer.Serialize(new + { + server = serverName, + hours_back = hoursBack, + /* #3541 A3 dialect: the page described as a page. No time bounds — the rows are per-event + aggregates over the whole window, so there is no page reach to report, only a cap. */ + wait_events_returned = result.Count, + truncated, + order = "total_wait_time_ms_desc", + /* #3604: the instrument, stated on this surface too so the three PostgreSQL wait tiers read + alike. Constant here rather than looked up: pg_wait_stats is Aurora-gated and reads the + engine's own counters, so nothing else can have fed this table. */ + instrument = PgWaitInstrument.EngineCumulative, + instrument_note = "Aurora's aurora_stat_system_waits(): every wait's count and measured time, " + + "accumulated by the engine since instance start - the finest of the three " + + "PostgreSQL wait instruments and the only one that measures time rather than " + + "estimating it from samples. Stock PostgreSQL targets are served by " + + "get_pg_wait_sampling instead, whose instrument field says whether the " + + "pg_wait_sampling extension or the service-side sampler fed them.", + /* The WINDOW's wait time, across every event that accrued any — the denominator of every + pct_of_total_wait above. NOT the sum of the rows; that is returned_wait_time_ms. */ + total_wait_time_ms = Math.Round(windowTotalMs, 1), + returned_wait_time_ms = Math.Round(returnedMs, 1), + returned_pct_of_total = windowTotalMs > 0 ? Math.Round(returnedMs / windowTotalMs * 100, 1) : 0, + note = "total_wait_time_ms is the WHOLE window's wait time across every event, computed in the " + + "same statement as the rows; each row's pct_of_total_wait divides by it, so the shares on " + + "a page do not sum to 100 unless the page is the whole window (truncated = false). " + + "returned_wait_time_ms is what the rows returned add up to.", + waits = result, + }, McpHelpers.JsonOptions); + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs index 1448af572..a4e28ef48 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPgXminTools.cs @@ -56,7 +56,7 @@ public sealed class DarlingMcpPgXminTools _ => "Unrecognized holder source.", }; - [McpServerTool(Name = "get_pg_xmin_horizon"), Description("Gets what is holding back the PostgreSQL xmin horizon, attributed by cause. Use this whenever dead tuples or table bloat are growing while autovacuum appears to be running normally - that symptom has four unrelated causes which look identical from the outside, and each needs a completely different fix: a long-running or idle-in-transaction session, an abandoned replication slot, a logical slot holding catalog_xmin, a standby feeding back its xmin, or an orphaned prepared transaction. Reports the oldest holder for each source, which one is currently winning, and how persistent each has been across the window, so a chronic holder can be told apart from a query that merely ran long. Also relevant to wraparound risk: a pinned horizon blocks freezing, so an unattended holder here is an upstream cause of the risk get_pg_wraparound_risk measures. Works on any PostgreSQL target.")] + [McpServerTool(Name = "get_pg_xmin_horizon"), Description("Gets what is holding back the PostgreSQL xmin horizon, attributed by cause. Use this whenever dead tuples or table bloat are growing while autovacuum appears to be running normally - that symptom has four unrelated causes which look identical from the outside, and each needs a completely different fix: a long-running or idle-in-transaction session, an abandoned replication slot, a logical slot holding catalog_xmin, a standby feeding back its xmin, or an orphaned prepared transaction. Reports the oldest holder for each source, which one is currently winning, and how persistent each has been across the window, so a chronic holder can be told apart from a query that merely ran long: pct_of_window_winning is the share of EVERY capture in the window (captures_in_window, from the collector's own log - unheld captures included, the same denominator the vacuum-horizon alert uses), not the share of the captures that happened to record this source. Also relevant to wraparound risk: a pinned horizon blocks freezing, so an unattended holder here is an upstream cause of the risk get_pg_wraparound_risk measures. Works on any PostgreSQL target. Zero holders is reported as no_holder only when the collector captured in the window; no holders AND no captures is unavailable, not an all-clear.")] public static async Task GetPgXminHorizon( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -74,6 +74,8 @@ public static async Task GetPgXminHorizon( var now = windowEnd; var rows = await DarlingPgXminReader.GetPgXminHorizonAsync( postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var capturesInWindow = await DarlingPgXminReader.GetXminCapturesInWindowAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); /* Nothing holding the horizon is the HEALTHY answer, and saying so plainly matters more here than for most tools: an operator arrives at this tool BECAUSE bloat is growing, so "no @@ -90,12 +92,29 @@ holding back the xmin horizon" is a confident all-clear about a mechanism that d return gated; } + /* Zero holders is a measurement only if the collector looked (#3541 A12): the collector + stores nothing on an unheld capture, so an empty holder table is ALSO what a collector + that never ran in this window leaves behind. The capture count is the witness. */ + if (capturesInWindow == 0) + { + return JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + status = "unavailable", + captures_in_window = 0, + message = $"No holder rows AND no successful pg_xmin_horizon captures are logged for {resolved.ServerName} in the last {hours_back} hour(s), so this is NOT a report that nothing holds the horizon — the collector did not look (or its collection_log rows are missing). Check get_collection_health for this server before reading the absence as clear.", + }, McpHelpers.JsonOptions); + } + return JsonSerializer.Serialize(new { server = resolved.ServerName, hours_back, status = "no_holder", - finding = "Nothing is holding back the xmin horizon in this window. Vacuum is free to " + captures_in_window = capturesInWindow, + finding = $"Nothing is holding back the xmin horizon in this window: the collector captured " + + $"{capturesInWindow} time(s) and recorded no holder. Vacuum is free to " + "reclaim dead rows, so bloat growth has a different cause — look at whether " + "autovacuum is being triggered at all (per-table thresholds and dead-tuple " + "counts) rather than at whether it is being blocked.", @@ -114,10 +133,18 @@ holding back the xmin horizon" is a confident all-clear about a mechanism that d /* Persistence, not just presence. A source that won nearly every sample is a standing problem someone must own; one that won twice was a query that ran long and finished. */ samples_as_winner = r.SamplesAsWinner, - samples = r.Samples, - pct_of_window_winning = r.Samples > 0 - ? Math.Round((double)r.SamplesAsWinner / r.Samples * 100, 1) - : 0, + /* Captures in which THIS source recorded a holder — its own rows, not the window. Named so it + cannot be read as the window's capture count, which is captures_in_window above. */ + captures_recording_this_source = r.Samples, + /* Over EVERY capture in the window (#3541 A12), not over this source's own rows: the collector + stores nothing on an unheld capture, so dividing by the source's rows made 2 wins in 2 rows + out of 288 captures read as 100% chronic. The same denominator the alert evaluator's + horizon arm fractions over (DarlingPostgresAlertReadAdapter.XminSql). Null, never 0, when + the log holds no captures to divide by; unclamped, so an undercounting log (a skipped + failure-isolated write) shows as a share above 100 rather than being rounded into a lie. */ + pct_of_window_winning = capturesInWindow > 0 + ? Math.Round((double)r.SamplesAsWinner / capturesInWindow * 100, 1) + : (double?)null, remedy = RemedyFor(r.Source), }).ToList(); @@ -128,6 +155,12 @@ holding back the xmin horizon" is a confident all-clear about a mechanism that d server = resolved.ServerName, hours_back, status = "holder_present", + /* The window's denominator: successful pg_xmin_horizon runs logged in the window, from + collection_log — every time the collector LOOKED, held or not. */ + captures_in_window = capturesInWindow, + pct_denominator = capturesInWindow > 0 + ? "pct_of_window_winning = samples_as_winner / captures_in_window; captures_in_window counts the collector's SUCCESS rows in collection_log for this window, so it includes captures that found no holder. It can undercount if a log write was skipped, in which case a share can exceed 100 — it is not clamped." + : "pct_of_window_winning is null: collection_log holds no successful pg_xmin_horizon capture in this window to divide by, though holder rows exist — read samples_as_winner as a count, not a share.", /* Lead with the actionable pair: which cause, and what to do about that cause. */ winning_source = winner.source, winning_holder = winner.holder, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCacheSchedulerTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCacheSchedulerTools.cs index a0797d68b..d55299692 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCacheSchedulerTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCacheSchedulerTools.cs @@ -22,19 +22,38 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// The plan-cache + CPU-scheduler snapshot MCP tools — get_plan_cache_bloat, get_cpu_scheduler_pressure — /// served over Darling's Postgres store. Each tool body mirrors the Dashboard's McpDiagnosticTools / -/// McpSchedulerTools field-for-field (Lite exposes neither, so the Dashboard is the only reference). +/// McpSchedulerTools field-for-field; Lite has since ported both names (Lite/Mcp/McpPlanCacheSchedulerTools), +/// and the two SKUs now share one parameter contract (below). /// Reads flow through — STORED reads of the latest snapshot, no /// live monitored-server hit. The Dashboard's bloat_level (#1410) and pressure_level / /// recommendation (#1410) classifications are reproduced from the reporting-view CASE logic. +/// +/// #3541 A10 — one contract on both SKUs, and the snapshot says when. Both tools take +/// (server_name, hours_back, as_of) with the SAME parameter descriptions as Lite's +/// McpPlanCacheSchedulerTools: hours_back is the span SEARCHED for the newest snapshot, not a +/// span aggregated, and the description says so in those words. get_cpu_scheduler_pressure used to take +/// server_name alone here and (server_name, hours_back, as_of) on Lite — the same tool name +/// with two parameter surfaces, on the one tool whose answer is a CRITICAL/HIGH/MEDIUM/NORMAL verdict. Every +/// payload publishes captured_at (the snapshot's own collection_time) and age_seconds +/// against the window's end, so a verdict computed from a stale row cannot pass as current. /// [McpServerToolType] public sealed class DarlingMcpPlanCacheSchedulerTools { - [McpServerTool(Name = "get_plan_cache_bloat"), Description("Gets plan cache composition showing single-use vs multi-use plans, with a bloat-level classification. High single-use plan counts indicate ad-hoc query bloat consuming buffer pool memory. Consider enabling 'optimize for ad hoc workloads'.")] + /// + /// get_cpu_scheduler_pressure's description, VERBATIM the text Lite's twin carries (#3541 A10): the same + /// tool name described two ways on two servers was half of the drift this lane closed, and a shared const + /// cannot be shared across the two assemblies, so the cross-SKU description census pins the two strings + /// equal instead. Change one, change both. + /// + internal const string CpuSchedulerPressureDescription = + "Gets CPU scheduler pressure from the latest snapshot: runnable task queue depth, worker thread utilization, queued/blocked requests, the collector's pressure warning flags, and the banded pressure_level verdict with its recommendation. Shows whether the server has enough worker threads and if tasks are queuing for CPU time. LATEST IS A TIME: this is the newest scheduler snapshot found within hours_back of as_of, not an aggregate over those hours - captured_at is the instant it was collected and age_seconds its distance from the window's end; the verdict is that instant's, so read age_seconds before reading pressure_level as current."; + + [McpServerTool(Name = "get_plan_cache_bloat"), Description("Gets plan cache composition showing single-use vs multi-use plans, with a bloat-level classification. High single-use plan counts indicate ad-hoc query bloat consuming buffer pool memory. Consider enabling 'optimize for ad hoc workloads'. LATEST IS A TIME: this is the newest plan-cache snapshot found within hours_back of as_of, not an aggregate over those hours - captured_at is the instant the snapshot was collected and age_seconds its distance from the window's end.")] public static async Task GetPlanCacheBloat( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, - [Description("Hours of data to analyze. Default 24.")] int hours_back = 24, + [Description("Hours of history to search for the latest snapshot. Default 24.")] int hours_back = 24, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -61,7 +80,8 @@ public static async Task GetPlanCacheBloat( return JsonSerializer.Serialize(new { server = resolved.ServerName, - collection_time = rows[0].CollectionTime.ToString("o"), + captured_at = rows[0].CollectionTime.ToString("o"), + age_seconds = LatestSnapshotStamp.AgeSeconds(rows[0].CollectionTime, now), summary = new { total_plans = totalPlans, @@ -93,20 +113,27 @@ public static async Task GetPlanCacheBloat( } } - [McpServerTool(Name = "get_cpu_scheduler_pressure"), Description("Gets CPU scheduler pressure: runnable task queue depth, worker thread utilization, and pressure warnings. Shows whether the server has enough worker threads and if tasks are queuing for CPU time.")] + [McpServerTool(Name = "get_cpu_scheduler_pressure"), Description(CpuSchedulerPressureDescription)] public static async Task GetCpuSchedulerPressure( NpgsqlDataSource postgres, - [Description("Server name or display name.")] string? server_name = null) + [Description("Server name or display name.")] string? server_name = null, + [Description("Hours of history to search for the latest snapshot. Default 24.")] int hours_back = 24, + [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); if (error != null) return error; + var validation = McpHelpers.ValidateWindow(hours_back, as_of, out var windowEnd); + if (validation != null) return validation; + try { - var item = await DarlingPlanCacheSchedulerReader.GetCpuSchedulerPressureAsync(postgres, resolved.ServerId); + var now = windowEnd; + var item = await DarlingPlanCacheSchedulerReader.GetCpuSchedulerPressureAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now); if (item == null) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "cpu_scheduler_stats") - ?? McpHelpers.Status("unavailable", "No CPU scheduler data available. The scheduler collector may not have run yet."); + ?? McpHelpers.Status("unavailable", "No CPU scheduler snapshot in the requested time range. The scheduler collector may not have run yet, or its newest snapshot is older than hours_back."); var workerUtilizationPercent = item.MaxWorkersCount > 0 ? Math.Round(item.TotalCurrentWorkersCount * 100.0 / item.MaxWorkersCount, 2) @@ -119,7 +146,8 @@ public static async Task GetCpuSchedulerPressure( return JsonSerializer.Serialize(new { server = resolved.ServerName, - collection_time = item.CollectionTime.ToString("o"), + captured_at = item.CollectionTime.ToString("o"), + age_seconds = LatestSnapshotStamp.AgeSeconds(item.CollectionTime, now), schedulers = item.SchedulerCount, runnable_tasks = item.TotalRunnableTasksCount, avg_runnable_per_scheduler = item.AvgRunnableTasksCount, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs index c7c2391f0..77c65cfae 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPlanCorrectionTools.cs @@ -30,12 +30,12 @@ namespace PerformanceMonitor.Darling.Service.Mcp; public sealed class DarlingMcpPlanCorrectionTools { [McpServerTool(Name = "get_plan_corrections"), Description( - "Gets SQL Server automatic plan correction (APC) activity: the engine's FORCE_LAST_GOOD_PLAN recommendations and actions over the window, plus each database's current automatic-tuning enablement state. Use when a query's plan changed suddenly - APC forcing or unforcing a plan is a first-class explanation - or to check whether automatic tuning is on and actually working (desired vs actual state). Rows come from sys.dm_db_tuning_recommendations captured on a schedule; a recommendation's state moves through Active/Verifying/Success/Reverted as the engine acts. Every timestamp here is UTC, including valid_since / last_refresh / execute_action_initiated_time / revert_action_initiated_time - sys.dm_db_tuning_recommendations reports those four in UTC and they are stored and returned unconverted - so they order correctly against collection_time and against get_query_store_regressions.")] + "Gets SQL Server automatic plan correction (APC) activity: the engine's FORCE_LAST_GOOD_PLAN recommendations and actions over the window, NEWEST CAPTURE FIRST, plus each database's current automatic-tuning enablement state. Use when a query's plan changed suddenly - APC forcing or unforcing a plan is a first-class explanation - or to check whether automatic tuning is on and actually working (desired vs actual state). Rows come from sys.dm_db_tuning_recommendations captured on a schedule, and the collector RE-CAPTURES every open recommendation on every cycle, so the same recommendation appears once per capture and a few open recommendations fill a page fast. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: recommendations_returned is how many rows you got, truncated says the window held more than limit, and oldest_returned_collection_time / newest_returned_collection_time bound the page - under newest-first ordering the oldest stamp IS how far back this read reached, and on a server with open recommendations a week-long request at the default limit reaches back hours, not days. Raise limit or narrow hours_back when truncated is true. automatic_tuning is a latest-snapshot read that ignores the window entirely; its as_of stamps say when. A recommendation's state moves through Active/Verifying/Success/Reverted as the engine acts. Every timestamp here is UTC, including valid_since / last_refresh / execute_action_initiated_time / revert_action_initiated_time - sys.dm_db_tuning_recommendations reports those four in UTC and they are stored and returned unconverted - so they order correctly against collection_time and against get_query_store_regressions.")] public static async Task GetPlanCorrections( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 24.")] int hours_back = 24, - [Description("Maximum recommendation rows. Default 50.")] int limit = 50, + [Description("Maximum recommendation rows to return, newest capture first. Default 50. This is what bounds the page - read truncated to know whether the window held more.")] int limit = 50, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -50,8 +50,13 @@ public static async Task GetPlanCorrections( { var now = windowEnd; var tuning = await DarlingPlanCorrectionReader.GetLatestAutomaticTuningAsync(postgres, resolved.ServerId); + /* #3541 A3: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal. The reader's LIMIT 200 over per-cycle re-captures gave every window the same ~16-hour + reach, and `total_recommendations` published that page as the window's count. */ var rows = await DarlingPlanCorrectionReader.GetPlanCorrectionsAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); + var truncated = rows.Count > limit; + var page = truncated ? rows.Take(limit).ToList() : rows; if (tuning.Count == 0 && rows.Count == 0) { @@ -62,7 +67,7 @@ public static async Task GetPlanCorrections( "that or has no databases with Query Store on."); } - var recommendations = rows.Take(limit).Select(r => new + var recommendations = page.Select(r => new { collection_time = r.CollectionTime.ToString("o"), database_name = r.DatabaseName, @@ -102,7 +107,14 @@ public static async Task GetPlanCorrections( force_last_good_plan_reason = t.Reason, as_of = t.CollectionTime.ToString("o"), }), - total_recommendations = rows.Count, + /* #3541 A3: the page described as a page. Newest-capture-first makes it a contiguous slice of + the window's tail, so the oldest stamp IS the reach; null when the window held no + recommendation rows and only the tuning snapshot answered. */ + recommendations_returned = page.Count, + truncated, + oldest_returned_collection_time = page.Count == 0 ? null : page.Min(r => r.CollectionTime).ToString("o"), + newest_returned_collection_time = page.Count == 0 ? null : page.Max(r => r.CollectionTime).ToString("o"), + order = "collection_time_desc", recommendations, }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPvsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPvsTools.cs index 4a798cc0d..4b8be6c16 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPvsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpPvsTools.cs @@ -31,7 +31,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; public sealed class DarlingMcpPvsTools { [McpServerTool(Name = "get_pvs_stats"), Description( - "Gets the Accelerated Database Recovery (ADR) persistent version store state per database: PVS size and percent-of-database, online-index version store size, aborted transaction count, version-cleaner run state (a start time without an end time means the cleaner is mid-run), and the oldest active/aborted transaction ids. Use when a database's size is growing without table growth, when ADR cleanup looks stuck, or alongside the PVS pressure alert. A large PVS is pinned by long-running or aborted transactions; the id gap shows how far cleanup is behind. Optionally returns the size trend for the top-5 databases over a window. Every timestamp here is UTC, the four cleaner times included - the DMV reports those in the monitored server's local clock and this read de-skews them - so a cleaner time compares directly against as_of.")] + "Gets the Accelerated Database Recovery (ADR) persistent version store state per database: PVS size and percent-of-database, online-index version store size, aborted transaction count, version-cleaner run state (a start time without an end time means the cleaner is mid-run), and the oldest active/aborted transaction ids. Use when a database's size is growing without table growth, when ADR cleanup looks stuck, or alongside the PVS pressure alert. A large PVS is pinned by long-running or aborted transactions; the id gap shows how far cleanup is behind. Optionally returns the size trend for the top-5 databases over a window. Every timestamp here is UTC, the four cleaner times included - the DMV reports those in the monitored server's local clock and this read de-skews them - so a cleaner time compares directly against as_of. pvs_measured says whether the DMV reported a size for that database at all; a measured 0 MB is published as pvs_size_mb 0 and pct_of_database 0.00 (the healthy, fully-cleaned state), and pct_of_database is null only when the numerator was not measured or the denominator is absent, with pct_of_database_reason saying which.")] public static async Task GetPvsStats( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -62,10 +62,17 @@ public static async Task GetPvsStats( database_name = r.DatabaseName, is_adr_on = r.IsAdrOn, pvs_size_mb = r.PvsSizeMb, - /* The SAME denominator the FinOps grid and the pressure alert use, so no surface disagrees. */ - pct_of_database = r.PvsSizeMb is > 0 && r.DatabaseDataSizeMb is > 0 - ? Math.Round(r.PvsSizeMb.Value / r.DatabaseDataSizeMb.Value * 100.0, 2) + /* Whether the DMV reported a size at all (#3541 A12, contract rule 5). A measured 0 MB — the + healthy, fully-cleaned state — used to be indistinguishable from a NULL the collector + could not read: both fell through to pct_of_database = null. */ + pvs_measured = r.PvsSizeMb.HasValue, + /* The SAME denominator the FinOps grid and the pressure alert use, so no surface disagrees. + Any MEASURED size divides — 0 MB of a 100 GB database is 0.00%, a measurement — and only an + unmeasured numerator or an absent/zero denominator yields null, with the reason beside it. */ + pct_of_database = r.PvsSizeMb is { } pvsMb && r.DatabaseDataSizeMb is > 0 + ? Math.Round(pvsMb / r.DatabaseDataSizeMb.Value * 100.0, 2) : (double?)null, + pct_of_database_reason = PctReason(r.PvsSizeMb.HasValue, r.DatabaseDataSizeMb), online_index_version_store_mb = r.OnlineIndexVersionStoreMb, database_data_size_mb = r.DatabaseDataSizeMb, aborted_transaction_count = r.AbortedTransactionCount, @@ -116,4 +123,20 @@ public static async Task GetPvsStats( return McpHelpers.FormatError("get_pvs_stats", ex); } } + + /// + /// Why pct_of_database is null, when it is (#3541 A12): the numerator was not measured, or the + /// denominator was absent or zero. Null when the percent is defined — including a defined 0.00 — so the + /// healthy row carries no note. Lite's twin words it identically. + /// + internal static string? PctReason(bool pvsMeasured, double? databaseDataSizeMb) + { + if (!pvsMeasured) + return "pvs_size_mb was not reported by sys.dm_tran_persistent_version_store_stats in this capture, so the share is unknown — not zero."; + if (databaseDataSizeMb is null) + return "database_data_size_mb was not captured for this database, so there is no denominator — the share is unknown, not zero."; + if (databaseDataSizeMb <= 0) + return "database_data_size_mb is 0, so the share has no denominator — the share is unknown, not zero."; + return null; + } } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs index 572e16c34..794b9a9e2 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs @@ -7,6 +7,7 @@ */ using System; +using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Text.Json; @@ -33,7 +34,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpQueryStoreRegressionTools { - [McpServerTool(Name = "get_query_store_regressions"), Description("Finds queries whose Query Store performance got WORSE, by comparing each (database, query_id) group's averages inside a recent window against its baseline - every capture BEFORE that window. Returns baseline vs recent duration, CPU and logical reads with the regression percent for each, the execution-count-weighted extra duration (the ranking key: a 5 ms regression executed a million times outranks a 5-second one executed twice), the plan counts on both sides, and a duration-driven severity band. get_query_store_top answers what is EXPENSIVE; the most expensive query is usually the one that always was. This answers what CHANGED. Rows are kept only where average CPU regressed by more than 25%.")] + [McpServerTool(Name = "get_query_store_regressions"), Description("Finds queries whose Query Store performance got WORSE, by comparing each (database, query_id) group's averages inside a recent window against its baseline - every capture BEFORE that window. Returns baseline vs recent duration, CPU and logical reads with the regression percent for each, the execution-count-weighted extra duration (the ranking key: a 5 ms regression executed a million times outranks a 5-second one executed twice), the plan counts on both sides, and a duration-driven severity band. get_query_store_top answers what is EXPENSIVE; the most expensive query is usually the one that always was. This answers what CHANGED. Rows are kept only where average CPU regressed by more than 25%. A regression percent whose BASELINE side is 0 has no denominator and is returned as null, with the reason under undefined_percents - never as 0, which would read as no change when the truth is the largest possible one; compare the two absolute figures instead. The ranking key is the absolute, execution-weighted duration delta, which exists whether or not a ratio does, so a null percent never sorts as 0. severity is banded from the duration percent and is null when that percent is.")] public static async Task GetQueryStoreRegressions( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -87,7 +88,10 @@ the recent window bigger AND the baseline shorter. { database_name = r.DatabaseName, query_id = r.QueryId, - severity = r.Severity, + /* Banded from the duration percent by the TVF's CASE, whose ELSE is 'LOW' — which for a + row with NO duration ratio is a verdict about a number that does not exist. Null there + (#3541 A12); the SQL's band is kept verbatim for the viewer it is shared with. */ + severity = r.DurationRegressionPercent is null ? null : r.Severity, baseline_duration_ms = r.BaselineDurationMs, recent_duration_ms = r.RecentDurationMs, duration_regression_percent = r.DurationRegressionPercent, @@ -97,8 +101,12 @@ the recent window bigger AND the baseline shorter. baseline_reads = r.BaselineReads, recent_reads = r.RecentReads, io_regression_percent = r.IoRegressionPercent, + /* Null percents, and why (#3541 A12): a 0 baseline has no ratio, and the reader used to + publish that as 0 — "no change" — for the row that changed the most. */ + undefined_percents = UndefinedPercentNotes(r), /* The ranking key, and the one number that says whether this regression MATTERS: a - 5 ms regression executed a million times outranks a 5-second one executed twice. */ + 5 ms regression executed a million times outranks a 5-second one executed twice. It is + an absolute delta, so it exists for every row and a null ratio never sorts as 0. */ additional_duration_ms = r.AdditionalDurationMs, baseline_exec_count = r.BaselineExecCount, recent_exec_count = r.RecentExecCount, @@ -117,6 +125,27 @@ 5 ms regression executed a million times outranks a 5-second one executed twice. } } + + /// + /// Which of a row's three regression percents are undefined, and why (#3541 A12, contract rule 5). Each + /// percent divides through NULLIF(baseline, 0), so a NULL means the baseline side was 0 — there is + /// no denominator, not no change — and the caller is pointed at the absolute pair it can still compare. + /// Null when every percent is defined, so the common row carries no noise. Lite's twin builds the same + /// sentences. + /// + private static List? UndefinedPercentNotes(DarlingQueryStoreRegressionReader.RegressionRow r) + { + List? notes = null; + void Note(string field, string baseline, string recent) + => (notes ??= new List()).Add( + $"{field} is null: no_baseline — {baseline} is 0, so the ratio has no denominator; this is NOT 0% change. Compare {baseline} to {recent} directly."); + + if (r.DurationRegressionPercent is null) Note("duration_regression_percent", "baseline_duration_ms", "recent_duration_ms"); + if (r.CpuRegressionPercent is null) Note("cpu_regression_percent", "baseline_cpu_ms", "recent_cpu_ms"); + if (r.IoRegressionPercent is null) Note("io_regression_percent", "baseline_reads", "recent_reads"); + return notes; + } + /// /// What zero regressions actually means, which is four different things. /// Only ONE of them is good news, and the other three all look identical to it in a bare empty diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpReadParameters.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpReadParameters.cs index 367e07fc1..641a729aa 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpReadParameters.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpReadParameters.cs @@ -33,6 +33,10 @@ public static void AddText(NpgsqlCommand command, string value) => public static void AddNullableText(NpgsqlCommand command, string? value) => command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = (object?)value ?? DBNull.Value }); + /// Binds a boolean flag — a read's NOT $N::boolean OR ... "filter off" branch (#3541 A13). + public static void AddBoolean(NpgsqlCommand command, bool value) => + command.Parameters.Add(new NpgsqlParameter { TypedValue = value }); + /// Binds a naive-UTC timestamp (Kind=Unspecified → the store's timestamp columns). public static void AddTimestamp(NpgsqlCommand command, DateTime value) => command.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(value, DateTimeKind.Unspecified) }); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs index 349861835..f9b74e39f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpServerAdminTools.cs @@ -50,8 +50,15 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// auth store no secret; a service principal stores its client secret exactly like a SQL password; the INTERACTIVE /// Entra modes (MFA / device-code / default-credential) are rejected (status:"invalid") — they need a broker /// or a signed-in user and cannot run headless, whereas ServicePrincipal and ManagedIdentity are non-interactive -/// and supported (#3484). The whole call returns {added, skipped, failed, results:[...]}. remove_server -/// resolves a name through the SAME the read tools use and DELETEs the +/// and supported (#3484). A server whose probed connection lands in a database another registration already +/// covers is refused as status:"collides" (#2280). The whole call returns +/// {requested, added, skipped, collided, failed, results:[...]}, and the four counters SUM to +/// requested — every per-row status is mapped to exactly one of them by +/// (#3541 A14: collides used to land in no counter, so a batch with a collided server summarised as +/// failed: 0 and the server went silently unmonitored under a clean summary). remove_server reads the +/// SAME servers registry the read tools resolve against, but with a stricter rule than theirs: an exact +/// match, or a partial match ONLY when it is unique — an ambiguous partial is refused with the candidates named, +/// because a first-wins partial on a DELETE removes whichever sibling sorts first. It then DELETEs the /// config.config_monitored_servers row. /// /// Security. These tools connect (like every MCP tool) as the least-privilege mcp role, granted @@ -97,13 +104,18 @@ private static Task DefaultProbeAsync(MonitoredServer ser "status \"connection_failed\" and does NOT stop the rest of the batch. The INTERACTIVE Microsoft Entra modes " + "(MFA / device-code / default-credential) are rejected (status \"invalid\") — they need a broker or a " + "signed-in user and cannot run headless; ServicePrincipal and ManagedIdentity are non-interactive and are " + - "supported. A SQL password or service-principal client secret is encrypted at rest (DPAPI, the service " + - "identity) and is never " + - "returned. Returns {added:N, skipped:N, failed:N, results:[{server, status:\"added\"|\"duplicate\"|" + - "\"connection_failed\"|\"invalid\", detail}]}, where an added server's detail reports what the probe found " + - "— for a PostgreSQL target that includes writer-vs-reader, Aurora-vs-not, and how many of the PostgreSQL " + - "collectors apply to it. NOTE: the password travels to this endpoint in the request; " + - "on a LAN use the documented TLS reverse proxy.")] + "supported. A server whose probed connection lands in a database that ANOTHER monitored server already " + + "covers (it names one database but connects to a different one, e.g. a wrong Initial Catalog) is refused as " + + "status \"collides\" — adding it would store one database's history under two identities and alert twice. " + + "A SQL password or service-principal client secret is encrypted at rest (DPAPI, the service identity) and " + + "is never returned. Returns {requested:N, added:N, skipped:N, collided:N, failed:N, results:[{server, " + + "status:\"added\"|\"duplicate\"|\"collides\"|\"connection_failed\"|\"invalid\", detail}]}. requested is " + + "the number of entries you sent and the four counters SUM to it — every entry lands in exactly one: " + + "\"added\" → added, \"duplicate\" → skipped, \"collides\" → collided, \"connection_failed\" and " + + "\"invalid\" → failed. Only added servers are monitored; read the other three counters before treating " + + "the batch as done. An added server's detail reports what the probe found — for a PostgreSQL target that " + + "includes writer-vs-reader, Aurora-vs-not, and how many of the PostgreSQL collectors apply to it. NOTE: the " + + "password travels to this endpoint in the request; on a LAN use the documented TLS reverse proxy.")] public static Task AddServers( NpgsqlDataSource postgres, [Description("A JSON ARRAY of server objects to add (see the tool description for the per-object fields), e.g. [{\"host\":\"sql01\",\"auth\":\"SQL\",\"username\":\"monitor\",\"password\":\"...\",\"encrypt_mode\":\"Mandatory\",\"trust_server_certificate\":true},{\"host\":\"aurora.cluster-abc.us-east-1.rds.amazonaws.com\",\"engine\":\"postgres\",\"auth\":\"SQL\",\"username\":\"darling_monitor\",\"password\":\"...\",\"trust_server_certificate\":true}].")] string servers_json) => @@ -157,7 +169,7 @@ is recorded and the batch CONTINUES — one unreachable server never aborts the var probeResult = await probe(entry.ProbeConfig, cancellationToken); if (!probeResult.Success) { - results.Add(new ServerResult(entry.Order, entry.DisplayName, "connection_failed", + results.Add(new ServerResult(entry.Order, entry.DisplayName, AddStatus.ConnectionFailed, string.IsNullOrWhiteSpace(probeResult.Error) ? "Could not connect to the server." : $"Could not connect: {probeResult.Error}")); @@ -176,7 +188,7 @@ reported at every connect by #2277's tripwire. var collision = ActualIdentityCollision(entry, probeResult.ConnectedDatabase, claimed); if (collision is not null) { - results.Add(new ServerResult(entry.Order, entry.DisplayName, "collides", collision)); + results.Add(new ServerResult(entry.Order, entry.DisplayName, AddStatus.Collides, collision)); continue; } @@ -185,7 +197,7 @@ reported at every connect by #2277's tripwire. leaves this method — it is not logged, not echoed in a result. */ var encryptedPassword = ProtectPasswordForStorage(entry.PlaintextPassword); await InsertServerAsync(postgres, entry, encryptedPassword, cancellationToken); - results.Add(new ServerResult(entry.Order, entry.DisplayName, "added", DescribeProbe(probeResult))); + results.Add(new ServerResult(entry.Order, entry.DisplayName, AddStatus.Added, DescribeProbe(probeResult))); } return Aggregate(results); @@ -197,15 +209,19 @@ reported at every connect by #2277's tripwire. } [McpServerTool(Name = "remove_server"), Description( - "Removes a monitored SQL Server from the fleet by name (the display name or address, resolved the same way " + - "the read tools resolve server_name — exact match first, then partial, against the storage name and the " + - "display name). Deletes the server's definition from the central monitoring store; the running service " + - "drops it from its collection set within one sweep. Already-collected historical data is NOT deleted. " + - "Returns {status:\"removed\", server} on success, or {status:\"not_found\", ...} when no monitored server " + - "matches the name.")] + "Removes a monitored server from the fleet by name (its display name or storage name / address, as " + + "list_servers reports them). Matching is exact first (case-insensitive, against the storage name and the " + + "display name); a PARTIAL match is honored ONLY when exactly one registered server contains the text. When " + + "the name is ambiguous — an exact name shared by two registrations, or a fragment such as \"-01\" that " + + "several servers contain — NOTHING is deleted and the response is {status:\"ambiguous\", candidates:[{server, " + + "display_name}], message}; re-issue with one candidate's full name. Deletes the server's definition from the " + + "central monitoring store; the running service drops it from its collection set within one sweep. " + + "Already-collected historical data is NOT deleted. Returns {status:\"removed\", server, matched_by:\"exact\"|" + + "\"partial\"} on success, {status:\"ambiguous\", ...} as above, or {status:\"not_found\", ...} when no " + + "registered server matches the name (the message lists the servers that are registered).")] public static async Task RemoveServer( NpgsqlDataSource postgres, - [Description("The name of the monitored server to remove — its display name or address (as list_servers / get_alert_history report it).")] string server_name) + [Description("The name of the monitored server to remove — its display name or storage name / address (as list_servers / get_alert_history report it). A partial name is accepted only when it matches exactly one server.")] string server_name) { try { @@ -214,22 +230,46 @@ public static async Task RemoveServer( return Outcome("invalid", "server_name is required."); } - /* Resolve through the SAME resolver the read tools use, against the servers registry (the id it returns - is the shared-identity server_id that keys config_monitored_servers). A miss returns the resolver's - available-servers listing — surfaced as not_found here. */ - var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); - if (error != null) + /* The SAME registry rows the read tools resolve against (the id is the shared-identity server_id that + keys config_monitored_servers), but NOT the same rule. The read resolver's first-wins partial match + is the right convenience for a read — an agent that lands on the wrong sibling sees its name in the + payload and re-asks. On a DELETE the payload IS the damage: "-01" against "-01"/"-02" removed + whichever sorted first, and said so only after the fact (#3541 A14). So the partial match survives, + as documented, but only when it is UNIQUE; anything else is refused with the candidates named. */ + var servers = await DarlingServerResolver.LoadEnabledAsync(postgres); + var target = ResolveForRemoval(servers, server_name); + + if (target.Candidates.Count == 0) { - return Outcome("not_found", error); + /* No exact and no partial match at all: the resolver's own miss message (the available-servers + listing, plus the #2339 peer disclosure) is the right answer here too, and with zero candidates + it cannot resolve to anything, so reusing it cannot pick a server this method declined to. */ + var (_, missMessage) = DarlingServerResolver.ResolveOrError(servers, server_name); + return Outcome("not_found", missMessage ?? $"Could not resolve server '{server_name}'."); } + if (target.Candidates.Count > 1) + { + return JsonSerializer.Serialize(new + { + status = "ambiguous", + message = $"'{server_name}' matches {target.Candidates.Count} registered servers " + + $"({(target.MatchedBy == "exact" ? "the same name on more than one registration" : "as a partial name")}); " + + "nothing was removed. Re-issue remove_server with ONE candidate's full name.", + matched_by = target.MatchedBy, + candidates = target.Candidates.Select(c => new { server = c.ServerName, display_name = c.DisplayName }), + }, McpHelpers.JsonOptions); + } + + var resolved = target.Candidates[0]; + await using var command = postgres.CreateCommand("DELETE FROM config_monitored_servers WHERE server_id = $1"); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; command.Parameters.Add(new NpgsqlParameter { TypedValue = resolved.ServerId }); var affected = await command.ExecuteNonQueryAsync(); return affected > 0 - ? JsonSerializer.Serialize(new { status = "removed", server = resolved.ServerName }, McpHelpers.JsonOptions) + ? JsonSerializer.Serialize(new { status = "removed", server = resolved.ServerName, matched_by = target.MatchedBy }, McpHelpers.JsonOptions) : Outcome("not_found", $"'{resolved.ServerName}' is registered but has no monitored-server definition to remove (it may have been added only via darling.json, or already removed)."); } @@ -250,10 +290,116 @@ public static async Task RemoveServer( internal sealed record ParsedServerEntry( int Order, string DisplayName, string StorageKey, MonitoredServer ProbeConfig, string? PlaintextPassword); - /// One per-server outcome (added / duplicate / connection_failed / invalid) - /// with a human-readable ; restores input order in the aggregate. + /// One per-server outcome (a value) with a human-readable ; + /// restores input order in the aggregate. internal sealed record ServerResult(int Order, string Server, string Status, string Detail); + /// + /// The per-row status vocabulary of add_servers — the five outcomes an entry can have, as + /// constants so a new one cannot be introduced as a bare literal at a result site without also being placed in + /// (the test census walks these fields and demands each has a counter). + /// + internal static class AddStatus + { + /// Probed, reachable, INSERTed — the server is now monitored. + public const string Added = "added"; + + /// A case-variant or exact duplicate of an existing server or an earlier entry in the batch; skipped + /// without a probe (#1549). + public const string Duplicate = "duplicate"; + + /// Probed, but its connection reached a database another registration already covers (#2280); NOT + /// added. + public const string Collides = "collides"; + + /// The in-process probe could not connect; NOT added, the batch continued. + public const string ConnectionFailed = "connection_failed"; + + /// Structurally invalid (a bad field, an unsupported auth mode); NOT added, never probed. + public const string Invalid = "invalid"; + } + + /// + /// Which summary counter each per-row status is counted under — the whole of the "writes report what happened" + /// contract for this tool (#3541 A14). Every status maps to exactly one counter and every result is counted + /// once, so added + skipped + collided + failed == requested by construction rather than by luck. + /// + /// The defect this replaces: the counters were three ad-hoc Count(...) filters naming four of the + /// five statuses, and the fifth — , added by #2280 after the counters were + /// written — fell into none of them. A batch of three with one collision summarised as + /// {added: 2, skipped: 0, failed: 0}: an agent reading the summary saw a clean run, and the collided + /// server was silently not monitored. A map the aggregator REFUSES to serialize without makes the next new + /// status a hard error at the first call instead of a silent hole. + /// + /// collided is its own counter rather than a kind of failed because the remedy differs: + /// a failed entry is retried after fixing the connection or the fields; a collided one must NOT be retried as + /// sent — it needs a different Initial Catalog or no registration at all, and folding it into failed + /// would invite exactly the retry the refusal exists to prevent. + /// + internal static readonly IReadOnlyDictionary CounterOfStatus = new Dictionary(StringComparer.Ordinal) + { + [AddStatus.Added] = "added", + [AddStatus.Duplicate] = "skipped", + [AddStatus.Collides] = "collided", + [AddStatus.ConnectionFailed] = "failed", + [AddStatus.Invalid] = "failed", + }; + + /// + /// The outcome of matching a remove_server name against the registry: the rows it matched and HOW. + /// Zero candidates is a miss, one is the row to delete, more than one is a refusal. + /// + internal sealed record RemovalTarget(IReadOnlyList Candidates, string MatchedBy); + + /// + /// The matching rule for a DELETE, over the same registry rows the read resolver uses: every exact match + /// (storage name OR display name, case-insensitive, trimmed) if there are any; otherwise every partial + /// (Contains) match. The CALLER decides what a count other than one means — this only refuses to + /// choose among equals. + /// + /// Why not the read resolver's rule. is first-wins on a partial + /// match, ordered by storage name. For a read that is a convenience: the answer names the server it resolved + /// to, and a caller who meant the other one re-asks having lost nothing. For a delete the same rule removed + /// -01 when the caller typed -01 meaning -01, and removed it just the same when the caller + /// typed a fragment that -01 and -02 both contain — a coin the caller did not know was being + /// flipped. The read tools keep their rule; this write does not borrow it. + /// + /// Why partial matching survives at all. The description has promised it since the tool shipped + /// ("resolved the same way the read tools resolve server_name"), display names are what an operator knows a + /// server by, and a unique partial is unambiguous — refusing it would be refusing something the tool CAN + /// honor. The rule the contract asks for is "refuse what you cannot honor", and what cannot be honored here is + /// a choice, not a fragment. + /// + /// Exact matches are collected, not first-taken. display_name is not unique in the registry, + /// so two registrations can share one display name exactly; picking the first would be the same coin under a + /// better-looking name. Two rows matching exactly is reported as ambiguous with matched_by: "exact", and + /// the caller disambiguates on the storage name, which IS unique. + /// + internal static RemovalTarget ResolveForRemoval(IReadOnlyList servers, string serverName) + { + var name = (serverName ?? string.Empty).Trim(); + if (name.Length == 0) + { + return new RemovalTarget(Array.Empty(), "none"); + } + + var exact = servers + .Where(s => string.Equals(s.ServerName, name, StringComparison.OrdinalIgnoreCase) + || string.Equals(s.DisplayName, name, StringComparison.OrdinalIgnoreCase)) + .ToList(); + if (exact.Count > 0) + { + return new RemovalTarget(exact, "exact"); + } + + var partial = servers + .Where(s => s.ServerName.Contains(name, StringComparison.OrdinalIgnoreCase) + || (s.DisplayName?.Contains(name, StringComparison.OrdinalIgnoreCase) ?? false)) + .ToList(); + + return new RemovalTarget(partial, partial.Count == 0 ? "none" : "partial"); + } + /// /// PURE structural validation of the servers_json request — no store, no probe, no crypto — so the /// validate-before-write behavior is unit-testable without a live SQL Server. Returns the structurally-valid @@ -311,13 +457,13 @@ private static (ParsedServerEntry? Entry, ServerResult? Result) ParseEntry(int i { if (node is not JsonObject obj) { - return (null, new ServerResult(index, $"(entry {index + 1})", "invalid", "Each entry must be a JSON object.")); + return (null, new ServerResult(index, $"(entry {index + 1})", AddStatus.Invalid, "Each entry must be a JSON object.")); } var host = TryGetString(obj, "host"); var label = string.IsNullOrWhiteSpace(host) ? $"(entry {index + 1})" : host!.Trim(); - ServerResult Invalid(string message) => new(index, label, "invalid", message); + ServerResult Invalid(string message) => new(index, label, AddStatus.Invalid, message); if (string.IsNullOrWhiteSpace(host)) { @@ -541,7 +687,7 @@ internal static (List Ready, List Duplicates) P } else { - duplicates.Add(new ServerResult(entry.Order, entry.DisplayName, "duplicate", + duplicates.Add(new ServerResult(entry.Order, entry.DisplayName, AddStatus.Duplicate, "Already monitored (or a duplicate of an earlier entry in this batch); skipped.")); } } @@ -654,15 +800,45 @@ private static async Task InsertServerAsync( return DarlingSecrets.Protect(plaintextPassword); } - /// Builds the {added, skipped, failed, results} envelope, results in input order. - private static string Aggregate(List results) + /// + /// Builds the {requested, added, skipped, collided, failed, results} envelope, results in input order, + /// every result counted under the ONE counter names for its status. + /// + /// A status the map does not know is thrown on, not dropped: the caller's catch turns it into the tool's + /// error envelope, which is a loud wrong answer where the old shape gave a quiet one. It cannot fire in + /// production while the census test holds (every constant is mapped), and if a future + /// status is added as a literal and the test is skipped, the first real call says so instead of summarising + /// the batch short. + /// + internal static string Aggregate(List results) { var ordered = results.OrderBy(r => r.Order).ToList(); + var counters = new Dictionary(StringComparer.Ordinal) + { + ["added"] = 0, + ["skipped"] = 0, + ["collided"] = 0, + ["failed"] = 0, + }; + + foreach (var result in ordered) + { + if (!CounterOfStatus.TryGetValue(result.Status, out var counter)) + { + throw new InvalidOperationException( + $"add_servers produced status '{result.Status}' for '{result.Server}', which no summary counter accounts for."); + } + + counters[counter]++; + } + return JsonSerializer.Serialize(new { - added = ordered.Count(r => r.Status == "added"), - skipped = ordered.Count(r => r.Status == "duplicate"), - failed = ordered.Count(r => r.Status is "connection_failed" or "invalid"), + requested = ordered.Count, + added = counters["added"], + skipped = counters["skipped"], + collided = counters["collided"], + failed = counters["failed"], results = ordered.Select(r => new { server = r.Server, status = r.Status, detail = r.Detail }), }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpSessionTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpSessionTools.cs index af2f78bc7..660eedbb9 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpSessionTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpSessionTools.cs @@ -87,14 +87,14 @@ public static async Task GetSessionStats( } } - [McpServerTool(Name = "get_active_queries"), Description("Gets active query snapshots captured from sys.dm_exec_requests. Shows what queries were running at each collection point: session ID, query text, wait type, CPU time, elapsed time, blocking info, DOP, and memory grants. Use hours_back to look at a specific time window — critical for finding what was running during a CPU spike or blocking event.")] + [McpServerTool(Name = "get_active_queries"), Description("Gets active query snapshots captured from sys.dm_exec_requests. Shows what queries were running at each collection point: session ID, query text, wait type, CPU time, elapsed time, blocking info, DOP, and memory grants. Use hours_back to look at a specific time window — critical for finding what was running during a CPU spike or blocking event. EVERY FILTER IS PART OF THE QUERY: database_name and blocking_only are applied in SQL before the page is cut, total_snapshots is the count of snapshot rows in the window that pass your filters, snapshots_returned is how many you got, and truncated says the filtered population held more than limit — raise limit or narrow hours_back when it is true (NEWEST CAPTURE FIRST, highest CPU first within a capture; oldest_returned_collection_time / newest_returned_collection_time bound the page). HEAD BLOCKERS ARE NEVER STRIPPED: a session another row in the same capture names as its blocker is on the page whatever its text (including a WAITFOR shell holding locks), flagged is_head_blocker. A victim whose blocker is NOT on the page says why in blocker_not_shown: not_captured (the blocker held no running request at that capture — an idle open transaction is the classic case; get_blocking has its input buffer from the blocked-process report), filtered (your database_name filter excluded it), or past_page (it is in the filtered population but beyond limit).")] public static async Task GetActiveQueries( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of data to retrieve. Default 1.")] int hours_back = 1, - [Description("Filter to a specific database.")] string? database_name = null, - [Description("Show only queries involved in blocking (blocking_session_id > 0 or is a head blocker).")] bool blocking_only = false, - [Description("Maximum number of rows to return. Default 50.")] int limit = 50, + [Description("Filter to a specific database. Applied in SQL; a head blocker in ANOTHER database is then not on the page, and its victims say blocker_not_shown = filtered.")] string? database_name = null, + [Description("Show only queries involved in blocking: rows with blocking_session_id > 0, plus the head blockers those rows name in the same capture. Applied in SQL, so total_snapshots counts the blocking population and truncated is measured against it.")] bool blocking_only = false, + [Description("Maximum number of rows to return. Default 50. The page is bounded by limit, not by hours_back — truncated says whether the filtered window held more.")] int limit = 50, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -108,22 +108,40 @@ public static async Task GetActiveQueries( try { var now = windowEnd; - var rows = await DarlingSessionReader.GetActiveQueriesAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var filter = string.IsNullOrWhiteSpace(database_name) ? null : database_name.Trim(); + + /* #3541 A13: the filters ride INTO the read (see ActiveQueriesSql), and the read is asked for one + row past the cap so truncation is OBSERVED on the filtered population rather than inferred from + the page. total_snapshots is the SQL's COUNT(*) OVER () of that same population — the number + used to be rows.Count of an unfiltered window read, a different population from the rows. */ + var page = await DarlingSessionReader.GetActiveQueriesAsync( + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1, filter, blocking_only); + var rows = page.Rows; + if (rows.Count == 0) + { + /* A filtered miss is not a collection miss: with the filters in the query, an empty page under + database_name or blocking_only means the window held no snapshot matching them. */ + if (filter != null || blocking_only) + { + return McpHelpers.Status( + "empty", + $"No active query snapshots on {resolved.ServerName} in the last {hours_back} hour(s) matched " + + DescribeActiveQueryFilters(filter, blocking_only) + + ". The filters were applied in SQL over the whole window, so unfiltered snapshots may well exist — drop them to see what the window holds."); + } + return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "query_snapshots") ?? McpHelpers.Status("empty", "No active query snapshots found in the requested time range."); + } - IEnumerable filtered = rows; - - if (!string.IsNullOrEmpty(database_name)) - filtered = filtered.Where(r => (r.DatabaseName ?? "").Equals(database_name, StringComparison.OrdinalIgnoreCase)); + var truncated = rows.Count > limit; + var shown = truncated ? rows.GetRange(0, limit) : rows; - if (blocking_only) - filtered = filtered.Where(r => r.BlockingSessionId > 0 - || rows.Any(other => other.BlockingSessionId == r.SessionId)); + /* The page's own (capture, session) pairs, so a victim can say whether its blocker made the page. */ + var onPage = new HashSet<(DateTime, int)>(shown.Select(r => (r.CollectionTime, r.SessionId))); - var result = filtered.Take(limit).Select(r => new + var result = shown.Select(r => new { collection_time = r.CollectionTime.ToString("o"), session_id = r.SessionId, @@ -138,6 +156,10 @@ public static async Task GetActiveQueries( wait_type = string.IsNullOrEmpty(r.WaitType) ? null : r.WaitType, wait_time_ms = r.WaitTimeMs > 0 ? r.WaitTimeMs : (long?)null, blocking_session_id = r.BlockingSessionId > 0 ? r.BlockingSessionId : (int?)null, + /* #3541 A13: the two blocking disclosures. is_head_blocker is why a WAITFOR row can be here; + blocker_not_shown names the ONE reason a victim's blocker is not, or is null when it is. */ + is_head_blocker = r.IsHeadBlocker ? true : (bool?)null, + blocker_not_shown = BlockerNotShown(r, onPage), dop = r.Dop > 0 ? r.Dop : (int?)null, parallel_worker_count = r.ParallelWorkerCount > 0 ? r.ParallelWorkerCount : (int?)null, granted_query_memory_gb = r.GrantedQueryMemoryGb > 0 ? r.GrantedQueryMemoryGb : (double?)null, @@ -153,8 +175,18 @@ public static async Task GetActiveQueries( { server = resolved.ServerName, hours_back, - total_snapshots = rows.Count, - shown = result.Count, + filters_applied = new + { + database_name = filter, + blocking_only, + }, + /* The FILTERED population's size, computed in SQL on the same statement as the rows. */ + total_snapshots = page.PopulationCount, + snapshots_returned = result.Count, + truncated, + order = "collection_time_desc", + oldest_returned_collection_time = shown[^1].CollectionTime.ToString("o"), + newest_returned_collection_time = shown[0].CollectionTime.ToString("o"), queries = result }, McpHelpers.JsonOptions); } @@ -164,12 +196,39 @@ public static async Task GetActiveQueries( } } - [McpServerTool(Name = "get_waiting_tasks"), Description("Gets recently captured waiting tasks — queries that were actively waiting on a resource at collection time. Shows session ID, wait type, duration, blocking session, and database. Complements get_wait_stats by showing individual waiting queries rather than aggregated stats.")] + /// + /// The one reason a victim's head blocker is not on the page, or null when it is (or the row is not a + /// victim). Ordered from the reader's facts outward: never captured (no running request — the idle + /// open-transaction case) beats filtered beats past the page, because each later reason presupposes the + /// earlier one did not apply. + /// + private static string? BlockerNotShown(DarlingSessionReader.ActiveQueryRow row, HashSet<(DateTime, int)> onPage) + { + if (row.BlockingSessionId <= 0) + return null; + if (!row.BlockerInCapture) + return "not_captured"; + if (!row.BlockerInPopulation) + return "filtered"; + return onPage.Contains((row.CollectionTime, row.BlockingSessionId)) ? null : "past_page"; + } + + /// Names the active filters for the filtered-miss sentence, in the caller's own vocabulary. + private static string DescribeActiveQueryFilters(string? databaseName, bool blockingOnly) + { + if (databaseName != null && blockingOnly) + return $"database_name '{databaseName}' with blocking_only"; + if (databaseName != null) + return $"database_name '{databaseName}'"; + return "blocking_only"; + } + + [McpServerTool(Name = "get_waiting_tasks"), Description("Gets recently captured waiting tasks — queries that were actively waiting on a resource at collection time — NEWEST CAPTURE FIRST, longest wait first within a capture. Shows session ID, wait type, duration, blocking session, and database. Complements get_wait_stats by showing individual waiting queries rather than aggregated stats. THE PAGE IS BOUNDED BY limit, NOT BY hours_back: tasks_returned is how many rows you got, truncated says the window held more than limit, and oldest_returned_collection_time / newest_returned_collection_time bound the page — under newest-first ordering the oldest stamp IS how far back this read reached, and one busy capture can fill the whole page by itself. Raise limit or narrow hours_back when truncated is true.")] public static async Task GetWaitingTasks( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of history. Default 1.")] int hours_back = 1, - [Description("Maximum rows. Default 30.")] int limit = 30, + [Description("Maximum rows to return, newest capture first. Default 30. This is what bounds the page — read truncated to know whether the window held more.")] int limit = 30, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, server_name); @@ -183,13 +242,18 @@ public static async Task GetWaitingTasks( try { var now = windowEnd; + /* #3541 A3: the caller's limit + 1 as the fetch, the extra row as the observed truncation + signal. The reader's LIMIT 500 was invisible, and the envelope stated no bound at all. */ var rows = await DarlingSessionReader.GetWaitingTasksAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + postgres, resolved.ServerId, now.AddHours(-hours_back), now, limit + 1); if (rows.Count == 0) return await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "waiting_tasks") ?? McpHelpers.Status("empty", "No waiting tasks captured in the specified time range."); - var result = rows.Take(limit).Select(r => new + var truncated = rows.Count > limit; + var page = truncated ? rows.Take(limit).ToList() : rows; + + var result = page.Select(r => new { session_id = r.SessionId, wait_type = r.WaitType, @@ -203,6 +267,14 @@ public static async Task GetWaitingTasks( return JsonSerializer.Serialize(new { server = resolved.ServerName, + /* #3541 A3: the envelope was bare — server and rows, no window, no count, no bound. Now the + span requested, the page described as a page, and the span the page covers. */ + hours_back, + tasks_returned = page.Count, + truncated, + oldest_returned_collection_time = page.Min(r => r.CollectionTime).ToString("o"), + newest_returned_collection_time = page.Max(r => r.CollectionTime).ToString("o"), + order = "collection_time_desc", tasks = result }, McpHelpers.JsonOptions); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs index 34b8bb99b..5e6bbfc1c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -22,9 +23,13 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// /// The store self-metrics MCP surface (#2068) — "how fast is the monitoring store growing and what's /// driving it" as a query instead of an expedition. Reads the series the hourly StoreSelfMetrics -/// sweep persists: the latest size/compression snapshot per object (each hypertable, each payload -/// dimension table, the whole store) plus the daily series for the window, with the whole-store daily -/// growth and the derived per-server ingest rate — the number onboarding N servers multiplies. Store-level +/// sweep persists: the latest size/compression snapshot per object (each hypertable, each continuous +/// aggregate, each payload dimension table, each named plain table, the two catch-all rows, the whole +/// store) plus the daily series for the window, with the whole-store daily growth and the derived +/// per-server ingest rate — the number onboarding N servers multiplies. Since #3582 it also states its own +/// coverage: how much of pg_database_size the named rows account for, how much sits in relations +/// nobody named, and whether the whole reconciles — because on the largest production store the previous +/// inventory answered "here is the store" for 38% of it and a growth investigation trusted it. Store-level /// by nature, so unlike almost every other read tool it takes no server_name: the store is the /// server. /// @@ -36,7 +41,7 @@ public sealed class DarlingMcpStoreMetricsTools public const int MaxDaysBack = StoreSelfMetrics.RetentionDays; [McpServerTool(Name = "get_store_metrics"), Description( - "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot: per-hypertable total size, pre/post-compression bytes and chunk count; the query-text and query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; the whole store's size with the enabled-server count; and one row per TimescaleDB background job (CAGG refresh, compression, retention) with its last run duration, schedule interval, duration-vs-cadence percent, and run/failure totals — the jobs whose runtimes scale with fleet size. Returns the latest snapshot per object plus a daily series over the window, with the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth / enabled servers). Each daily point is that day's LAST snapshot, never its maximum or its mean — the settled figure a growth question wants, but it means a MAXIMUM question (what was this job's longest run that day, did it enter its warning band) cannot be answered from this series: the day's peak is DROPPED rather than smoothed, so a day whose worst run crossed a threshold reads as a day that never approached it. The route that carries one row per run is TimescaleDB's own job history (timescaledb_information.job_history) — but ONLY while timescaledb.enable_job_execution_logging is on, and it defaults OFF, so on a store that has never had it turned on a maximum over that table returns zero rows, which reads as 'no run exceeded the line' rather than 'this instrument is off'. The job_history block in every response reports that setting's EFFECTIVE value and its source (plus the file that set it, where the connection is privileged enough to see it), so this redirect is never issued blind: read it before treating an empty job_history as an answer, and note that logging covers runs only from the point it was switched on because nothing earlier was recorded to recover. The hourly snapshot behind the series has the same limit one grain down: it samples last_run_duration once an hour at a fixed offset, so a run longer than that offset is never recorded at all. Also reports, read LIVE from the catalog rather than from the recorded series, every retention policy the rollup-coverage gate is holding PAUSED, with the tier's actual data span and how many times its configured drop_after horizon it is really holding — a held policy records zero failures and a normal-looking last run, so it is invisible in the stored job telemetry and is a common cause of unexplained store growth. Use for capacity forecasting: what is driving store growth, how fast, what adding N servers would multiply, which background job is closest to outgrowing its own cadence, and whether retention is actually running.")] + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot: per-hypertable total size, pre/post-compression bytes and chunk count; per-continuous-aggregate total size, pre/post-compression bytes and chunk count, sized through the aggregate's materialization hypertable and reported under the aggregate's view name (object_kind continuous_aggregate) — on one production store the twenty aggregates were 57% of the database and the previous inventory showed none of them, because timescaledb_information.hypertables never lists a materialization; the query-text and query-plan payload dimension tables' total size (the store's dominant payloads) and row counts; the product's named plain tables (object_kind table: collect.query_store_text, which stores statement text inline by design, collect.query_store_plan_map, config.config_alert_log) with total size and the planner's row-count estimate; two catch-all rows — object_kind other, every relation in a user schema that no named row accounts for, and object_kind system, the PostgreSQL catalogs and TimescaleDB's own bookkeeping — each with its byte total and relation count; the whole store's size with the enabled-server count; and one row per TimescaleDB background job (CAGG refresh, compression, retention) with its last run duration, schedule interval, duration-vs-cadence percent, and run/failure totals — the jobs whose runtimes scale with fleet size. The inventory block RECONCILES the newest sweep against its own pg_database_size and states coverage in so many words: enumerated_percent is the share under named objects, attributed_percent the share any row accounts for, residual_bytes what no row explains, and reconciled is false when that residual exceeds the larger of 1% and 64 MiB — a gap that does not close is itself a finding, and the note says so. bytes_by_kind gives the per-kind subtotals (which is where 'what is driving growth' is answered in one glance), and largest_unenumerated names the biggest relations inside the other row, read live, so that figure is something to act on. Each continuous_aggregate object also carries, read LIVE from the catalog, compression_enabled and the job ids of its refresh, compression and retention policies (null where none exists) — the facts a growth investigation into the aggregates otherwise assembles by hand. Returns the latest snapshot per object plus a daily series over the window, with the whole-store daily growth in bytes and the derived per-server ingest rate (daily growth / enabled servers). Each daily point is that day's LAST snapshot, never its maximum or its mean — the settled figure a growth question wants, but it means a MAXIMUM question (what was this job's longest run that day, did it enter its warning band) cannot be answered from this series: the day's peak is DROPPED rather than smoothed, so a day whose worst run crossed a threshold reads as a day that never approached it. The route that carries one row per run is TimescaleDB's own job history (timescaledb_information.job_history) — but it records a SUCCESSFUL run only while timescaledb.enable_job_execution_logging is on, and that setting defaults OFF (a FAILED run's row is written regardless of it), so on a store that has never had it turned on a maximum over that table covers zero rows of successful runs, which reads as 'no run exceeded the line' rather than 'this instrument is off'. The job_history block in every response reports that setting's EFFECTIVE value and its source (plus the file that set it, where the connection is privileged enough to see it), so this redirect is never issued blind: read it before treating an empty job_history as an answer, and note that logging covers runs only from the point it was switched on because nothing earlier was recorded to recover. The setting answers 'is it on'; whether YOU will see rows is a second question, because job_history is ownership-filtered — its rows are visible only to members of the job's owner role or of the database owner, while the jobs and job_stats views show every role every job — so a role that can list all the jobs can still read an empty history on a store that is recording perfectly. The same block therefore also reports which role it read as (reader_role), that role's standing under the view's own predicate (visibility: All, Partial or None, with the job counts behind it), the rows it actually observed over a fixed 24-hour window (rows_observed, newest_row_at) beside how many jobs job_stats says started a run in that window (jobs_run_in_window), and a contradiction flag that is true only when recording is on, a reader the view admits to every job's history saw no rows, and jobs ran — the finding to investigate. In managed mode this tool reads as the least-privilege mcp role, which the view filters out (visibility None), so its own rows_observed is zero by construction and the note names the role that can see. That role is the service's owner, and the service's hourly self-metrics sweep runs as it: the sweep persists the owner's own count over the same 24-hour window into the series, and the block reports it beside this connection's verdict as owner_evidence (Observed, Stale, Filtered or Absent), owner_role, owner_observed_at, owner_rows_observed, owner_newest_row_at and owner_jobs_run_in_window — so on a managed store 'recording' is a measurement after all, taken by the service's sweep rather than by this connection, the note says which, and the contradiction flag is computed from the owner's numbers when a fresh owner reading exists. The hourly snapshot behind the series has the same limit one grain down: it samples last_run_duration once an hour at a fixed offset, so a run longer than that offset is never recorded at all. Also reports, read LIVE from the catalog rather than from the recorded series, every retention policy the rollup-coverage gate is holding PAUSED, with the tier's actual data span and how many times its configured drop_after horizon it is really holding — a held policy records zero failures and a normal-looking last run, so it is invisible in the stored job telemetry and is a common cause of unexplained store growth. Use for capacity forecasting: what is driving store growth, how fast, what adding N servers would multiply, which background job is closest to outgrowing its own cadence, whether retention is actually running, and how much of the store the inventory can actually see.")] public static async Task GetStoreMetrics( NpgsqlDataSource postgres, [Description("Days of daily-series history. Default 30; max 400 (the series' own retention).")] int days_back = 30) @@ -97,8 +102,41 @@ the code does is worse than none. */ to. Read here rather than left to the caller because an empty job_history and a quiet fleet are the same result set, so a reader who follows the redirect cannot tell whether the answer they get back is a census or an artefact. Failure-isolated inside the reader, so this cannot - fail the response it qualifies. */ + fail the response it qualifies. + + #3574: and then the EVIDENCE behind that state — what this connection actually sees in the + view, and whether the view's ownership predicate would show it anything. The GUC answers "is + it on"; the reader's question is "will I see rows", and job_history filters by role membership + where the jobs view a reader checks first does not. On a managed store this very connection is + the least-privilege mcp role, which that predicate filters OUT, so the second read has to + evaluate the predicate for itself rather than count and assume. A second statement rather than + a column on the first: the two fail independently, and a count that timed out must not make + the GUC read unknown. */ var jobLogging = await DarlingStoreMetricsReader.GetJobExecutionLoggingAsync(postgres); + var jobEvidence = await DarlingStoreMetricsReader.GetJobHistoryEvidenceAsync(postgres, jobLogging); + + /* #3574, the managed-mode half: the OWNER's reading, decoded from the row the hourly sweep + persisted. Pure over rows already in hand — no fourth read — and dated, so the note can say + how old the owner's count is rather than presenting an hour-old measurement as this instant's. */ + var ownerEvidence = DarlingStoreMetricsReader.OwnerJobHistoryEvidence.FromLatest(latest, DateTime.UtcNow); + + /* #3582: the coverage statement. Pure over the same rows; null only when no store row exists to + reconcile against, in which case the block says coverage is unknown instead of computing a + percentage of nothing. */ + var inventory = DarlingStoreMetricsReader.ComputeInventory(latest); + + /* #3582: two more LIVE catalog reads, both failure-isolated to null (never to an empty list, + which would read as "no aggregates" / "nothing un-enumerated" and drop a section without a + word). The aggregate flags are STATE the series deliberately does not carry — the same + reasoning as the #2813 retention holds one screen up — and the top-N is what makes the + other row's byte count actionable: a growth investigation reads the table's name here + instead of running the pg_class census by hand. Both skip the TimescaleDB catalogs where the + GUC probe established they do not exist for this database. */ + var aggregateStates = await DarlingStoreMetricsReader.GetContinuousAggregateStatesAsync(postgres, jobLogging); + var largestUnenumerated = await DarlingStoreMetricsReader.GetLargestUnenumeratedAsync(postgres, jobLogging); + var aggregateStateByView = (aggregateStates ?? Array.Empty()) + .GroupBy(s => s.ViewName, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); return JsonSerializer.Serialize(new { @@ -116,6 +154,46 @@ fail the response it qualifies. */ per_server_bytes = g.PerServerBytes is { } rate ? Math.Round(rate) : (double?)null, }), }, + /* #3582. Present on EVERY response: the coverage statement is the qualifier on everything + in objects[] below, and a response without it is the response that answered for 38% of + a store and let a growth investigation trust it. Null fields where the newest sweep did + not produce the row they come from, never a zero standing in for a missing reading. */ + inventory = inventory is null ? null : new + { + sweep_at = inventory.SweepAt.ToString("o"), + database_bytes = inventory.DatabaseBytes, + enumerated_bytes = inventory.EnumeratedBytes, + enumerated_percent = inventory.EnumeratedPercent, + attributed_bytes = inventory.AttributedBytes, + attributed_percent = inventory.AttributedPercent, + residual_bytes = inventory.ResidualBytes, + tolerance_bytes = inventory.ToleranceBytes, + reconciled = inventory.Reconciled, + stale_object_rows = inventory.StaleRowCount, + bytes_by_kind = inventory.BytesByKind + .OrderByDescending(kv => kv.Value) + .ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal), + unenumerated = inventory.UnenumeratedBytes is null ? null : new + { + total_bytes = inventory.UnenumeratedBytes, + relation_count = inventory.UnenumeratedRelationCount, + }, + system = inventory.SystemBytes is null ? null : new + { + total_bytes = inventory.SystemBytes, + relation_count = inventory.SystemRelationCount, + }, + /* Live, not from the sweep: what the other row is MADE of, largest first. Null when + the read did not complete; an empty list means the census found nothing. */ + largest_unenumerated = largestUnenumerated?.Select(r => new + { + relation = r.Relation, + relkind = r.RelKind, + total_bytes = r.TotalBytes, + }), + aggregate_state = aggregateStates is null ? "Unreadable" : "Observed", + note = InventoryNote(inventory, latest, aggregateStates, largestUnenumerated), + }, /* #2813. Present on EVERY response, including when nothing is held — an absent block and "nothing is held" must not look alike, which is the entire failure this reports on. */ retention = new @@ -154,10 +232,52 @@ different readings of an empty job_history. */ setting = jobLogging.Setting, source = jobLogging.Source, source_file = jobLogging.SourceFile, - note = JobHistoryNote(jobLogging), + /* #3574. The evidence fields, all null unless evidence = Observed. reader_role first, + because every count that follows is a count through THAT role's eyes and the view + decides per role what it shows; visibility is the predicate's verdict on that role, + derived from the two membership facts beside it rather than asserted. rows_observed + travels with its window so the number never leaves without its denominator, and + jobs_run_in_window is the population half from the UNFILTERED job_stats view — the + proof that there was something to see. contradiction is the new finding class as one + bool, true only when all four of its conditions hold; the note spells them out. */ + evidence = jobEvidence.Status.ToString(), + reader_role = jobEvidence.ReaderRole, + reader_is_database_owner_member = jobEvidence.ReaderIsDatabaseOwnerMember, + visibility = jobEvidence.Status == DarlingStoreMetricsReader.JobHistoryEvidenceStatus.Observed + ? jobEvidence.Visibility.ToString() + : null, + job_count = jobEvidence.JobCount, + history_visible_job_count = jobEvidence.HistoryVisibleJobCount, + observed_window_hours = DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours, + rows_observed = jobEvidence.RowsObserved, + newest_row_at = jobEvidence.NewestRowAt?.ToString("o", CultureInfo.InvariantCulture), + jobs_run_in_window = jobEvidence.JobsRunInWindow, + newest_run_started_at = jobEvidence.NewestRunStartedAt?.ToString("o", CultureInfo.InvariantCulture), + /* #3574, managed-mode self-proof. The OWNER's reading from the hourly sweep, beside this + connection's. owner_evidence says whether there is one to lean on (Observed / Stale / + Filtered / Absent); the rest are null unless a row exists. owner_observed_at is the + instant the count is true of — an hour-old measurement is reported as one. */ + owner_evidence = ownerEvidence.Status.ToString(), + owner_role = ownerEvidence.ReaderRole, + owner_observed_at = ownerEvidence.ObservedAt?.ToString("o", CultureInfo.InvariantCulture), + owner_observed_age_hours = ownerEvidence.AgeHours, + owner_observed_window_hours = ownerEvidence.WindowHours, + owner_rows_observed = ownerEvidence.RowsObserved, + owner_newest_row_at = ownerEvidence.NewestRowAt?.ToString("o", CultureInfo.InvariantCulture), + owner_jobs_run_in_window = ownerEvidence.JobsRunInWindow, + /* True from EITHER admitted reader's numbers: this connection's where it has owner + standing, the sweep's where it does not. Each conjunction is its own record's, and + the note says which one fired. */ + contradiction = jobEvidence.ContradictsRecording(jobLogging.Recording) + || ownerEvidence.ContradictsRecording(jobLogging.Recording), + note = JobHistoryNote(jobLogging, jobEvidence, ownerEvidence), }, + /* The job_history row is deliberately NOT in objects[] or daily[]: it carries no bytes, and + its columns are the overloaded ones the reader decodes into the block above — rendered + raw here, last_run_duration_ms would read as a run's duration. */ objects = latest - .Where(r => r.ObjectKind != StoreSelfMetrics.StoreObjectKind) + .Where(r => r.ObjectKind != StoreSelfMetrics.StoreObjectKind + && r.ObjectKind != StoreSelfMetrics.JobHistoryObjectKind) .OrderByDescending(r => r.TotalBytes ?? 0) .Select(r => new { @@ -173,7 +293,24 @@ way the operator already talks about it (6-36x measured on the motivating store) ? Math.Round(r.CompressedBeforeBytes.Value / (double)r.CompressedAfterBytes.Value, 1) : (double?)null, chunk_count = r.ChunkCount, + /* #3582: on the two catch-all rows chunk_count is the RELATION count the sum spans + (the column-mapping paragraph on StoreSelfMetrics); surfaced under its own name + so a reader is not left inferring it. Null on every other kind. */ + relation_count = r.ObjectKind == StoreSelfMetrics.OtherObjectKind || r.ObjectKind == StoreSelfMetrics.SystemObjectKind + ? r.ChunkCount + : null, row_count = r.RowCount, + /* #3582: the live catalog state of a continuous aggregate, joined by view name — + null on every other kind, and null on an aggregate when the live read did not + complete (inventory.aggregate_state says so) or the view has since been dropped. + Policy fields are job ids, null where no such policy exists: the three facts a + growth investigation into the aggregates otherwise assembles by hand. */ + compression_enabled = AggregateState(r, aggregateStateByView)?.CompressionEnabled, + materialized_only = AggregateState(r, aggregateStateByView)?.MaterializedOnly, + source = AggregateState(r, aggregateStateByView)?.SourceName, + refresh_policy_job_id = AggregateState(r, aggregateStateByView)?.RefreshJobId, + compression_policy_job_id = AggregateState(r, aggregateStateByView)?.CompressionJobId, + retention_policy_job_id = AggregateState(r, aggregateStateByView)?.RetentionJobId, /* #2136 background_job rows only (NULL elsewhere): last run duration, the job's own cadence, and how much of that cadence the run consumed — the ceiling-proximity number an onboarding wave moves first. */ @@ -186,7 +323,8 @@ number an onboarding wave moves first. */ total_failures = r.TotalFailures, }), daily = daily - .Where(p => p.ObjectKind != StoreSelfMetrics.StoreObjectKind) + .Where(p => p.ObjectKind != StoreSelfMetrics.StoreObjectKind + && p.ObjectKind != StoreSelfMetrics.JobHistoryObjectKind) .GroupBy(p => (p.ObjectKind, p.ObjectName)) .OrderBy(g => g.Key.ObjectKind, StringComparer.Ordinal) .ThenBy(g => g.Key.ObjectName, StringComparer.Ordinal) @@ -216,14 +354,186 @@ number an onboarding wave moves first. */ } } + private static DarlingStoreMetricsReader.ContinuousAggregateState? AggregateState( + DarlingStoreMetricsReader.StoreMetricRow row, + Dictionary byView) + => row.ObjectKind == StoreSelfMetrics.ContinuousAggregateObjectKind + && byView.TryGetValue(row.ObjectName, out var state) + ? state + : null; + + /// + /// The coverage statement (#3582), in the issue's own words: "inventory covers N% of the database; X in + /// K un-enumerated relations" — then what the reconciliation found. One sentence per fact, and the + /// facts that are FINDINGS say so: a residual over the bar, catch-all rows missing from the newest + /// sweep, object rows the newest sweep never reached, aggregates holding bytes with compression off. + /// Bytes are stated in GiB to one decimal for a reader, beside the exact fields; counts invariant. + /// + internal static string InventoryNote( + DarlingStoreMetricsReader.InventoryReconciliation inventory, + IReadOnlyList latest, + IReadOnlyList? aggregateStates, + IReadOnlyList? largestUnenumerated) + { + var sb = new System.Text.StringBuilder(); + + sb.Append("The inventory's named rows (hypertables, continuous aggregates, payload dimensions, named tables) ") + .Append("account for ").Append(Gib(inventory.EnumeratedBytes)).Append(" of the ") + .Append(Gib(inventory.DatabaseBytes)).Append(" database") + .Append(inventory.EnumeratedPercent is { } ep ? $" ({Invariant(ep)}%)" : "") + .Append(" as of the sweep at ").Append(inventory.SweepAt.ToString("o", CultureInfo.InvariantCulture)).Append('.'); + + if (inventory.UnenumeratedBytes is { } otherBytes) + { + sb.Append(' ').Append(Gib(otherBytes)).Append(" sits in ") + .Append(Invariant(inventory.UnenumeratedRelationCount ?? 0)) + .Append(" un-enumerated user-schema relation(s) (object_kind other)"); + if (largestUnenumerated is { Count: > 0 }) + { + sb.Append(", the largest being ") + .Append(string.Join(", ", largestUnenumerated.Take(3).Select(r => $"{r.Relation} ({Gib(r.TotalBytes)})"))); + } + else if (largestUnenumerated is null) + { + sb.Append(" — the live census naming them did not complete"); + } + + sb.Append("; "); + } + else + { + sb.Append(" The 'other' catch-all row is MISSING from this sweep, so the un-enumerated share is unknown; "); + } + + if (inventory.SystemBytes is { } systemBytes) + { + sb.Append(Gib(systemBytes)).Append(" is PostgreSQL catalog and TimescaleDB bookkeeping in ") + .Append(Invariant(inventory.SystemRelationCount ?? 0)).Append(" relation(s) (object_kind system)."); + } + else + { + sb.Append("the 'system' catch-all row is MISSING from this sweep."); + } + + if (inventory.Reconciled) + { + sb.Append(" RECONCILED: every row together accounts for ") + .Append(inventory.AttributedPercent is { } ap ? $"{Invariant(ap)}%" : "an unknown share") + .Append(" of pg_database_size; the residual of ").Append(Invariant(inventory.ResidualBytes)) + .Append(" bytes is the database directory's non-relation files plus whatever moved between the ") + .Append("sweep's statements, inside the ").Append(Gib(inventory.ToleranceBytes)).Append(" bar."); + } + else if (!inventory.CatchAllPresent) + { + sb.Append(" NOT RECONCILED: without both catch-all rows the residual cannot be judged — the newest sweep ") + .Append("did not produce them, which is a sweep failure to look at before reading any coverage figure here."); + } + else + { + sb.Append(" NOT RECONCILED — a finding: ").Append(Gib(Math.Abs(inventory.ResidualBytes))) + .Append(inventory.ResidualBytes >= 0 + ? " of pg_database_size is attributed to NO row of this inventory" + : " MORE is attributed to rows than pg_database_size holds") + .Append(", past the ").Append(Gib(inventory.ToleranceBytes)) + .Append(" bar. Bytes the database holds that nothing here names are exactly the shape this block exists to ") + .Append("catch; compare a pg_class census against objects[] before trusting any per-object figure."); + } + + if (inventory.StaleRowCount > 0) + { + sb.Append(' ').Append(Invariant(inventory.StaleRowCount)) + .Append(" object row(s) in objects[] are from an OLDER sweep than the store row and are excluded from these ") + .Append("sums: the newest sweep did not reach them, so the sweep is not completing — its own Warning line says why."); + } + + var hasTimescaleRows = latest.Any(r => + r.ObjectKind == StoreSelfMetrics.HypertableObjectKind || r.ObjectKind == StoreSelfMetrics.ContinuousAggregateObjectKind); + if (!hasTimescaleRows) + { + sb.Append(" No hypertable or continuous-aggregate rows were recorded (a plain-PostgreSQL store, or TimescaleDB ") + .Append("unavailable to the sweep): on such a store the collector tables are ordinary tables and are counted ") + .Append("under 'other' rather than by name, so a low enumerated share here is that, not a fault."); + } + + if (aggregateStates is { Count: > 0 }) + { + var uncompressed = aggregateStates.Where(s => !s.CompressionEnabled).Select(s => s.ViewName).ToHashSet(StringComparer.Ordinal); + var uncompressedBytes = latest + .Where(r => r.ObjectKind == StoreSelfMetrics.ContinuousAggregateObjectKind + && r.MetricTime == inventory.SweepAt + && uncompressed.Contains(r.ObjectName)) + .Sum(r => r.TotalBytes ?? 0); + if (uncompressed.Count > 0) + { + sb.Append(' ').Append(Invariant(uncompressed.Count)).Append(" of ").Append(Invariant(aggregateStates.Count)) + .Append(" continuous aggregate(s) have compression DISABLED and hold ").Append(Gib(uncompressedBytes)) + .Append(" between them (compression_enabled on each continuous_aggregate object; the policy job ids beside it)."); + } + } + else if (aggregateStates is null) + { + sb.Append(" The live read of each aggregate's compression and policy state did not complete, so those fields are null on the continuous_aggregate objects."); + } + + return sb.ToString(); + } + + /// Bytes for a reader, in the largest binary unit that gives a whole-number part: a 235 GiB + /// aggregate family reads as GiB, a 73 KiB registry table as KiB, and neither as "0.0 GiB". The exact + /// byte fields sit beside every sentence this decorates; this is for the sentence. + internal static string Gib(long bytes) + { + var magnitude = Math.Abs(bytes); + return magnitude >= 1L << 30 ? (bytes / (double)(1L << 30)).ToString("0.0", CultureInfo.InvariantCulture) + " GiB" + : magnitude >= 1L << 20 ? (bytes / (double)(1L << 20)).ToString("0.0", CultureInfo.InvariantCulture) + " MiB" + : magnitude >= 1L << 10 ? (bytes / (double)(1L << 10)).ToString("0.0", CultureInfo.InvariantCulture) + " KiB" + : bytes.ToString(CultureInfo.InvariantCulture) + " bytes"; + } + + private static string Invariant(double value) => value.ToString(CultureInfo.InvariantCulture); + /// - /// What an empty timescaledb_information.job_history means on THIS store (#3175). One sentence - /// per state, and the states deliberately do not share one: the whole defect is that "off" and "nothing - /// happened" produce the same empty result, so a note that hedged across both would reproduce it in - /// prose. Off splits again on whether anything SET it off, because the two need different - /// actions — one heals itself, the other needs an override removed. + /// What an empty timescaledb_information.job_history means on THIS store (#3175), and for WHOM + /// (#3574). Two halves, concatenated. The first is the GUC's: one sentence per state, and the states + /// deliberately do not share one — the whole defect is that "off" and "nothing happened" produce the + /// same empty result, so a note that hedged across both would reproduce it in prose; Off splits + /// again on whether anything SET it off, because the two need different actions (one heals itself, the + /// other needs an override removed). The second half is the evidence's, from + /// : the ownership rule the view enforces, which role this block + /// read as, what that role is allowed to see, what it saw, and whether the four conditions of the + /// contradiction hold. It is appended to every arm on which the view exists — including the GUC-off + /// arms, because a reader who heals the GUC and then checks as the wrong role walks into the same trap + /// one step later — and omitted only where there is no view to be filtered. /// - internal static string JobHistoryNote(DarlingStoreMetricsReader.JobExecutionLoggingReading reading) + internal static string JobHistoryNote( + DarlingStoreMetricsReader.JobExecutionLoggingReading reading, + DarlingStoreMetricsReader.JobHistoryEvidence evidence) + => JobHistoryNote(reading, evidence, DarlingStoreMetricsReader.OwnerJobHistoryEvidence.Absent); + + /// + /// The full note (#3175 + #3574 + the managed-mode self-proof): the GUC's half, the connection's own + /// visibility half, then — wherever this connection is NOT itself an admitted reader — the OWNER's + /// half from . Omitted where the view does not exist + /// (NotApplicable) and where this connection already has All standing, because there the + /// connection's own count IS the census and a second census would be noise beside it. + /// + internal static string JobHistoryNote( + DarlingStoreMetricsReader.JobExecutionLoggingReading reading, + DarlingStoreMetricsReader.JobHistoryEvidence evidence, + DarlingStoreMetricsReader.OwnerJobHistoryEvidence owner) + { + var note = GucNote(reading) + JobHistoryVisibilityNote(reading, evidence); + + if (evidence.Status == DarlingStoreMetricsReader.JobHistoryEvidenceStatus.NotApplicable + || evidence.Visibility == DarlingStoreMetricsReader.JobHistoryVisibility.All) + { + return note; + } + + return note + OwnerEvidenceNote(reading, owner); + } + + private static string GucNote(DarlingStoreMetricsReader.JobExecutionLoggingReading reading) => reading.Status switch { DarlingStoreMetricsReader.JobExecutionLoggingStatus.On => @@ -232,10 +542,20 @@ internal static string JobHistoryNote(DarlingStoreMetricsReader.JobExecutionLogg + "the moment logging was turned on, never before: nothing was written for earlier runs, so an empty " + "window that predates that point is expected and is not evidence about those runs.", + /* #3175 arms, corrected (#3582 follow-up): an OFF setting does not make the view EMPTY. TimescaleDB + 2.28.1 writes a FAILED run's history row regardless of this GUC (job_stat_history.c gates only + the success path; measured on a fresh rig with the GUC off, the telemetry job's one failure was + the view's one row), so the honest reading of an OFF store's job_history is "a census of + failures, not of runs". The earlier wording — "returns zero rows" — contradicted the visibility + arm appended right after it, which correctly says any rows an admitted reader sees now are + failures. */ DarlingStoreMetricsReader.JobExecutionLoggingStatus.Off when reading.OffByExplicitOverride => "timescaledb.enable_job_execution_logging is OFF and something SET it off — 'source' is not 'default'. " - + "timescaledb_information.job_history is NOT recording, so a maximum over it returns zero rows and " - + "that means 'this instrument is off', NOT 'no run exceeded the line'. The service's managed conf " + + "timescaledb_information.job_history is recording FAILED runs only: TimescaleDB writes a failure's row " + + "regardless of this setting and a successful run's only while it is on, so a maximum over it is a " + + "census of failures and NOT of runs — an empty result means no failure was recorded while the setting " + + "was off, NOT 'no run exceeded the line', and a non-empty one is failures, not a sign logging is secretly " + + "on. The service's managed conf " + "block does not correct this one, because whatever set it is winning by last-occurrence: either an " + "ALTER SYSTEM (which lands in postgresql.auto.conf, read after postgresql.conf and so beating the " + "managed block outright, cleared with ALTER SYSTEM RESET) or a hand-added line placed after the " @@ -245,9 +565,12 @@ internal static string JobHistoryNote(DarlingStoreMetricsReader.JobExecutionLogg + "which carries one sample per job, not one row per run.", DarlingStoreMetricsReader.JobExecutionLoggingStatus.Off => - "timescaledb.enable_job_execution_logging is OFF, so timescaledb_information.job_history is NOT " - + "recording. A maximum over it returns zero rows, and that means 'this instrument is off', NOT 'no " - + "run exceeded the line' — do not read an empty job_history on this store as a clean result. A " + "timescaledb.enable_job_execution_logging is OFF, so timescaledb_information.job_history is recording " + + "FAILED runs only: TimescaleDB writes a failure's row regardless of this setting and a successful run's " + + "only while it is on. A maximum over it is therefore a census of failures and NOT of runs — an empty " + + "result means no failure was recorded while the setting was off, NOT 'no run exceeded the line', so do " + + "not read an empty job_history on this store as a clean result, and do not read a non-empty one as " + + "logging being secretly on. A " + "managed store turns it on by gaining the v11 postgresql.conf block on the next server start the " + "service owns; runs before that point wrote nothing and cannot be recovered. Until then the only " + "surface is job_stats — the last_run_duration_ms in this response — which carries one sample per " @@ -266,4 +589,254 @@ internal static string JobHistoryNote(DarlingStoreMetricsReader.JobExecutionLogg + "timescaledb_information.job_history is recording is UNKNOWN — which is not the same as off. Treat " + "an empty job_history on this store as unexplained rather than as a clean result until this reads.", }; + + /// + /// The visibility rule, stated with a measurement (#3574). The rule itself is one sentence and it is the + /// same on every arm: history rows are visible only to members of the job's owner role or of the database + /// owner, and the jobs / job_stats views a reader checks first are NOT filtered, which is + /// the trap. What follows it depends on what the evidence read established about THIS reader: + /// + /// None — the managed-mode mcp role's reading on every store: the view shows this + /// connection nothing by construction, so its zero is the filter and not the table, and the note names + /// the role that can see and says what the unfiltered job_stats saw in the meantime. + /// All — the count is a census, and the flag is self-proving: rows seen means recording is + /// a measurement; zero rows with runs in the window and the GUC on is the CONTRADICTION, named as such, + /// with the one benign cause and how to settle it; zero rows with no runs proves nothing either way and + /// the note says so rather than calling it clean. + /// Partial — a census of the jobs this reader owns, stated as a fraction, no verdict. + /// Unknown after an Observed read — no jobs exist, so there is no owner to be a + /// member of; after an Unreadable one — the flag stays a GUC echo and the note says so. + /// + /// Empty for : the view + /// does not exist on that connection, and a rule about who may read a view that is not there would be + /// noise on the one arm whose existing text already says everything true. + /// + /// Counts are formatted invariant and timestamps as ISO 8601 UTC so the sentence a caller reads + /// agrees byte-for-byte with the fields beside it. + /// + internal static string JobHistoryVisibilityNote( + DarlingStoreMetricsReader.JobExecutionLoggingReading reading, + DarlingStoreMetricsReader.JobHistoryEvidence evidence) + { + const string rule = + " WHO CAN SEE THE ROWS: timescaledb_information.job_history shows a row only to a member of the job's " + + "owner role or of the database owner (its own WHERE clause: pg_has_role(current_user, , 'MEMBER') OR pg_has_role(current_user, , 'MEMBER')), while the jobs and " + + "job_stats views show every role every job — so a role that can list all the jobs can still read an " + + "empty history on a store that is recording perfectly, and a zero from a role outside those " + + "memberships contradicts nothing."; + + switch (evidence.Status) + { + case DarlingStoreMetricsReader.JobHistoryEvidenceStatus.NotApplicable: + return ""; + + case DarlingStoreMetricsReader.JobHistoryEvidenceStatus.Unreadable: + return rule + + " The evidence read behind this block did not complete, so what THIS connection sees in " + + "job_history is UNKNOWN and 'recording' above is the GUC's word alone, not a measurement."; + } + + var role = evidence.ReaderRole ?? "(unknown role)"; + var window = Invariant(DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours); + var rows = Invariant(evidence.RowsObserved ?? 0); + var ran = Invariant(evidence.JobsRunInWindow ?? 0); + var newestRow = evidence.NewestRowAt is { } nr ? nr.ToString("o", CultureInfo.InvariantCulture) : null; + var newestRun = evidence.NewestRunStartedAt is { } ns ? ns.ToString("o", CultureInfo.InvariantCulture) : null; + + switch (evidence.Visibility) + { + case DarlingStoreMetricsReader.JobHistoryVisibility.None: + return rule + + $" This block read as '{role}', which is a member of NEITHER, so the view shows this connection " + + $"NOTHING by construction: rows_observed = {rows} here is the filter, not the table, and says " + + "nothing about whether recording works. The service's owner role — the role that created the " + + "jobs — sees the rows; read job_history as that role before concluding anything from an empty " + + $"result. Meanwhile job_stats, which is not filtered, says {ran} job(s) started a run in the " + + $"last {window} hours" + + (newestRun is null ? "." : $" (newest start {newestRun})."); + + case DarlingStoreMetricsReader.JobHistoryVisibility.Partial: + return rule + + $" This block read as '{role}', which is a member of the owner of " + + $"{Invariant(evidence.HistoryVisibleJobCount ?? 0)} of {Invariant(evidence.JobCount ?? 0)} jobs " + + "and not of the database owner, so the count is a census of THOSE jobs only: " + + $"{rows} row(s) with a start in the last {window} hours" + + (newestRow is null ? ", none ever." : $", newest {newestRow}.") + + " A job outside that set shows this role nothing, whatever it recorded."; + + case DarlingStoreMetricsReader.JobHistoryVisibility.All: + { + var standing = evidence.ReaderIsDatabaseOwnerMember == true + ? "a member of the database owner" + : "a member of every job's owner"; + var census = + $" This block read as '{role}', which is {standing}, so the count is a census: {rows} row(s) with a " + + $"start in the last {window} hours" + + (newestRow is null ? ", none ever" : $", newest {newestRow}") + + $"; job_stats says {ran} job(s) started a run in the same window" + + (newestRun is null ? "." : $" (newest start {newestRun})."); + + if (evidence.ContradictsRecording(reading.Recording)) + { + return rule + census + + " CONTRADICTION: the GUC says recording, this role can see every job's history, jobs " + + "started runs inside the window, and the view showed NONE of them — do not read this zero " + + "as quiet. The one benign cause is logging switched on AFTER the last of those starts (the " + + "managed v11 heal lands on a service-owned server start and writes nothing for earlier " + + "runs), which the next hourly run settles: re-read after it. A zero that persists while " + + "jobs_run_in_window climbs means the instrument is not writing what the GUC says it is, " + + "and that is a finding, not a quiet hour."; + } + + if (reading.Recording && evidence.RowsObserved is > 0) + { + return rule + census + + " 'recording' is therefore a measurement here, not a GUC echo."; + } + + if (reading.Recording) + { + /* Zero rows, zero runs: nothing happened for the instrument to catch, so the zero is not + evidence either way — and saying "clean" here would be the exact mis-reading #3175 and + #3574 exist to prevent, one level down. */ + return rule + census + + " No job started a run in the window, so there was nothing to record and this zero " + + "proves nothing either way; it is not a clean result, it is an absence of information."; + } + + if (reading.Status == DarlingStoreMetricsReader.JobExecutionLoggingStatus.Unreadable) + { + /* The GUC could not be read but the view could: the rows are real and this reader sees + them all, but whether they are the successes logging records or the failures TimescaleDB + writes regardless cannot be said without the setting — so it is not said. */ + return rule + census + + " Whether logging is on could not be read, so these rows are not classified: TimescaleDB " + + "writes a FAILED run's row regardless of the setting and a successful run's only while " + + "it is on."; + } + + /* GUC off, reader admitted: say what it will see once logging is on, and what it sees even + now — TimescaleDB writes a FAILED run's row regardless of the setting (the + bgw_job_stat_history_update path in 2.28.1 logs failures unconditionally), so a non-zero + count with the GUC off is failures, not a sign that logging is secretly on. */ + return rule + census + + " Once logging is on this connection will see the rows; any it sees now are FAILED runs, " + + "which TimescaleDB writes regardless of the setting — successes need it on."; + } + + default: + return rule + + $" This block read as '{role}'; timescaledb_information.jobs lists no jobs on this connection, " + + "so there is no owner to be a member of and nothing for history to record yet."; + } + } + + /// + /// The OWNER's half of the note (#3574, managed-mode self-proof) — appended only where this connection + /// is not itself an admitted reader, which in managed mode is every response. It says where the number + /// came from in so many words: the service's hourly self-metrics sweep, running as the owner role, not + /// this connection. Then, per what the series holds: + /// + /// Observed — the owner's count, its instant, and the population beside it; rows seen with + /// the GUC on is recording proven by the owner's measurement; zero rows with runs and the GUC on + /// is the CONTRADICTION, from the owner's numbers, with the same benign cause and the same way to + /// settle it; zero rows with nothing run proves nothing; GUC off classifies any rows as failures. + /// Stale — the count and its age, and that it is not read as current: the sweep is hourly + /// and this row is past , which means + /// the sweep itself is not landing. + /// Filtered — the sweep's role was not admitted either, named, so the reader knows the + /// connection string to look at; no count was recorded. + /// Absent — the series holds no owner row yet, so nothing on this block is a measurement. + /// + /// + internal static string OwnerEvidenceNote( + DarlingStoreMetricsReader.JobExecutionLoggingReading reading, + DarlingStoreMetricsReader.OwnerJobHistoryEvidence owner) + { + const string source = + " THE OWNER'S OWN COUNT: the service's hourly self-metrics sweep runs on the owner pool — the role that created " + + "the jobs and that the view admits — and persists what it sees into the series, so the owner_* fields here " + + "come from the service's sweep, not from this connection."; + + switch (owner.Status) + { + case DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Absent: + return source + + " The series holds no such row yet (it lands within an hour of a service start on a store at this " + + "build, and only where TimescaleDB is available), so nothing in this block is a measurement of " + + "recording until it does."; + + case DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Filtered: + return source + + $" The sweep's role '{owner.ReaderRole}' was itself NOT admitted to every job's history at " + + $"{Stamp(owner.ObservedAt)}, so it recorded no count — the service's own connection string names a role " + + "outside the owner memberships, and that, not recording, is the thing to fix" + + (owner.JobsRunInWindow is { } filteredRan + ? $"; the unfiltered job_stats saw {Invariant(filteredRan)} job(s) start a run in the {Window(owner)}-hour window before it." + : "."); + } + + var rows = Invariant(owner.RowsObserved ?? 0); + var ran = Invariant(owner.JobsRunInWindow ?? 0); + var census = + $" Reading as '{owner.ReaderRole}' at {Stamp(owner.ObservedAt)}, the sweep saw {rows} row(s) with a start in the " + + $"{Window(owner)} hours before that instant" + + (owner.NewestRowAt is { } newest ? $" (newest {Stamp(newest)})" : ", none ever") + + $", while job_stats says {ran} job(s) started a run in the same window."; + + if (owner.Status == DarlingStoreMetricsReader.OwnerJobHistoryEvidenceStatus.Stale) + { + return source + census + + $" That reading is {Invariant(owner.AgeHours ?? 0)} hours old — the sweep is hourly, so at least two " + + "consecutive sweeps have not landed (its own Warning line says why) — and it describes a window that no " + + "longer speaks for now: it is shown, not read as current evidence, and no contradiction is judged from it."; + } + + if (owner.ContradictsRecording(reading.Recording)) + { + return source + census + + " CONTRADICTION (from the owner's numbers): the GUC says recording, the owner sees every job's history, " + + "jobs started runs inside the window, and the view showed the owner NONE of them — do not read this zero " + + "as quiet. The one benign cause is logging switched on AFTER the last of those starts (the managed v11 " + + "heal lands on a service-owned server start and writes nothing for earlier runs), which the next hourly " + + "sweep settles: re-read after it. A zero that persists while owner_jobs_run_in_window climbs means the " + + "instrument is not writing what the GUC says it is, and that is a finding, not a quiet hour."; + } + + if (reading.Recording && owner.RowsObserved is > 0) + { + return source + census + + " 'recording' is therefore a measurement on this store after all — the service's, taken hourly as the " + + "owner — not a GUC echo."; + } + + if (reading.Recording) + { + return source + census + + " No job started a run in that window, so there was nothing to record and the owner's zero proves " + + "nothing either way; it is not a clean result, it is an absence of information."; + } + + if (reading.Status == DarlingStoreMetricsReader.JobExecutionLoggingStatus.Unreadable) + { + return source + census + + " Whether logging is on could not be read, so the owner's rows are not classified: TimescaleDB writes a " + + "FAILED run's row regardless of the setting and a successful run's only while it is on."; + } + + return source + census + + " With the setting off, any rows the owner saw are FAILED runs, which TimescaleDB writes regardless of it — " + + "successes need it on."; + } + + private static string Stamp(DateTime? at) => + at is { } value ? value.ToString("o", CultureInfo.InvariantCulture) : "(unknown instant)"; + + private static string Window(DarlingStoreMetricsReader.OwnerJobHistoryEvidence owner) => + owner.WindowHours is { } hours + ? hours.ToString("0.##", CultureInfo.InvariantCulture) + : Invariant(DarlingStoreMetricsReader.JobHistoryEvidenceWindowHours); + + private static string Invariant(long value) => value.ToString(CultureInfo.InvariantCulture); } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs index e4addaa31..7b7bbd117 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTools.cs @@ -36,7 +36,16 @@ namespace PerformanceMonitor.Darling.Service.Mcp; [McpServerToolType] public sealed class DarlingMcpTools { - [McpServerTool(Name = "analyze_server"), Description("Runs the diagnostic inference engine against a server's collected data. Scores wait stats, blocking, memory, config, and other facts, then traverses a relationship graph to build evidence-backed stories about what's wrong and why. Anomaly detection compares the analysis window against 30-day time-bucketed baselines (hour-of-day x day-of-week) to identify deviations that are unusual for this specific time slot, not just unusual overall. Returns structured findings with severity scores, evidence chains, baseline context for anomalies, and recommended next tools to call. A remediable finding also carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose. Set as_of to analyze a PAST window instead of the present — hours_back stays the window's LENGTH, and the anomaly baseline moves with it, so the findings are the ones that window deserves rather than today's findings over older rows. An anchored run is EXPLORATORY: its findings are returned in full but deliberately NOT written to the store, because a finding row is stamped with the time the analysis RAN and would then be read as this server's current state by get_analysis_findings and by the viewer. The result says so in persisted / persistence_note.")] + /// + /// The source parameter's description on both SKUs' get_analysis_facts, built from + /// so the documented set IS the accepted set (#3541 A13). The old + /// text named four sources of fifteen; a caller who typed any of the other eleven got an empty list. + /// + internal const string FactSourceFilterDescription = + "Filter to one source category. Accepted values (the engine's complete source registry, refused otherwise): " + + "anomaly, bad_actor, blocking, config, coverage, cpu, database_config, disk, io, jobs, memory, queries, sessions, tempdb, waits. Omit for all."; + + [McpServerTool(Name = "analyze_server"), Description("Runs the diagnostic inference engine against a server's collected data. Scores wait stats, blocking, memory, config, and other facts, then traverses a relationship graph to build evidence-backed stories about what's wrong and why. Anomaly detection compares the analysis window against 30-day time-bucketed baselines (hour-of-day x day-of-week) to identify deviations that are unusual for this specific time slot, not just unusual overall. Returns structured findings with severity scores, evidence chains, baseline context for anomalies, and recommended next tools to call. Each finding's confidence is an EVIDENCE score, not a probability: 0.20 for the fired symptom alone, plus up to 0.48 for the share of the root fact's amplifier checks (its expected companions) that matched and up to 0.32 for the depth of the evidence chain, so a lone uncorroborated symptom reads 0.20 and a fully corroborated deep chain approaches 1.0; confidence_basis says in words what each value rests on. Rank by severity for impact and by confidence for how much of the engine's own corroboration showed up; do not multiply them. A remediable finding also carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose. Set as_of to analyze a PAST window instead of the present — hours_back stays the window's LENGTH, and the anomaly baseline moves with it, so the findings are the ones that window deserves rather than today's findings over older rows. An anchored run is EXPLORATORY: its findings are returned in full but deliberately NOT written to the store, because a finding row is stamped with the time the analysis RAN and would then be read as this server's current state by get_analysis_findings and by the viewer. The result says so in persisted / persistence_note.")] public static async Task AnalyzeServer( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -80,18 +89,56 @@ make every ordinary run look anchored to the engine — which is exactly the set ? null : "as_of was supplied, so this analysis ran over a PAST window and is exploratory: the findings below are complete but were NOT written to the store. A finding row carries the time the analysis RAN, and the reads that consume those rows (get_analysis_findings, the viewer's Recommendations tab) treat the newest analysis_time as this server's CURRENT state — so persisting a backdated run would make last week's findings today's headline and would inflate the occurrence stats of any live incident sharing a story path. Re-run without as_of to analyze and persist the present."; + if (analysisService.WindowEmptyMessage != null) + { + /* #3524: zero facts in the window is a DEAD-COLLECTOR shape, not a clean bill of + health — the data-span gate passes on lifetime history, so a server whose + collection broke last week lands here, and the "empty" all-clear below would tell + the caller in prose that all metrics are normal when nothing was measured at all. + Same miss vocabulary as get_analysis_facts' zero-facts case; same hints block as + the all-clear, because an anchored empty-window run still owes the caller the + persistence disclosure. */ + return McpHelpers.Status( + "unavailable", + analysisService.WindowEmptyMessage + + " Check get_collection_health to see when collectors last succeeded and why they stopped.", + new + { + analysis_time = analysisService.LastAnalysisTime?.ToString("o"), + persisted = anchor is null, + persistence_note = persistenceNote + }); + } + + /* #3538 A2: how much of the window the collector actually observed. Every rate in this pass + was divided by that time rather than by the nominal window, so the numbers are right at any + coverage — but a reader still needs to know the window had a hole in it, because "nothing + significant in the hour we saw" and "nothing significant in four hours" are different + claims. Below the partial bar the caveat is prose; the coverage block is always present. */ + var coverage = analysisService.LastWindowCoverage; + var coverageCaveat = coverage is { IsPartial: true } + ? $"PARTIAL COVERAGE: {coverage.Describe()}. Rates and fractions below are per observed time, so they are not deflated by the gap — but the unobserved stretch could have held anything, and nothing here speaks for it. Check get_collection_health for why collection stopped." + : null; + if (findings.Count == 0) { /* A successful analysis that found nothing wrong: a true negative ("all clear"), - surfaced with the shared miss vocabulary so callers branch on it uniformly. */ + surfaced with the shared miss vocabulary so callers branch on it uniformly. Facts + WERE collected and scored this time — the window-collected-nothing case returned + above as unavailable instead (#3524). With partial coverage the all-clear is scoped + to the time that was seen (#3538 A2): the same status, because facts were scored and + nothing fired, but prose that no longer claims the whole window. */ return McpHelpers.Status( "empty", - "No significant findings. All metrics are within normal ranges.", + coverageCaveat is null + ? "No significant findings. All metrics are within normal ranges." + : $"No significant findings in the {coverage!.Fraction:P0} of this window the collector observed — a PARTIAL reading, not a full all-clear. {coverage.Describe()}. The unobserved stretch could have held anything, and nothing here speaks for it; check get_collection_health for why collection stopped.", new { analysis_time = analysisService.LastAnalysisTime?.ToString("o"), persisted = anchor is null, - persistence_note = persistenceNote + persistence_note = persistenceNote, + coverage = coverage?.ToPayload() }); } @@ -110,6 +157,9 @@ surfaced with the shared miss vocabulary so callers branch on it uniformly. */ /* Null on the ordinary unanchored run — nothing needs saying when the answer is the one every caller already assumed. */ persistence_note = persistenceNote, + /* Null at full coverage (#3538 A2) — same rule. */ + caveat = coverageCaveat, + coverage = coverage?.ToPayload(), time_range = new { start = findings[0].TimeRangeStart?.ToString("o"), @@ -122,6 +172,11 @@ one every caller already assumed. */ { severity = Math.Round(f.Severity, 2), confidence = Math.Round(f.Confidence, 2), + // #3538 A6: what the number rests on. Corroboration-derived since this change + // (matched amplifier share + path depth); a row persisted under the old path-shape + // formula is labelled as such, derived from the finding's own shape at read time + // because the store carries no version marker (no schema change). + confidence_basis = StoryConfidence.DescribeBasis(f.RootFactKey, f.Confidence, f.FactCount), category = f.Category, root_fact = new { key = f.RootFactKey, value = f.RootFactValue }, leaf_fact = f.LeafFactKey != null @@ -173,13 +228,13 @@ one every caller already assumed. */ } } - [McpServerTool(Name = "get_analysis_facts"), Description("Exposes the raw scored facts from the inference engine's collect+score pipeline WITHOUT graph traversal. Shows every observation the engine sees: wait stats as fraction-of-period, blocking rates, config settings, memory stats, plus base severity, final severity after amplifiers, and which amplifiers matched. Use this to understand exactly what the engine is working with, or to investigate facts that didn't reach the severity threshold for findings.")] + [McpServerTool(Name = "get_analysis_facts"), Description("Exposes the raw scored facts from the inference engine's collect+score pipeline WITHOUT graph traversal. Shows every observation the engine sees: wait stats as fraction-of-period, blocking rates, config settings, memory stats, plus base severity, final severity after amplifiers, and which amplifiers matched. For ANOMALY_* facts the metadata carries baseline_confidence — the baseline's own trustworthiness (tier x sample density), which the scorer multiplies into that fact's severity; it is a different quantity from a finding's confidence in analyze_server. Use this to understand exactly what the engine is working with, or to investigate facts that didn't reach the severity threshold for findings.")] public static async Task GetAnalysisFacts( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, [Description("Hours of data to analyze. Default 4.")] int hours_back = 4, - [Description("Filter to a specific source category: waits, blocking, config, memory. Omit for all.")] string? source = null, + [Description(FactSourceFilterDescription)] string? source = null, [Description("Minimum severity to include. Default 0 (all facts). Use 0.5 to see only significant facts.")] double min_severity = 0, [Description(McpHelpers.AsOfDescription)] string? as_of = null) { @@ -189,13 +244,18 @@ public static async Task GetAnalysisFacts( var validation = McpHelpers.ValidateWindow(hours_back, as_of, out var windowEnd); if (validation != null) return validation; + /* #3541 A13: an unknown source is refused with the whole accepted set, never applied as a filter + that matches nothing. The set is the scorer's registry, not a copy of it. */ + validation = McpHelpers.ValidateChoice(source, FactScorer.KnownSources, "source"); + if (validation != null) return validation; + /* Null for an absent anchor — see analyze_server's note. Nothing here persists, so the distinction costs nothing; it is kept so AnalysisContext.AsOfUtc means one thing everywhere. */ var anchor = string.IsNullOrWhiteSpace(as_of) ? (DateTime?)null : windowEnd; try { - var facts = await analysisService.CollectAndScoreFactsAsync( + var (facts, coverage) = await analysisService.CollectAndScoreFactsAsync( resolved.ServerId, resolved.ServerName, hours_back, asOfUtc: anchor); if (facts.Count == 0) @@ -207,6 +267,22 @@ public static async Task GetAnalysisFacts( "No facts collected. The collector may not have run yet, or no data exists in the requested time range."); } + if (coverage is null || !coverage.IsObserved) + { + /* #3538 A2: facts exist but the window was never observed — the point-in-time config and + state facts read from the latest row regardless of window, and every windowed rate is + absent because there was no time to divide by. The tool's own description promises + "wait stats as fraction-of-period, blocking rates"; none of those can be shown, so + this is the unavailable envelope, with the count of what COULD be read so a caller + after configuration alone knows audit_config still has it. */ + return McpHelpers.Status( + "unavailable", + $"The collector observed none of the requested window for {resolved.ServerName}, so no windowed fact (wait fractions, blocking or deadlock rates) exists to show. " + + $"{facts.Count} point-in-time fact(s) — configuration and current state — could still be read; audit_config reports those. " + + "Check get_collection_health to see when collectors last succeeded and why they stopped.", + new { coverage = coverage?.ToPayload() }); + } + var filtered = facts.AsEnumerable(); if (source != null) filtered = filtered.Where(f => f.Source.Equals(source, StringComparison.OrdinalIgnoreCase)); @@ -222,8 +298,17 @@ public static async Task GetAnalysisFacts( value = Math.Round(f.Value, 6), base_severity = Math.Round(f.BaseSeverity, 4), severity = Math.Round(f.Severity, 4), + // #3538 A6: an anomaly fact's metadata["confidence"] is the BASELINE's confidence (tier x + // density, BaselineBucket.Confidence) — the trustworthiness of the distribution the + // deviation was measured against, which the scorer multiplies into severity. It is not + // the story confidence analyze_server publishes, and one word for two quantities in one + // client session is the confusion this campaign item exists to remove, so the payload + // names it baseline_confidence. The fact's own metadata key is unchanged (the scorer + // reads it); this is a read-time projection only. metadata = f.Metadata.ToDictionary( - m => m.Key, + m => m.Key == "confidence" && string.Equals(f.Source, "anomaly", StringComparison.Ordinal) + ? "baseline_confidence" + : m.Key, m => Math.Round(m.Value, 2)), amplifiers = f.AmplifierResults.Count > 0 ? f.AmplifierResults.Select(a => new @@ -242,6 +327,13 @@ public static async Task GetAnalysisFacts( total_facts = facts.Count, shown = result.Count, filters = new { source, min_severity }, + /* #3538 A2: null at full coverage; below the partial bar it says what share of the window + the fractions and rates were divided over, because a 25%-of-observed-time wait on a + quarter-collected window is a different claim from 25% of four hours. */ + caveat = coverage.IsPartial + ? $"PARTIAL COVERAGE: {coverage.Describe()}. Every fraction-of-period and per-hour value below is per OBSERVED time (period_duration_ms × coverage_fraction, or observed_hours), not per nominal window; the COLLECTION_GAP fact carries the hole." + : null, + coverage = coverage.ToPayload(), facts = result }, McpHelpers.JsonOptions); } @@ -251,7 +343,7 @@ public static async Task GetAnalysisFacts( } } - [McpServerTool(Name = "compare_analysis"), Description("Compares two time periods by running the inference engine's fact collection and scoring on each, then showing what changed. Use this to compare peak vs off-peak, before vs after a change, or yesterday vs today. Returns facts from both periods side-by-side with severity deltas. Note: for routine anomaly detection, use analyze_server instead — it automatically compares against 30-day time-bucketed baselines (hour-of-day x day-of-week). This tool is for explicit window-to-window comparisons.")] + [McpServerTool(Name = "compare_analysis"), Description("Compares two time periods by running the inference engine's fact collection and scoring on each, then showing what changed. Use this to compare peak vs off-peak, yesterday vs today, or the windows around a change. Returns facts from both periods side-by-side, each banded worse / better / stable by how far the VALUE moved on the server's own scale, not by the severity formula's slope: a key with a stored per-server baseline (CPU %, read latency, connections) is banded in that baseline's robust sigma for the comparison hour (delta_sigma, band_source \"baseline\"); every other key changes status only when the value moved at least a quarter of the larger side AND registers at least a quarter of the way up its own severity ladder (band_source \"absolute\"); the rules are stated in band_rules. Rows are grouped into physical-cause families (one I/O stall is one family row, not four worse keys), and BAD_ACTOR_ appearances are reported as plan_cache_churn rather than as new or resolved issues. What \"worse\" does NOT mean: this is one window against one window — same-hour-yesterday at N=1 vs N=1 cannot show that a change CAUSED anything (DB time on an unchanged server routinely varies severalfold day to day), and a partly collected side flags every verdict with coverage_caveat. Note: for routine anomaly detection, use analyze_server instead — it automatically compares against 30-day time-bucketed baselines (hour-of-day x day-of-week). This tool is for explicit window-to-window comparisons.")] public static async Task CompareAnalysis( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -281,46 +373,46 @@ silently change what the two windows are relative to each other. */ var baselineEnd = windowEnd.AddHours(-baseline_hours_back + hours_back); var baselineStart = windowEnd.AddHours(-baseline_hours_back); - var (baselineFacts, comparisonFacts) = await analysisService.ComparePeriodsAsync( + var (baselineFacts, comparisonFacts, baselineCoverage, comparisonCoverage, dispersion) = await analysisService.ComparePeriodsAsync( resolved.ServerId, resolved.ServerName, baselineStart, baselineEnd, comparisonStart, comparisonEnd); - var baselineByKey = baselineFacts.ToFactLookup(); - var comparisonByKey = comparisonFacts.ToFactLookup(); - var allKeys = baselineByKey.Keys.Union(comparisonByKey.Keys).ToHashSet(); + /* The COLLECTION_GAP context fact (#3538 A2) is an observation of the COLLECTOR, not of the + server, and it is reported through the coverage blocks and caveat below. Left in the + comparison it would count as a "stable" (or, on one side, a "new") entry in the summary + and pad fact rows with a key no advice speaks to. */ + var baselineServerFacts = baselineFacts.Where(f => f.Source != WindowCoverage.FactSource).ToList(); + var comparisonServerFacts = comparisonFacts.Where(f => f.Source != WindowCoverage.FactSource).ToList(); - var comparisons = allKeys - .Select(key => - { - var baseline = baselineByKey.GetValueOrDefault(key); - var comparison = comparisonByKey.GetValueOrDefault(key); - var severityDelta = (comparison?.Severity ?? 0) - (baseline?.Severity ?? 0); + /* + #3538 A3: the coverage caveat is COMPOSED into the verdicts, not restated. The prose below + says which side was partly collected; every verdict row and family row carries + coverage_caveat: true when either side was, so a reader of one row cannot take "worse" at + face value without being told the side it rests on speaks for a fraction of its window. + */ + var baselinePartial = baselineCoverage is not null && (baselineCoverage.IsPartial || !baselineCoverage.IsObserved); + var comparisonPartial = comparisonCoverage is not null && (comparisonCoverage.IsPartial || !comparisonCoverage.IsObserved); - return new - { - key, - source = baseline?.Source ?? comparison?.Source ?? "unknown", - baseline_value = baseline != null ? Math.Round(baseline.Value, 6) : (double?)null, - comparison_value = comparison != null ? Math.Round(comparison.Value, 6) : (double?)null, - baseline_severity = baseline != null ? Math.Round(baseline.Severity, 4) : (double?)null, - comparison_severity = comparison != null ? Math.Round(comparison.Severity, 4) : (double?)null, - severity_delta = Math.Round(severityDelta, 4), - status = severityDelta > 0.1 ? "worse" : severityDelta < -0.1 ? "better" : "stable" - }; - }) - .OrderByDescending(c => Math.Abs(c.severity_delta)) - .ToList(); + /* + Every verdict — sigma-banded where a per-server baseline exists, ladder-banded where it + does not, plan-cache churn kept out of the issue counters, one family row per physical + cause — is decided in the shared ComparisonBanding, so this SKU and its twin cannot band + the same two windows differently. The tool only serializes. + */ + var comparison = ComparisonBanding.Compare( + baselineServerFacts, comparisonServerFacts, dispersion, + coverageCaveat: baselinePartial || comparisonPartial); - if (comparisons.Count == 0) + if (comparison.IsEmpty) { /* Neither window produced a single fact, and the old payload said that with all-zero counters and facts: [] -- which reads as "nothing changed" when it actually means "there was nothing to compare". Those are opposite conclusions about the same server. - No probe is needed to tell them apart: comparisons is the UNION of both windows' keys, - so zero entries is exactly "both fact sets were empty" and the fact_counts already in - hand are the whole answer. + No probe is needed to tell them apart: the comparison is over the UNION of both windows' + keys, so an empty one is exactly "both fact sets were empty" and the fact_counts already + in hand are the whole answer. */ return McpHelpers.Status( "unavailable", @@ -332,6 +424,11 @@ hand are the whole answer. baseline_end = baselineEnd.ToString("o"), comparison_start = comparisonStart.ToString("o"), comparison_end = comparisonEnd.ToString("o"), + /* #3538 A2: WHICH kind of nothing — a window the collector never observed, or one + it observed and found idle — is the difference between "check collection" and + "the server was quiet", and only the coverage can tell them apart. */ + baseline_coverage = baselineCoverage?.ToPayload(), + comparison_coverage = comparisonCoverage?.ToPayload() }); } @@ -342,39 +439,65 @@ because it has nothing to be compared against. "47 resolved issues" on a server window simply was not collected is a worse answer than no answer. Data-bearing results keep their own shape rather than the status envelope, so the warning rides in the payload. */ - var caveat = - baselineFacts.Count == 0 + /* + #3538 A2 extends the same warning to the case it never reached: a window that was only + PARTLY collected. Before the observed-time divisor, a comparison window with a three-hour + hole reported every rate at a quarter of its true value and this tool called that + "better" with no caveat at all — the empty-window arms above fire only when a side has NO + facts. The rates are now per observed time on both sides, so the deltas are honest; what + a reader still cannot know without being told is that one side speaks for an hour and the + other for four. Either side under the partial bar, or unobserved, earns its sentence + whether or not it produced facts — an idle hour the collector saw a quarter of is still a + quarter-seen window, and the empty-window sentence alone would send the reader to the + collection log without saying what they will find there. Both sides can earn one. + */ + var emptyCaveat = + baselineServerFacts.Count == 0 ? "The BASELINE window produced no facts at all, so every fact below counts as a new issue only because there was nothing to compare it against. Confirm collection covered the baseline window (get_collection_log) before reading new_issues as a regression." - : comparisonFacts.Count == 0 + : comparisonServerFacts.Count == 0 ? "The COMPARISON window produced no facts at all, so every fact below counts as a resolved issue only because there is nothing in the recent window to compare against. Confirm collection is running (get_collection_log) before reading resolved_issues as an improvement." : null; + var coverageCaveats = new List(2); + if (baselinePartial) + coverageCaveats.Add($"The BASELINE window was only partly collected: {baselineCoverage!.Describe()}. Its rates are per observed time, and its windowed facts are absent where nothing was observed."); + if (comparisonPartial) + coverageCaveats.Add($"The COMPARISON window was only partly collected: {comparisonCoverage!.Describe()}. Its rates are per observed time, and its windowed facts are absent where nothing was observed."); + if (coverageCaveats.Count > 0) + coverageCaveats.Add("A side that was not fully observed cannot be read as the whole period: a wait that is absent because the collector was down is not a wait that resolved. Confirm coverage (get_collection_log, get_collection_health) before reading worse/better/resolved_issues as change."); + + var caveat = emptyCaveat is null && coverageCaveats.Count == 0 + ? null + : string.Join(" ", new[] { emptyCaveat }.Concat(coverageCaveats).Where(s => s is not null)); + return JsonSerializer.Serialize(new { server = resolved.ServerName, - /* Null when both windows produced facts — the ordinary case, where nothing needs saying. */ + /* Null when both windows produced facts at full coverage — the ordinary case, where + nothing needs saying. */ caveat, + /* #3538 A3: what a verdict can and cannot carry, stated on every payload because the tool's + description is not in front of the reader when the numbers are. */ + reading = "Each row is banded by how far its VALUE moved on this server's own scale (band_source says which rule; band_rules states them), not by the severity formula's slope. One window against one window cannot show that a change caused anything: a same-hour-yesterday comparison at N=1 vs N=1 is a difference, not an experiment. Count families, not rows, to count causes.", + band_rules = ComparisonBanding.BandRulesPayload, baseline = new { start = baselineStart.ToString("o"), end = baselineEnd.ToString("o"), - fact_count = baselineFacts.Count + fact_count = baselineServerFacts.Count, + coverage = baselineCoverage?.ToPayload() }, comparison = new { start = comparisonStart.ToString("o"), end = comparisonEnd.ToString("o"), - fact_count = comparisonFacts.Count + fact_count = comparisonServerFacts.Count, + coverage = comparisonCoverage?.ToPayload() }, - summary = new - { - worse = comparisons.Count(c => c.status == "worse"), - better = comparisons.Count(c => c.status == "better"), - stable = comparisons.Count(c => c.status == "stable"), - new_issues = comparisons.Count(c => c.baseline_severity == null && c.comparison_severity > 0), - resolved_issues = comparisons.Count(c => c.baseline_severity > 0 && c.comparison_severity == null) - }, - facts = comparisons + summary = comparison.SummaryPayload(), + families = comparison.Families.Select(f => f.ToPayload()).ToList(), + plan_cache_churn = comparison.Churn.ToPayload(), + facts = comparison.Rows.Select(r => r.ToPayload()).ToList() }, McpHelpers.JsonOptions); } catch (Exception ex) @@ -394,7 +517,10 @@ public static async Task AuditConfig( try { - var facts = await analysisService.CollectAndScoreFactsAsync( + /* Coverage is discarded here on purpose (#3538 A2): this tool reads point-in-time + configuration facts, which are the latest row regardless of window, and a one-hour window + the collector missed changes nothing about what the server is configured to. */ + var (facts, _) = await analysisService.CollectAndScoreFactsAsync( resolved.ServerId, resolved.ServerName, 1); var factsByKey = facts.ToFactLookup(); @@ -602,7 +728,7 @@ public static async Task AuditConfig( } } - [McpServerTool(Name = "get_analysis_findings"), Description("Gets persisted findings from previous analysis runs without running a new analysis, deduplicated to one entry per diagnostic chain (story_path_hash + incident_id) - the engine re-persists the same stories every cycle, so each entry is the chain's LATEST occurrence plus occurrence stats (occurrences, first_seen, last_seen, peak_severity) spanning the window. Use this to review historical findings or check if anything has changed since the last analysis. A remediable finding carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the finding's persisted action and including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose. Set include_drilldown to also return each chain's persisted evidence rows (the specific plans/queries behind the finding, capped at write time with an explicit _truncation_note; null on findings persisted before the column existed).")] + [McpServerTool(Name = "get_analysis_findings"), Description("Gets persisted findings from previous analysis runs without running a new analysis, deduplicated to one entry per diagnostic chain (story_path_hash + incident_id) - the engine re-persists the same stories every cycle, so each entry is the chain's LATEST occurrence plus occurrence stats (occurrences, first_seen, last_seen, peak_severity) spanning the window. Use this to review historical findings or check if anything has changed since the last analysis. Each finding's confidence is an EVIDENCE score (see analyze_server): 0.20 for the fired symptom alone, plus corroboration from matched amplifier checks and chain depth. Rows persisted before this definition carried a PATH-LENGTH statistic under the same name, with a lone symptom at 1.0 — confidence_basis labels those rows path-shape (pre-#3538) and they must not be read as corroborated. A remediable finding carries remediation_command: the full copy-paste T-SQL remediation (identical to the viewer card), rendered from the finding's persisted action and including a two-sided risk-disclosure comment header on destructive changes; it is advisory only and never executed. A force-plan remediation additionally carries structured_remediation: the same decision as machine-readable fields — eligible, named blockers (parameter_sensitivity_cofired, secondary_replica_evidence), evidence numbers, and split force_sql/unforce_sql/verify_sql artifacts — so agents consume the verdict as data instead of parsing comment prose. Set include_drilldown to also return each chain's persisted evidence rows (the specific plans/queries behind the finding, capped at write time with an explicit _truncation_note; null on findings persisted before the column existed).")] public static async Task GetAnalysisFindings( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -697,6 +823,11 @@ stats. The store keeps every row — this shapes the read only. */ analysis_time = f.AnalysisTime.ToString("o"), severity = Math.Round(f.Severity, 2), confidence = Math.Round(f.Confidence, 2), + // #3538 A6: what the number rests on. Corroboration-derived since this change + // (matched amplifier share + path depth); a row persisted under the old path-shape + // formula is labelled as such, derived from the finding's own shape at read time + // because the store carries no version marker (no schema change). + confidence_basis = StoryConfidence.DescribeBasis(f.RootFactKey, f.Confidence, f.FactCount), category = f.Category, root_fact = new { key = f.RootFactKey, value = f.RootFactValue }, leaf_fact = f.LeafFactKey != null @@ -743,7 +874,7 @@ stats. The store keeps every row — this shapes the read only. */ } } - [McpServerTool(Name = "mute_analysis_finding"), Description("Mutes a finding pattern so it won't appear in future analysis runs. Use the story_path_hash from analyze_server or get_analysis_findings output. Muting is per-pattern, not per-occurrence — the same diagnostic chain won't be reported again until unmuted.")] + [McpServerTool(Name = "mute_analysis_finding"), Description("Mutes a finding pattern so it won't appear in future analysis runs. Use the story_path_hash from analyze_server or get_analysis_findings output. Muting is per-pattern, not per-occurrence — the same diagnostic chain won't be reported again until unmuted. The response reports what the write DID: registered says whether the mute row was stored, and matched_now is how many stored findings in the mute's scope carry that hash at this moment. status is \"muted\" when the mute is registered AND matched_now is at least 1; \"muted_unmatched\" when it is registered but matched_now is 0 — the pattern is not in the retained findings, which is what a mistyped hash looks like (the mute is kept, because the registry is by pattern and the pattern may return after retention purged its history, but check the hash against analyze_server output before relying on it); \"error\" when the row could not be written (nothing is muted).")] public static async Task MuteAnalysisFinding( DarlingAnalysisService analysisService, NpgsqlDataSource postgres, @@ -753,6 +884,11 @@ public static async Task MuteAnalysisFinding( { try { + if (string.IsNullOrWhiteSpace(story_path_hash)) + { + return JsonSerializer.Serialize(new { status = "invalid", message = "story_path_hash is required." }, McpHelpers.JsonOptions); + } + int? serverId = null; if (server_name != null) { @@ -768,14 +904,39 @@ public static async Task MuteAnalysisFinding( StoryPath = story_path_hash }; - await analysisService.MuteFindingAsync(finding, reason); + /* #3541 A14: report what happened, not what was asked. Before this the verb returned "muted" for any + hash — a mistyped one, a hash from another store, one whose INSERT the store swallowed — and the + agent walked away believing a pattern was silenced. Now: registered is the store's own answer, + and matched_now is counted AFTER the write so the two are read against the same moment. The + registry is a pattern registry (no row references a finding), so an unmatched hash is still + stored — legitimately, when a pattern's history has been purged — and the status names that case + instead of folding it into success. See PgFindingStore.CountStoredFindingsAsync. */ + var registered = await analysisService.MuteFindingAsync(finding, reason); + if (!registered) + { + return JsonSerializer.Serialize(new + { + status = "error", + message = "The mute could not be written to the monitoring store (see the service log); nothing is muted.", + story_path_hash, + server = server_name ?? "(all servers)", + registered = false, + }, McpHelpers.JsonOptions); + } + + var matchedNow = await analysisService.CountStoredFindingsAsync(serverId, story_path_hash); return JsonSerializer.Serialize(new { - status = "muted", + status = matchedNow > 0 ? "muted" : "muted_unmatched", story_path_hash, server = server_name ?? "(all servers)", - reason + reason, + registered = true, + matched_now = matchedNow, + note = matchedNow > 0 + ? $"The mute is registered; {matchedNow} stored finding(s) in this scope carry the hash and the pattern will be dropped from future analysis runs." + : "The mute is registered, but no stored finding in this scope carries this story_path_hash. If you copied it from analyze_server or get_analysis_findings it is still valid (the pattern will be dropped if it recurs); a hash from anywhere else may be mistyped and would mute nothing.", }, McpHelpers.JsonOptions); } catch (Exception ex) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs index 86c34f9e0..d512e9b66 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpTrendTools.cs @@ -11,6 +11,7 @@ using System.ComponentModel; using System.Linq; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using ModelContextProtocol.Server; using Npgsql; @@ -42,11 +43,26 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// it, while a server the collector has never sampled answers "unavailable" and says so outright. A /// response-shape change here must land in Lite's Mcp*Tools too, and vice versa. /// +/// +/// +/// The four reads over ROLLED tables route by retention tier and say what they served (#2353, #3541 +/// A2). query_stats, procedure_stats and query_store_stats have their raw rows +/// dropped at on a TimescaleDB store while the tools accept +/// hours_back up to seven days, so get_query_trend, get_query_duration_trend and +/// get_procedure_duration_trend read the hourly rollup for a window raw cannot hold and +/// get_query_store_duration_trend reads the corrected rollup for the region it has materialized (#2736) — +/// and every one of them publishes source, effective_start, effective_hours_back, +/// truncated and bucket, on the data path and on the empty one. "Quiet, widen hours_back" is +/// said only where widening can help: a window whose head the store no longer holds says that instead, +/// because the rows were dropped, not absent, and a wider window cannot recover them. Lite's twins publish +/// the same fields with Lite's truth (raw, unbounded within its retention), so the contract is one shape +/// across SKUs even where the depth differs. +/// /// [McpServerToolType] public sealed class DarlingMcpTrendTools { - [McpServerTool(Name = "get_memory_trend"), Description("Gets memory usage trend over time: total server memory, target memory, buffer pool, plan cache, and granted memory. Useful for identifying memory growth patterns or pressure periods.")] + [McpServerTool(Name = "get_memory_trend"), Description("Gets memory usage trend over time: total server memory, target memory, buffer pool, plan cache, and granted memory joined per point from the memory-grant series. total_granted_mb is null on points the grants series does not cover — a granted_note explains any gap; use get_memory_grants for grant detail. Useful for identifying memory growth patterns or pressure periods.")] public static async Task GetMemoryTrend( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -87,25 +103,44 @@ neither. A server that collected fine and was simply quiet in THIS window wants $"No memory stats have EVER been recorded for {resolved.ServerName}. This is not an empty window — the memory_stats collector has stored nothing at all for this server. Check that collection is running and that the server is enabled; get_memory_stats will be equally empty until it does."); } - var result = points.Select(p => new + var grants = await DarlingTrendReader.GetMemoryGrantTrendAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var granted = AlignGrantSeries( + points.Select(p => p.CollectionTime).ToArray(), + grants.Select(g => (g.CollectionTime, g.TotalGrantedMb)).ToArray()); + + var result = points.Select((p, i) => new { time = p.CollectionTime.ToString("o"), total_server_memory_mb = p.TotalServerMemoryMb, target_server_memory_mb = p.TargetServerMemoryMb, buffer_pool_mb = p.BufferPoolMb, plan_cache_mb = p.PlanCacheMb, - /* Lite carries total_granted_mb on its MemoryTrendPoint but GetMemoryTrendAsync (a - memory_stats-only read) leaves it unset — the grant overlay is a separate chart series. - Reproduced here as the same 0 placeholder for field-for-field parity with Lite's tool. */ - total_granted_mb = 0.0 + /* Joined from the memory-grant series (#3548): the nearest memory_grant_stats snapshot + within 30 seconds of this memory sample, SUM(granted_memory_mb) across pools — the same + series the viewer's Memory Overview overlay charts. null when no snapshot aligns, never + a fabricated 0: a literal zero read as "granted was 0 all window" and steered callers + away from memory grants at exactly the wrong moment (#3529). A genuine 0.0 still appears + when a snapshot exists with nothing granted. Field-for-field parity with Lite's tool, + which joins it the same way. */ + total_granted_mb = granted[i] }); - return JsonSerializer.Serialize(new - { - server = resolved.ServerName, - hours_back, - trend = result - }, McpHelpers.JsonOptions); + /* The note exists to explain null points; a fully covered window gets no note at all rather + than a null-valued key (JsonOptions writes nulls). */ + return granted.Any(v => v is null) + ? JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + granted_note = GrantGapNote, + trend = result + }, McpHelpers.JsonOptions) + : JsonSerializer.Serialize(new + { + server = resolved.ServerName, + hours_back, + trend = result + }, McpHelpers.JsonOptions); } catch (Exception ex) { @@ -113,6 +148,51 @@ neither. A server that collected fine and was simply quiet in THIS window wants } } + /// + /// Half the 1-minute cadence floor both collectors share (CollectorScheduleDefaults). The two + /// series each stamp their own capture clock per collector run, so same-cycle rows sit seconds + /// apart and can never be equality-joined — while a grants series on a slower cadence must NOT smear + /// onto every memory point. Within half the finest cadence, at most one snapshot can claim a point. + /// + private static readonly TimeSpan GrantJoinTolerance = TimeSpan.FromSeconds(30); + + /// Why a point is null, stated once per payload — and only when a null point exists. + private const string GrantGapNote = + "total_granted_mb is null where no memory-grant snapshot lies within 30 seconds of the memory sample — the memory_grant_stats series is collected on its own schedule, so a gap means no grant measurement at that moment, not zero granted. Use get_memory_grants for the full grant picture."; + + /// + /// Nearest-match join of the memory-grant series onto the memory-trend points (#3548): for each trend + /// point, the closest grants snapshot within , else null — no grant + /// measurement at that moment, which is not the same claim as a genuine 0.0 from a snapshot with + /// nothing granted. Both inputs are time-ascending (both reads ORDER BY collection_time), so one + /// forward pointer finds every nearest neighbor. Twin of Lite's + /// McpMemoryTools.AlignGrantSeries — the two must stay in step so both SKUs join the same way. + /// + private static double?[] AlignGrantSeries( + DateTime[] trendTimes, + (DateTime Time, double TotalGrantedMb)[] grants) + { + var aligned = new double?[trendTimes.Length]; + if (grants.Length == 0) return aligned; + + var g = 0; + for (var t = 0; t < trendTimes.Length; t++) + { + var target = trendTimes[t]; + while (g + 1 < grants.Length && (grants[g + 1].Time - target).Duration() <= (grants[g].Time - target).Duration()) + { + g++; + } + + if ((grants[g].Time - target).Duration() <= GrantJoinTolerance) + { + aligned[t] = grants[g].TotalGrantedMb; + } + } + + return aligned; + } + [McpServerTool(Name = "get_perfmon_trend"), Description("Gets a time-series trend for a specific performance counter. Use get_perfmon_stats first to see available counter names.")] public static async Task GetPerfmonTrend( NpgsqlDataSource postgres, @@ -279,7 +359,17 @@ public static async Task GetQueryTrend( try { var now = windowEnd; - var history = await DarlingTrendReader.GetQueryHistoryAsync(postgres, resolved.ServerId, database_name, query_hash, now.AddHours(-hours_back), now); + + /* #3541 A2: the store's measured shape rides along so the tier decision degrades to what exists + and to what has materialized (see DarlingTrendReader.ResolveTier) — the age-only #2353 rule + answered 42P01 on a plain-PostgreSQL store for any window past four days, and read an empty + rollup while raw still held the rows on a never-backfilled one. Cached per data source and + shared with the composer, so this is not a probe per call. */ + var (rollups, coverage) = await ComposeStoreAvailability.GetRollupsAsync(postgres, CancellationToken.None); + var history = await DarlingTrendReader.GetQueryHistoryAsync( + postgres, resolved.ServerId, database_name, query_hash, now.AddHours(-hours_back), now, + hourlyAvailable: rollups.QueryGrainHourly, + coverage: coverage.For(TimescaleSupport.QueryStatsHourlyView, TimescaleSupport.QueryStatsDailyView)); var rows = history.Points; if (rows.Count == 0) { @@ -348,7 +438,7 @@ concluding the query did not run. */ } } - [McpServerTool(Name = "get_query_duration_trend"), Description("Gets a time-series of average query duration over time. Useful for spotting overall performance degradation or improvement trends across all queries.")] + [McpServerTool(Name = "get_query_duration_trend"), Description("Gets a time-series of average query duration over time. Useful for spotting overall performance degradation or improvement trends across all queries. On the per-collection (raw) route every point is a rate over the gap since the PREVIOUS collection, so the window's first collection - which has no previous one to difference against - carries null rates: unknowable, never reported as 0 (unrated_points counts them, unrated_note says why). The hourly rollup route divides by the bucket width and has no such point.")] public static async Task GetQueryDurationTrend( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -364,31 +454,44 @@ public static async Task GetQueryDurationTrend( try { var now = windowEnd; - var points = await DarlingTrendReader.GetQueryDurationTrendAsync(postgres, resolved.ServerId, now.AddHours(-hours_back), now); - if (points.Count == 0) + var startUtc = now.AddHours(-hours_back); + + /* + #3541 A2: route by the tier that can actually serve the window. This read went to raw + query_stats only, whose rows a TimescaleDB store drops at four days, while hours_back + accepts 168 — so a 7-day request came back as 4 days under a label saying 7, and when + nothing survived the empty branch called the window "genuinely quiet" and advised widening + it, which cannot help with rows that were dropped. Same ladder as get_query_trend + (DarlingTrendReader.ResolveTier), measured by the reader against the WALL CLOCK because + retention drops by age, never by where a point sits inside the requested window — the + as_of anchor decides the window, not how old its rows are. + */ + var (rollups, coverage) = await ComposeStoreAvailability.GetRollupsAsync(postgres, CancellationToken.None); + var route = DarlingTrendReader.ResolveQueryDurationTrendRoute(startUtc, rollups, coverage); + var result = await DarlingTrendReader.GetQueryDurationTrendAsync(postgres, resolved.ServerId, startUtc, now, route); + + if (result.Points.Count == 0) { - /* Same two states again. The probe reads the BASE query_stats table because this trend - does — v_query_stats is the payload-resolving view on a V38+ store, and probing a - different relation from the one the read walks is how an existence probe ends up - reporting the wrong branch. */ var gated = await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "query_stats"); if (gated != null) { return gated; } - return await DarlingTrendReader.HasAnyQueryStatAsync(postgres, resolved.ServerId) - ? McpHelpers.Status( - "empty", - $"No query samples recorded for {resolved.ServerName} in the last {hours_back} hour(s). This server HAS collected query stats before, so this window is genuinely quiet rather than broken — widen hours_back to find the most recent samples.") - : McpHelpers.Status( - "unavailable", - $"No query stats have EVER been recorded for {resolved.ServerName}. This is not an empty window — the query_stats collector has stored nothing at all for this server. Check that collection is running and that the server is enabled; get_top_queries_by_cpu will be equally empty until it does."); + /* The raw probe reads the BASE query_stats table because the raw trend does — v_query_stats + is the payload-resolving view on a V38+ store, and probing a different relation from the + one the read walks is how an existence probe ends up reporting the wrong branch. On the + hourly route the rollup is probed too, because a server whose raw rows have all aged out + is not a server nothing was ever stored for. */ + return await EmptyRoutedTrendAsync( + DarlingTrendReader.HasAnyQueryStatAsync(postgres, resolved.ServerId), + postgres, resolved.ServerId, resolved.ServerName, hours_back, startUtc, now, route, "query", + "Check that collection is running and that the server is enabled; get_top_queries_by_cpu will be equally empty until it does."); } /* The two siblings below serialize through the SAME helper, so the three Performance-Trends reads cannot advertise three different field sets for one shape. */ - return SerializeTrend(resolved.ServerName, hours_back, points); + return SerializeTrend(resolved.ServerName, hours_back, result.Points, DescribeRoute(result, now)); } catch (Exception ex) { @@ -396,7 +499,7 @@ reporting the wrong branch. */ } } - [McpServerTool(Name = "get_procedure_duration_trend"), Description("Gets a time-series of stored-procedure elapsed time per second and executions per second over time, summed across every procedure. The sibling of get_query_duration_trend, and NOT a duplicate of it: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs. This charges the whole call to the procedure. Read the two together to tell an ad-hoc SQL regression from a procedure regression.")] + [McpServerTool(Name = "get_procedure_duration_trend"), Description("Gets a time-series of stored-procedure elapsed time per second and executions per second over time, summed across every procedure. The sibling of get_query_duration_trend, and NOT a duplicate of it: query_stats attributes a procedure's work to the individual statements inside it, so a procedure that got slower is smeared across however many statements it runs. This charges the whole call to the procedure. Read the two together to tell an ad-hoc SQL regression from a procedure regression. On the per-collection (raw) route every point is a rate over the gap since the PREVIOUS collection, so the window's first collection - which has no previous one to difference against - carries null rates: unknowable, never reported as 0 (unrated_points counts them, unrated_note says why). The hourly rollup route divides by the bucket width and has no such point.")] public static async Task GetProcedureDurationTrend( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -412,10 +515,14 @@ public static async Task GetProcedureDurationTrend( try { var now = windowEnd; - var points = await DarlingTrendReader.GetProcedureDurationTrendAsync( - postgres, resolved.ServerId, now.AddHours(-hours_back), now); + var startUtc = now.AddHours(-hours_back); - if (points.Count == 0) + /* #3541 A2 — the same routing as get_query_duration_trend, over the procedure pair. */ + var (rollups, coverage) = await ComposeStoreAvailability.GetRollupsAsync(postgres, CancellationToken.None); + var route = DarlingTrendReader.ResolveProcedureDurationTrendRoute(startUtc, rollups, coverage); + var result = await DarlingTrendReader.GetProcedureDurationTrendAsync(postgres, resolved.ServerId, startUtc, now, route); + + if (result.Points.Count == 0) { var gated = await DarlingEngineCapability.NotCollectedStatusAsync(postgres, resolved.ServerId, resolved.ServerName, "procedure_stats"); if (gated != null) @@ -423,13 +530,13 @@ public static async Task GetProcedureDurationTrend( return gated; } - return await EmptyTrendAsync( + return await EmptyRoutedTrendAsync( DarlingTrendReader.HasAnyProcedureStatAsync(postgres, resolved.ServerId), - resolved.ServerName, hours_back, "stored-procedure", + postgres, resolved.ServerId, resolved.ServerName, hours_back, startUtc, now, route, "stored-procedure", "Check that collection is running and that the server is enabled. A server that genuinely runs no stored procedures also lands here, and that is a real answer rather than a fault."); } - return SerializeTrend(resolved.ServerName, hours_back, points); + return SerializeTrend(resolved.ServerName, hours_back, result.Points, DescribeRoute(result, now)); } catch (Exception ex) { @@ -437,7 +544,7 @@ public static async Task GetProcedureDurationTrend( } } - [McpServerTool(Name = "get_query_store_duration_trend"), Description("Gets a time-series of Query Store duration per second and executions per second over time, summed across every query. Where get_query_duration_trend reads the plan cache and loses everything an eviction or a restart takes with it, this reads Query Store, which persists per interval - so it is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work ran.")] + [McpServerTool(Name = "get_query_store_duration_trend"), Description("Gets a time-series of Query Store duration per second and executions per second over time, summed across every query. Where get_query_duration_trend reads the plan cache and loses everything an eviction or a restart takes with it, this reads Query Store, which persists per interval - so it is the series that survives a failover and the one to reach for when a regression is older than the cache. Each interval is counted once, at the hour the work ran. Every point is a rate over the gap since the PREVIOUS point (interval start or rollup bucket), so the window's first point - which has no previous one to difference against - carries null rates: unknowable, never reported as 0 (unrated_points counts them, unrated_note says why).")] public static async Task GetQueryStoreDurationTrend( NpgsqlDataSource postgres, [Description("Server name or display name.")] string? server_name = null, @@ -461,11 +568,15 @@ public static async Task GetQueryStoreDurationTrend( window width on a large store — so the materialized window portion is served from query_store_stats_corrected_hourly and only the unmaterialized tail is ranked raw. The payload discloses the routing (grain and boundary) rather than presenting the two regions - as one estimator. + as one estimator. The same route is what keeps this sibling honest about DEPTH (#3541 A2): + raw query_store_stats is dropped at four days only once its rollups cover it (the #1680 + arming gate), so wherever raw is short the rollup is the tier holding the history, and a + raw-only route means raw is complete. */ var route = await QueryStoreTrendRouting.ResolveAsync(postgres); var points = await DarlingTrendReader.GetQueryStoreDurationTrendAsync( postgres, resolved.ServerId, startUtc, now, route); + var disclosure = DescribeQueryStoreRoute(route, points, startUtc, now); if (points.Count == 0) { @@ -488,18 +599,40 @@ every database on the instance. A server with no Query Store data is not a serve */ if (route.UseRollup && route.RollupFloorUtc is DateTime floor && now < floor) { - return McpHelpers.Status( + return EmptyStatus( + "empty", + $"The requested window ends before {floor:o}, the oldest hour the corrected Query Store rollup has materialized. This read serves history from query_store_stats_corrected_hourly rather than ranking the raw Query Store slab (#2736), so windows before that floor come back empty even when rows were collected — run --backfill-rollups to materialize deeper history.", + disclosure); + } + + var everSampled = await DarlingTrendReader.HasAnyQueryStoreStatAsync(postgres, resolved.ServerId); + if (!everSampled) + { + return EmptyStatus( + "unavailable", + NeverSampledMessage(resolved.ServerName, "Query Store", + "Query Store may be OFF on this server's databases — that, not an absence of slow queries, is the usual cause. Check QUERY_STORE = ON per database, then that collection is running for this server."), + disclosure); + } + + /* + Sampled, nothing in the window — but a window whose HEAD sits below the rollup's floor is + only quiet in the part the rollup has reached. The unserved head is named so "widen" is + read for what it can do (find the most recent samples) and not for what it cannot (reach + history nothing has materialized). + */ + if (route.UseRollup && route.RollupFloorUtc is DateTime head && head > startUtc) + { + return EmptyStatus( "empty", - $"The requested window ends before {floor:o}, the oldest hour the corrected Query Store rollup has materialized. This read serves history from query_store_stats_corrected_hourly rather than ranking the raw Query Store slab (#2736), so windows before that floor come back empty even when rows were collected — run --backfill-rollups to materialize deeper history."); + $"No Query Store samples were recorded for {resolved.ServerName} between {head:o} — the oldest hour the corrected rollup has materialized — and the end of the window. This server HAS been sampled before, so that stretch is genuinely quiet; the part of the window before {head:o} is unserved rather than quiet (the rollup has not materialized it and this read no longer ranks the raw slab for it, #2736) — run --backfill-rollups to materialize it. Widening hours_back finds newer samples only; it cannot reach the unserved head.", + disclosure); } - return await EmptyTrendAsync( - DarlingTrendReader.HasAnyQueryStoreStatAsync(postgres, resolved.ServerId), - resolved.ServerName, hours_back, "Query Store", - "Query Store may be OFF on this server's databases — that, not an absence of slow queries, is the usual cause. Check QUERY_STORE = ON per database, then that collection is running for this server."); + return EmptyStatus("empty", QuietWindowMessage(resolved.ServerName, hours_back, "Query Store"), disclosure); } - return SerializeTrend(resolved.ServerName, hours_back, points, DescribeQueryStoreRoute(route, startUtc)); + return SerializeTrend(resolved.ServerName, hours_back, points, disclosure); } catch (Exception ex) { @@ -507,6 +640,98 @@ every database on the instance. A server with no Query Store data is not a serve } } + /// + /// What a tiered trend says about itself beside its points (#2353's vocabulary, applied to the trio by + /// #3541 A2): which tier served (source), where the served series actually begins + /// (effective_start, effective_hours_back), whether that head sits later than asked + /// (truncated), the grain of a point (bucket), and the prose a degraded tier owes the reader + /// (aggregate_note, null on raw). Routing is the Query Store sibling's #2736 seam detail and + /// is emitted only when the rollup route was taken — the other two have no seam to describe. + /// + private sealed record TrendDisclosure( + string Source, DateTime EffectiveStartUtc, DateTime WindowEndUtc, bool Truncated, string Bucket, + string? AggregateNote, Dictionary? Routing = null) + { + /// The disclosure keys in the order they are written, so the data envelope and the empty + /// envelope carry the same block in the same shape. + public void WriteTo(Dictionary envelope) + { + envelope["source"] = Source; + /* Written in the store's own frame (naive UTC, Kind=Unspecified) so it prints exactly like the + points' `time` beside it. The requested start arrives Kind=Utc from ValidateWindow and would + otherwise carry a trailing Z the points do not, which reads as two frames in one payload. */ + envelope["effective_start"] = DateTime.SpecifyKind(EffectiveStartUtc, DateTimeKind.Unspecified).ToString("o"); + envelope["effective_hours_back"] = Math.Round((WindowEndUtc - EffectiveStartUtc).TotalHours, 1); + envelope["truncated"] = Truncated; + envelope["bucket"] = Bucket; + envelope["aggregate_note"] = AggregateNote; + if (Routing is not null) + { + envelope["routing"] = Routing; + } + } + } + + /// The prose the hourly tier owes a reader of the query-stats and procedure-stats trends. + private static string HourlyAggregateNote(DarlingTrendReader.DurationTrendRoute route) => + $"Served from the hourly rollup ({route.HourlyView}) because the requested window reaches past the raw tier's " + + $"{TimescaleSupport.RawRetentionSpan.TotalDays:0}-day retention. Each point is one hour's summed work divided by " + + "3,600 seconds, so an hour the collector covered only partly reads LOW, never high; the rollup trails the " + + "clock by up to two hours (the current hour is never materialized and the previous one lands on the next refresh)."; + + /// The routed trio's disclosure, from the route and what the read returned. + private static TrendDisclosure DescribeRoute(DarlingTrendReader.DurationTrendResult result, DateTime windowEndUtc) => + new( + result.Route.Source, + result.EffectiveStartUtc, + windowEndUtc, + result.Truncated, + result.Route.Tier == RetentionTier.Raw ? "per-collection" : "1 hour", + result.Route.Tier == RetentionTier.Raw ? null : HourlyAggregateNote(result.Route)); + + /// + /// The routed trio's disclosure for an EMPTY answer: the tier is described from its floor rather than from + /// a first point it does not have. Where the tier's oldest instant is measured and sits above the requested + /// start, that instant is what the read could reach — effective_start says so and truncated + /// follows , exactly as it would had a point been there. + /// Unmeasured, the requested start stands (#2353's rule: an empty result narrows nothing it cannot describe). + /// A floor beyond the window's END is clamped to the end: the tier held none of the window, and + /// effective_hours_back reads 0 rather than a negative span. + /// + private static TrendDisclosure DescribeEmptyRoute(DarlingTrendReader.DurationTrendRoute route, DateTime startUtc, DateTime windowEndUtc) + { + var reach = route.Tier == RetentionTier.Raw ? route.Coverage.RawOldestUtc : route.Coverage.HourlyFloorUtc; + var (effectiveStart, truncated) = DescribeEmptyCoverage(reach, startUtc, windowEndUtc); + + return new TrendDisclosure( + route.Source, effectiveStart, windowEndUtc, truncated, + route.Tier == RetentionTier.Raw ? "per-collection" : "1 hour", + route.Tier == RetentionTier.Raw ? null : HourlyAggregateNote(route)); + } + + /// + /// Coverage for an EMPTY answer, from the tier's measured floor rather than from a first point it does not + /// have. Three shapes: a floor at or before the start (or unmeasured) reached the whole window, so the + /// requested start stands and nothing is truncated (#2353's rule — an empty result narrows nothing it + /// cannot describe); a floor inside the window is where the tier could first have answered, and + /// judges it by the shared slack exactly as it would a + /// first point; a floor BEYOND the window's end means the tier held none of the window, so the served + /// span is honestly zero (effective_start clamped to the end) and the answer is truncated outright + /// — the slack is for a head that arrived late, not for a window that never arrived at all. + /// + private static (DateTime EffectiveStartUtc, bool Truncated) DescribeEmptyCoverage( + DateTime? tierFloorUtc, DateTime startUtc, DateTime windowEndUtc) + { + if (tierFloorUtc is not DateTime floor || floor <= startUtc) + { + return (startUtc, false); + } + + return floor > windowEndUtc + ? (windowEndUtc, true) + : DarlingTrendReader.DescribeCoverage(floor, startUtc); + } + /// /// The one payload shape the three Performance-Trends siblings share, so a caller can chart them on one /// axis without learning three field names. @@ -514,86 +739,200 @@ every database on the instance. A server with no Query Store data is not a serve /// shipped truncated to an integer, which on a quiet server turns 0.4 executions a second into a /// reported ZERO - an idle server, when the truth was a slow one. It is kept so a consumer reading it /// does not break; read executions_per_second. + /// value and elapsed_ms_per_second are the same quantity too (#3541): a bare + /// value named no unit, and a reasoning agent charted it as whatever it guessed. The named field is + /// the one to read; value stays for the consumer already reading it, on the precedent above. + /// The disclosure block sits between the request echo and the points on every sibling, and the + /// empty envelope () carries the same block — the same six keys in the same + /// order whichever branch answered, which is what lets a caller read source without first checking + /// whether it got data. /// private static string SerializeTrend( string serverName, int hours_back, List points, - object? source = null) + TrendDisclosure disclosure) { - var trend = points.Select(p => new + var envelope = new Dictionary + { + ["server"] = serverName, + ["hours_back"] = hours_back, + }; + disclosure.WriteTo(envelope); + /* #3541 A12: a point with no rate is published as null, never as 0, and the envelope says how many + and why. On the raw route the window's first collection has no previous one to difference + against; the hourly route divides by the bucket width and produces none. */ + var unrated = points.Count(p => !p.HasRate); + envelope["unrated_points"] = unrated; + envelope["unrated_note"] = unrated == 0 + ? null + : $"{unrated} point(s) carry null rates: a per-collection rate is the work since the PREVIOUS collection divided by the seconds between them, and the window's first collection has no previous one inside the window (a collection landing in the same second as its predecessor has no denominator either). Unknowable is not 0 — the point is kept so effective_start is the first collection the store held, and its rates are null."; + envelope["trend"] = points.Select(p => new { time = p.CollectionTime.ToString("o"), value = p.Value, + elapsed_ms_per_second = p.Value, execution_count = p.ExecutionCount, executions_per_second = p.ExecutionsPerSecond, }); - /* `source` is additive and only the Query Store sibling sends one (#2736) — the shared trend field - set stays identical across the three siblings, which is this helper's whole job. */ - return source is null - ? JsonSerializer.Serialize(new { server = serverName, hours_back, trend }, McpHelpers.JsonOptions) - : JsonSerializer.Serialize(new { server = serverName, hours_back, source, trend }, McpHelpers.JsonOptions); + return JsonSerializer.Serialize(envelope, McpHelpers.JsonOptions); } /// - /// The routing disclosure get_query_store_duration_trend attaches when the corrected rollup served part - /// of the window (#2736) — which relation served which region, at which grain, and (when the requested - /// window reaches below the rollup's materialized floor) what was NOT served and why. A degraded or - /// partial answer must label itself; the raw-only route attaches nothing because it is the original - /// single-estimator read. + /// with the trend's disclosure block beside status and + /// message: an empty answer still says which tier it read and how far that tier reached, because + /// "nothing here" means different things from a four-day raw tier and a ninety-day rollup. /// - private static Dictionary? DescribeQueryStoreRoute( - QueryStoreTrendRouting.QueryStoreTrendRoute route, DateTime windowStartUtc) + private static string EmptyStatus(string status, string message, TrendDisclosure disclosure) { + var envelope = new Dictionary + { + ["status"] = status, + ["message"] = message, + }; + disclosure.WriteTo(envelope); + return JsonSerializer.Serialize(envelope, McpHelpers.JsonOptions); + } + + /// + /// The routing disclosure get_query_store_duration_trend attaches (#2736): which relation served which + /// region, at which grain, and (when the requested window reaches below the rollup's materialized floor) + /// what was NOT served and why. A degraded or partial answer must label itself. Since #3541 A2 the block + /// speaks the shared vocabulary — source is the tier word (rollup+raw or raw), + /// aggregate_note the grain prose, and the seam's instants live under routing — so the three + /// siblings' envelopes carry the same keys with the same types. The raw-only route attaches no + /// routing block because it is the original single-estimator read. + /// + private static TrendDisclosure DescribeQueryStoreRoute( + QueryStoreTrendRouting.QueryStoreTrendRoute route, List points, + DateTime windowStartUtc, DateTime windowEndUtc) + { + /* Coverage from the first point; for an EMPTY rollup-routed answer whose head sits below the floor, + from the floor — the instant the tier could first have answered, the same rule DescribeEmptyRoute + applies to the other two siblings. */ + var (effectiveStart, truncated) = points.Count > 0 + ? DarlingTrendReader.DescribeCoverage(points[0].CollectionTime, windowStartUtc) + : DescribeEmptyCoverage(route.UseRollup ? route.RollupFloorUtc : null, windowStartUtc, windowEndUtc); + if (!route.UseRollup) { - return null; + return new TrendDisclosure("raw", effectiveStart, windowEndUtc, truncated, "per-interval", null); } - var source = new Dictionary + var routing = new Dictionary { - ["tier"] = "rollup+raw", ["rollup"] = TimescaleSupport.QueryStoreStatsCorrectedHourlyView, ["raw_from"] = route.RawStartUtc.ToString("o"), - ["note"] = - "Points before raw_from are 1-hour buckets from the corrected Query Store rollup (#1849): " + - "bucketed on the COLLECTION hour, deduped at interval grain, with an interval whose " + - "snapshots straddle an hour boundary contributing to both adjacent buckets. Points at or " + - "after raw_from are raw Query Store intervals deduped to their final snapshot and placed " + - "at interval_start_time_utc.", }; if (route.RollupFloorUtc is DateTime floor && floor > windowStartUtc) { - source["unserved_before"] = floor.ToString("o"); - source["unserved_note"] = + routing["unserved_before"] = floor.ToString("o"); + routing["unserved_note"] = "The rollup has not materialized history before unserved_before, and this read no longer " + "falls back to ranking the raw slab for it (that rank is the #2736 timeout) — points " + "before that instant are missing, not zero. Run --backfill-rollups to materialize deeper " + "history."; } - return source; + return new TrendDisclosure( + "rollup+raw", effectiveStart, windowEndUtc, truncated, + "1 hour before routing.raw_from, per-interval from it", + "Points before routing.raw_from are 1-hour buckets from the corrected Query Store rollup (#1849): " + + "bucketed on the COLLECTION hour, deduped at interval grain, with an interval whose " + + "snapshots straddle an hour boundary contributing to both adjacent buckets. Points at or " + + "after routing.raw_from are raw Query Store intervals deduped to their final snapshot and placed " + + "at interval_start_time_utc.", + routing); } + /// The two-state sentence pair the trio shares with Lite's twins, word for word (#2484, #2485). + private static string QuietWindowMessage(string serverName, int hours_back, string what) => + $"No {what} samples were recorded for {serverName} in the last {hours_back} hour(s). This server HAS been sampled before, so this window is genuinely quiet rather than broken — widen hours_back to find the most recent samples."; + + private static string NeverSampledMessage(string serverName, string what, string checkThis) => + $"No {what} samples have EVER been recorded for {serverName}. This is not an empty window — nothing at all has been stored for this server, so it is NOT a quiet server. {checkThis}"; + /// - /// The two-branch empty answer the two new Performance-Trends siblings share (#2484), phrased to match - /// the one get_query_duration_trend already ships (#2485) so the three reads tell one story. - /// Zero points is two facts wanting opposite responses. A server that HAS been sampled and was - /// quiet in this window wants the window widened; a server that has never been sampled wants somebody - /// to go look at why. Both are literally "no trend data", and the probe — one LIMIT 1 against the same - /// table the trend reads, run only on this path — is what separates them. + /// The empty answer for the two ROUTED Performance-Trends siblings (#2484, #2485, #3541 A2), in three + /// states rather than the two the pre-routing version knew. + /// Zero points is still two facts wanting opposite responses — sampled-and-quiet wants the window + /// widened, never-sampled wants somebody to look at collection — and the probe (one LIMIT 1 against the + /// relation the trend reads, run only on this path; on the hourly route the rollup too, see + /// ) still separates them. The third state is + /// the one this fix exists for: sampled, nothing in the window, and the tier that was read does NOT + /// reach the window's start. That is neither quiet nor broken; the rows are DROPPED (raw past its + /// retention) or NOT MATERIALIZED (a rollup whose floor sits above the start), and "widen hours_back" + /// is exactly the wrong advice, because a wider window reaches further into what the tier does not + /// hold. The message names the tier, its measured reach, and the remedy that can work. + /// "Quiet, widen" is kept word for word with Lite's twin for the state where it is true: the tier + /// reaches the whole window (a plain-PostgreSQL store, where nothing drops raw; a rollup whose floor + /// covers the start; a measured raw oldest at or before the start). /// - private static async Task EmptyTrendAsync( - Task probe, string serverName, int hours_back, string what, string checkThis) + private static async Task EmptyRoutedTrendAsync( + Task rawProbe, NpgsqlDataSource postgres, int serverId, string serverName, int hours_back, + DateTime startUtc, DateTime windowEndUtc, DarlingTrendReader.DurationTrendRoute route, + string what, string checkThis) { - var everSampled = await probe; - return everSampled - ? McpHelpers.Status( + var disclosure = DescribeEmptyRoute(route, startUtc, windowEndUtc); + + if (!await DarlingTrendReader.HasAnySampleOnRouteAsync(postgres, rawProbe, route, serverId)) + { + return EmptyStatus("unavailable", NeverSampledMessage(serverName, what, checkThis), disclosure); + } + + if (route.Tier == RetentionTier.Hourly) + { + var floor = route.Coverage.HourlyFloorUtc; + var reach = floor is DateTime f + ? (f <= startUtc + ? $"The rollup has materialized history from {f:o}, which covers the whole window, so nothing was recorded for this server in it in the tier searched — a quiet stretch, or a refresh gap inside the rollup (check get_collection_health)." + : $"The rollup has materialized history only from {f:o}; the part of the window before that is UNSERVED rather than quiet, and the raw rows for it were dropped by retention. Run --backfill-rollups to materialize deeper history.") + : "The rollup has materialized NOTHING yet, so this is a coverage gap rather than a quiet server: the raw rows for this span were dropped by retention and only --backfill-rollups can materialize them."; + + return EmptyStatus( "empty", - $"No {what} samples were recorded for {serverName} in the last {hours_back} hour(s). This server HAS been sampled before, so this window is genuinely quiet rather than broken — widen hours_back to find the most recent samples.") - : McpHelpers.Status( - "unavailable", - $"No {what} samples have EVER been recorded for {serverName}. This is not an empty window — nothing at all has been stored for this server, so it is NOT a quiet server. {checkThis}"); + $"No {what} samples in the hourly rollup ({route.HourlyView}) for {serverName} over the last {hours_back} hour(s), from {startUtc:o}. The window reaches past the raw tier's {TimescaleSupport.RawRetentionSpan.TotalDays:0}-day retention, so this read served the rollup, not raw. {reach} Widening hours_back cannot help here.", + disclosure); + } + + /* + Raw route. Honest "quiet" needs raw to reach the window's start. Raw's oldest row is MEASURED + (the coverage probe reads min(collection_time) on every rolled table, rollups or not), so where it + sits at or before the start the window is fully served and quiet is the truth. Where it is later, + the head is unserved for one of two reasons the reader must not confuse: the window predates the + store's own history (a young store, any engine — nothing was dropped, nothing ever existed), or + retention dropped it (this grain's rollup exists, so its raw purge can be armed; the window is past + the raw horizon; and coverage routed here because the rollup has materialized LESS than raw holds + — the only way a past-horizon window routes to raw when the rollup exists). Unmeasured (an empty + table), the horizon decides, and only where retention applies to this grain at all: a grain whose + rollup is missing has its purge held by the arming gate, so raw is complete there. + */ + var pastRawHorizon = route.RawRetentionApplies && startUtc < route.ResolvedAtUtc - TimescaleSupport.RawRetentionSpan; + var rawReaches = route.RawReaches(startUtc) ?? !pastRawHorizon; + if (rawReaches) + { + return EmptyStatus("empty", QuietWindowMessage(serverName, hours_back, what), disclosure); + } + + if (!pastRawHorizon && route.Coverage.RawOldestUtc is DateTime storeOldest) + { + return EmptyStatus( + "empty", + $"No {what} samples were recorded for {serverName} between {storeOldest:o} — the oldest {route.RawTable} row this store holds for any server — and the end of the window. This server HAS been sampled before, so that stretch is genuinely quiet; the part of the window before {storeOldest:o} predates the store's history rather than being quiet, and widening hours_back cannot reach it.", + disclosure); + } + + /* Past the horizon on the raw route with the rollup present: coverage put the read here because the + rollup has materialized less than raw holds (the #1759 held-purge shape), so raw's reach is the + store's reach and the rollup is the remedy. */ + var oldest = route.Coverage.RawOldestUtc is DateTime o + ? $"The raw tier's oldest row for any server is {o:o}" + : $"The raw tier keeps about {TimescaleSupport.RawRetentionSpan.TotalDays:0} days"; + + return EmptyStatus( + "empty", + $"No {what} samples were recorded for {serverName} in the part of the last {hours_back} hour(s) that the raw tier still holds. {oldest}, and the window as requested starts at {startUtc:o} — the part before raw's reach is UNSERVED rather than quiet, because the hourly rollup ({route.HourlyView}) that would serve deeper history has materialized less than raw holds. This server HAS been sampled before. Widening hours_back reaches further into what raw no longer holds and cannot help; run --backfill-rollups to materialize the rollup, which is what serves deeper history.", + disclosure); } /// diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs index 093ec0d0c..f0bf31d62 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMemoryGrantReader.cs @@ -25,35 +25,70 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// get_resource_semaphore is the semaphore/ceiling lens: one row per (resource_semaphore_id, pool_id) with the /// full workspace-memory sizing — target / max_target (the hard ceiling) / total / /// available / granted / used plus grantee/waiter/timeout/forced counts and deltas. It -/// carries max_target_memory_mb (present in the store, the Dashboard tool omitted it) and drops the -/// Dashboard's sample_interval_seconds (Darling's delta collector stores no sample interval). get_memory_grants +/// carries max_target_memory_mb (present in the store, the Dashboard tool omitted it) and, since V128 +/// (#3540), the Dashboard's sample_interval_seconds too — the measured seconds the two deltas accrued +/// over, which this tool dropped while the collector stored none; 0 is the calculator's "no delta +/// knowable" marker (a restart, not a quiet semaphore) and the tool reports it as null, and a pre-V128 +/// row that never recorded one reads as null with interval_known = false. get_memory_grants /// is Lite's pool-detail lens: the sizing + activity SUMMED per pool (Lite's GetMemoryGrantChartDataAsync /// shape). Every SQL string is a public const so Darling.Tests can pin the dialect + columns without a live Postgres. /// +/// +/// #3541 A10: the window is READ, not merely searched. Both tools accept hours_back, and +/// until this change the only thing it did was bound the search for the newest snapshot — a grant storm three +/// hours ago (waiters in the dozens, timeouts climbing) was invisible behind a calm latest row while the +/// parameter read as a window. The *WindowSql reads below aggregate the SAME rows the latest read picks +/// its snapshot from, per semaphore or per pool: the peak waiter count and WHEN it peaked, the peak grant, the +/// floor of available workspace, and the SUM of the per-interval timeout / forced-grant deltas across every +/// snapshot in the window. Summing the deltas is the whole reason the collector stores them; it needs no +/// interval arithmetic (the "naked family" #3540 rung that is still to land stores the interval the deltas +/// accrued over, which is a RATE question, not this one) and a restart's fabricated 0 delta adds 0. The +/// latest snapshot is served BESIDE the window figures, stamped, so a caller can tell "calm now" from "calm +/// all window". /// internal static class DarlingMemoryGrantReader { /* ─────────────────────────── result rows ─────────────────────────── */ /// One resource semaphore at the latest snapshot — the workspace-memory ceiling lens. + /// #3540 (V128): the measured seconds the two deltas accrued over; + /// 0 is the calculator's "no delta knowable" marker (first sighting, counter reset, a gap past the + /// policy — a restart, not a quiet semaphore); null is a pre-V128 row that never recorded one. public sealed record ResourceSemaphoreRow( DateTime CollectionTime, short ResourceSemaphoreId, int PoolId, double TargetMemoryMb, double MaxTargetMemoryMb, double TotalMemoryMb, double AvailableMemoryMb, double GrantedMemoryMb, double UsedMemoryMb, int GranteeCount, int WaiterCount, long TimeoutErrorCount, long ForcedGrantCount, - long TimeoutErrorCountDelta, long ForcedGrantCountDelta); + long TimeoutErrorCountDelta, long ForcedGrantCountDelta, int? SampleIntervalSeconds) + { + /// True when the row's deltas are the calculator's (0, 0) marker: no delta was knowable, so + /// the two *_delta zeros beside it are not "no timeouts this interval". + public bool IsUnknowable => SampleIntervalSeconds == 0; + } /// One resource pool at the latest snapshot (summed across its semaphores) — Lite's grant lens. public sealed record MemoryGrantRow( DateTime CollectionTime, int PoolId, double AvailableMemoryMb, double GrantedMemoryMb, double UsedMemoryMb, long GranteeCount, long WaiterCount, long TimeoutErrorCountDelta, long ForcedGrantCountDelta); + /// + /// One (resource_semaphore_id, pool_id) — or, for the pool lens, one pool with + /// null — aggregated over EVERY snapshot in the window (#3541 A10). / + /// is the storm detector: the most sessions ever seen waiting for a grant in the + /// window and the snapshot it happened at. The two *InWindow figures are SUMs of the stored per-interval + /// deltas — how many grants timed out / were forced across the whole window, not just the last interval. + /// + public sealed record MemoryGrantWindowRow( + short? ResourceSemaphoreId, int PoolId, long SnapshotsInWindow, DateTime FirstSnapshotAt, DateTime LastSnapshotAt, + long PeakWaiterCount, DateTime PeakWaitersAt, double PeakGrantedMemoryMb, double MinAvailableMemoryMb, + long TimeoutErrorsInWindow, long ForcedGrantsInWindow); + /* ─────────────────────────── resource semaphore (latest snapshot, per semaphore) ─────────────────────────── */ /// /// The latest snapshot's per-(semaphore, pool) rows — the Dashboard's get_resource_semaphore shape - /// over Darling's store (plus max_target_memory_mb from the store; minus the Dashboard's - /// unstored sample_interval_seconds). MB columns are numeric(18,2) → double precision. - /// $1 server_id, $2 window start, $3 window end (naive UTC). + /// over Darling's store (plus max_target_memory_mb from the store; plus, since V128, the + /// Dashboard's sample_interval_seconds, trailing so every existing ordinal is stable). MB columns + /// are numeric(18,2) → double precision. $1 server_id, $2 window start, $3 window end (naive UTC). /// public const string ResourceSemaphoreLatestSql = """ WITH latest AS @@ -79,7 +114,8 @@ FROM v_memory_grant_stats timeout_error_count, forced_grant_count, timeout_error_count_delta, - forced_grant_count_delta + forced_grant_count_delta, + sample_interval_seconds FROM v_memory_grant_stats WHERE server_id = $1 AND collection_time = (SELECT mx FROM latest) @@ -111,7 +147,10 @@ public static async Task> GetResourceSemaphoreLatestA reader.IsDBNull(11) ? 0 : reader.GetInt64(11), reader.IsDBNull(12) ? 0 : reader.GetInt64(12), reader.IsDBNull(13) ? 0 : reader.GetInt64(13), - reader.IsDBNull(14) ? 0 : reader.GetInt64(14))); + reader.IsDBNull(14) ? 0 : reader.GetInt64(14), + /* NULL stays NULL: a pre-V128 row never recorded its interval, and that is a different + statement from the 0 the calculator writes when no delta was knowable. */ + reader.IsDBNull(15) ? null : reader.GetInt32(15))); } return rows; @@ -176,6 +215,178 @@ public static async Task> GetMemoryGrantsLatestAsync( return rows; } + /* ─────────────────────────── the window, per semaphore / per pool (#3541 A10) ─────────────────────────── */ + + /// + /// Every snapshot in the window aggregated per (resource_semaphore_id, pool_id) — the window half of + /// get_resource_semaphore. The peak's instant comes from a DISTINCT ON over the same windowed rows + /// (highest waiter_count first, newest first on a tie, so a storm that plateaued reports its latest + /// snapshot); the two *_in_window figures SUM the stored per-interval deltas. MB aggregates CAST to + /// double precision, count aggregates to bigint, like the latest reads. $1 server_id, $2 window start, + /// $3 window end (naive UTC). + /// + public const string ResourceSemaphoreWindowSql = """ + WITH windowed AS + ( + SELECT + collection_time, + resource_semaphore_id, + pool_id, + waiter_count, + granted_memory_mb, + available_memory_mb, + timeout_error_count_delta, + forced_grant_count_delta + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ), + agg AS + ( + SELECT + resource_semaphore_id, + pool_id, + COUNT(*) AS snapshots_in_window, + MIN(collection_time) AS first_snapshot_at, + MAX(collection_time) AS last_snapshot_at, + CAST(MAX(waiter_count) AS bigint) AS peak_waiter_count, + CAST(MAX(granted_memory_mb) AS double precision) AS peak_granted_memory_mb, + CAST(MIN(available_memory_mb) AS double precision) AS min_available_memory_mb, + CAST(SUM(timeout_error_count_delta) AS bigint) AS timeout_errors_in_window, + CAST(SUM(forced_grant_count_delta) AS bigint) AS forced_grants_in_window + FROM windowed + GROUP BY resource_semaphore_id, pool_id + ), + peak AS + ( + SELECT DISTINCT ON (resource_semaphore_id, pool_id) + resource_semaphore_id, + pool_id, + collection_time AS peak_waiters_at + FROM windowed + ORDER BY resource_semaphore_id, pool_id, waiter_count DESC, collection_time DESC + ) + SELECT + a.resource_semaphore_id, + a.pool_id, + a.snapshots_in_window, + a.first_snapshot_at, + a.last_snapshot_at, + a.peak_waiter_count, + p.peak_waiters_at, + a.peak_granted_memory_mb, + a.min_available_memory_mb, + a.timeout_errors_in_window, + a.forced_grants_in_window + FROM agg AS a + JOIN peak AS p + ON p.resource_semaphore_id = a.resource_semaphore_id + AND p.pool_id = a.pool_id + ORDER BY a.resource_semaphore_id, a.pool_id + """; + + /// + /// Every snapshot in the window aggregated per pool — the window half of get_memory_grants. The pool lens + /// SUMs across a pool's semaphores at each snapshot first (the same per-snapshot SUM + /// serves), THEN takes the window's peak / floor / total over those + /// per-snapshot pool figures, so "peak waiters" is the most sessions waiting on the pool at any one + /// instant, not the largest single semaphore's count. $1 server_id, $2 window start, $3 window end (naive UTC). + /// + public const string MemoryGrantsWindowSql = """ + WITH per_snapshot AS + ( + SELECT + collection_time, + pool_id, + SUM(waiter_count) AS waiter_count, + SUM(granted_memory_mb) AS granted_memory_mb, + SUM(available_memory_mb) AS available_memory_mb, + SUM(timeout_error_count_delta) AS timeout_error_count_delta, + SUM(forced_grant_count_delta) AS forced_grant_count_delta + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + GROUP BY collection_time, pool_id + ), + agg AS + ( + SELECT + pool_id, + COUNT(*) AS snapshots_in_window, + MIN(collection_time) AS first_snapshot_at, + MAX(collection_time) AS last_snapshot_at, + CAST(MAX(waiter_count) AS bigint) AS peak_waiter_count, + CAST(MAX(granted_memory_mb) AS double precision) AS peak_granted_memory_mb, + CAST(MIN(available_memory_mb) AS double precision) AS min_available_memory_mb, + CAST(SUM(timeout_error_count_delta) AS bigint) AS timeout_errors_in_window, + CAST(SUM(forced_grant_count_delta) AS bigint) AS forced_grants_in_window + FROM per_snapshot + GROUP BY pool_id + ), + peak AS + ( + SELECT DISTINCT ON (pool_id) + pool_id, + collection_time AS peak_waiters_at + FROM per_snapshot + ORDER BY pool_id, waiter_count DESC, collection_time DESC + ) + SELECT + CAST(NULL AS smallint) AS resource_semaphore_id, + a.pool_id, + a.snapshots_in_window, + a.first_snapshot_at, + a.last_snapshot_at, + a.peak_waiter_count, + p.peak_waiters_at, + a.peak_granted_memory_mb, + a.min_available_memory_mb, + a.timeout_errors_in_window, + a.forced_grants_in_window + FROM agg AS a + JOIN peak AS p ON p.pool_id = a.pool_id + ORDER BY a.pool_id + """; + + public static Task> GetResourceSemaphoreWindowAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) => + ReadWindowAsync(postgres, ResourceSemaphoreWindowSql, serverId, startUtc, endUtc, cancellationToken); + + public static Task> GetMemoryGrantsWindowAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) => + ReadWindowAsync(postgres, MemoryGrantsWindowSql, serverId, startUtc, endUtc, cancellationToken); + + /// The two window reads project the SAME eleven columns in the same order (the pool lens fills + /// resource_semaphore_id with a typed NULL), so one materialiser serves both. + private static async Task> ReadWindowAsync( + NpgsqlDataSource postgres, string sql, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) + { + var rows = new List(); + await using var command = postgres.CreateCommand(sql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new MemoryGrantWindowRow( + reader.IsDBNull(0) ? null : reader.GetInt16(0), + reader.IsDBNull(1) ? 0 : reader.GetInt32(1), + reader.GetInt64(2), + reader.GetDateTime(3), + reader.GetDateTime(4), + reader.IsDBNull(5) ? 0 : reader.GetInt64(5), + reader.GetDateTime(6), + reader.IsDBNull(7) ? 0 : reader.GetDouble(7), + reader.IsDBNull(8) ? 0 : reader.GetDouble(8), + reader.IsDBNull(9) ? 0 : reader.GetInt64(9), + reader.IsDBNull(10) ? 0 : reader.GetInt64(10))); + } + + return rows; + } + /* ─────────────────────────── memory pressure events (RING_BUFFER_RESOURCE_MONITOR) ─────────────────────────── */ /// One RING_BUFFER_RESOURCE_MONITOR sample — the sample time plus the SQL Server (process) and OS diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs index 4da3a897b..c8c849e5d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingObjectStatsReader.cs @@ -37,10 +37,58 @@ internal static class DarlingObjectStatsReader { /* ─────────────────────────── result rows ─────────────────────────── */ - /// One per-table size + growth row (indexes rolled up per table). + /// + /// One per-table size + growth row (indexes rolled up per table), carrying the raw baselines the growth + /// figures are derived from rather than the derived figures alone (#3541 A12, contract rule 5). + /// The SQL this replaced computed growth_7d / growth_30d / growth_pct_30d through a + /// COALESCE(p30, p7, oldest, current) chain, which is three lies in one expression: with ten days + /// of history "30-day growth" was growth since the SEVEN-day snapshot; with two days it was growth since + /// the oldest snapshot, still labelled 30d; and a table absent from every baseline (created this week) + /// fell through to current - current = 0, "not growing", for the one table that is nothing BUT + /// growth. A nominal window the store cannot reach is not a smaller window — it is no measurement, and + /// the payload has to say so. So the row carries each baseline as the store holds it (null where the + /// snapshot exists but the table was not in it, or where no snapshot old enough exists) plus the + /// store's span, and the derivations live in the properties below where each can refuse. + /// + /// The table's reserved MB at the newest snapshot at or before the 7-day cutoff; null when + /// no such snapshot exists or the table was not in it. + /// Same for the 30-day cutoff. + /// The table's reserved MB at the store's EARLIEST snapshot; null when the table was not + /// in it (created since). + /// The snapshot the 7-day baseline was read from; null when the store holds nothing that old. + /// Same for 30 days. + /// The store's oldest index_object_stats capture for this server. + /// The store's newest — the snapshot every current_* figure is read from. + /// Whole calendar days between the earliest and latest snapshots — how much history the + /// growth figures can honestly span. 0 means one day of snapshots: no growth is knowable. public sealed record ObjectSizeGrowthRow( string DatabaseName, string SchemaName, string TableName, double CurrentReservedMb, double CurrentUsedMb, - long TotalRows, int IndexCount, double Growth7dMb, double Growth30dMb, double DailyGrowthRateMb, double GrowthPct30d); + long TotalRows, int IndexCount, + double? ReservedMb7dAgo, double? ReservedMb30dAgo, double? ReservedMbOldest, + DateTime? Snapshot7dTime, DateTime? Snapshot30dTime, DateTime EarliestSnapshotTime, DateTime LatestSnapshotTime, int DaysOfData) + { + /// Growth since the 7-day baseline; null when there is no such baseline for this table. + public double? Growth7dMb => ReservedMb7dAgo is { } b ? CurrentReservedMb - b : null; + + /// Growth since the 30-day baseline; null when there is no such baseline for this table. + public double? Growth30dMb => ReservedMb30dAgo is { } b ? CurrentReservedMb - b : null; + + /// Percent growth over the 30-day baseline; null without a baseline, and null when the baseline + /// is 0 (no denominator — a table that was empty 30 days ago has no ratio, not an infinite one). + public double? GrowthPct30d => ReservedMb30dAgo is > 0 ? (CurrentReservedMb - ReservedMb30dAgo.Value) * 100.0 / ReservedMb30dAgo.Value : null; + + /// Growth since the store's earliest snapshot — the honest figure when the nominal windows + /// are out of reach. Null when the store holds a single day (no span) or the table was not in the + /// earliest snapshot. + public double? GrowthOverAvailableHistoryMb => DaysOfData >= 1 && ReservedMbOldest is { } o ? CurrentReservedMb - o : null; + + /// Percent form of ; null on a 0 baseline. + public double? GrowthOverAvailableHistoryPct => + DaysOfData >= 1 && ReservedMbOldest is > 0 ? (CurrentReservedMb - ReservedMbOldest.Value) * 100.0 / ReservedMbOldest.Value : null; + + /// MB per day over the available span; null when there is no span to divide by. + public double? DailyGrowthRateMb => GrowthOverAvailableHistoryMb is { } g ? g / DaysOfData : null; + } /// One per-index usage row with its Unused / Write-only / Active classification. public sealed record IndexUsageRow( @@ -63,17 +111,26 @@ public sealed record DatabaseSizeRow( /// /// Per-table size + growth over the daily snapshots — Lite's GetObjectSizeGrowthAsync ported to - /// Postgres: roll indexes up per (database, schema, table) at the latest snapshot, compare against the - /// newest snapshot at/older-than the 7-day ($2) and 30-day ($3) cutoffs (and the earliest snapshot as a - /// fallback), and derive the daily rate from the span of collected data. Ranks by current reserved size - /// descending, cap $4. $1 server_id. + /// Postgres: roll indexes up per (database, schema, table) at the latest snapshot, and read the same + /// table's reserved size at the newest snapshot at/older-than the 7-day ($2) and 30-day ($3) cutoffs and + /// at the store's earliest snapshot. Ranks by current reserved size descending, cap $4. $1 server_id. + /// Baselines are projected RAW, not folded (#3541 A12). The previous shape derived the growth + /// columns in SQL through COALESCE(p30, p7, oldest, current), so a baseline the store did not hold + /// was silently replaced by a nearer one and labelled with the farther window's name — and a table in no + /// baseline at all read as growth 0. Each baseline now comes back as its own nullable column, beside the + /// snapshot time it was read from and the store's span, and derives + /// each growth figure from exactly the baseline it names or declines to. The two cutoff snapshots are + /// resolved once in boundaries with FILTER so the baseline CTEs and the projected snapshot + /// times cannot disagree about which capture was used. /// public const string ObjectSizeGrowthSql = """ WITH boundaries AS ( SELECT MAX(collection_time) AS latest_time, MIN(collection_time) AS earliest_time, - CAST(MAX(collection_time) AS date) - CAST(MIN(collection_time) AS date) AS days_of_data + CAST(MAX(collection_time) AS date) - CAST(MIN(collection_time) AS date) AS days_of_data, + MAX(collection_time) FILTER (WHERE collection_time <= $2) AS snapshot_7d_time, + MAX(collection_time) FILTER (WHERE collection_time <= $3) AS snapshot_30d_time FROM v_index_object_stats WHERE server_id = $1 ), @@ -90,15 +147,13 @@ FROM v_index_object_stats past_7d AS ( SELECT database_name, schema_name, table_name, SUM(reserved_mb) AS reserved_mb FROM v_index_object_stats - WHERE server_id = $1 AND collection_time = ( - SELECT MAX(collection_time) FROM v_index_object_stats WHERE server_id = $1 AND collection_time <= $2) + WHERE server_id = $1 AND collection_time = (SELECT snapshot_7d_time FROM boundaries) GROUP BY database_name, schema_name, table_name ), past_30d AS ( SELECT database_name, schema_name, table_name, SUM(reserved_mb) AS reserved_mb FROM v_index_object_stats - WHERE server_id = $1 AND collection_time = ( - SELECT MAX(collection_time) FROM v_index_object_stats WHERE server_id = $1 AND collection_time <= $3) + WHERE server_id = $1 AND collection_time = (SELECT snapshot_30d_time FROM boundaries) GROUP BY database_name, schema_name, table_name ), oldest AS ( @@ -115,15 +170,14 @@ FROM v_index_object_stats CAST(l.current_used_mb AS double precision) AS current_used_mb, l.total_rows, l.index_count, - CAST(l.current_reserved_mb - COALESCE(p7.reserved_mb, o.reserved_mb, l.current_reserved_mb) AS double precision) AS growth_7d_mb, - CAST(l.current_reserved_mb - COALESCE(p30.reserved_mb, p7.reserved_mb, o.reserved_mb, l.current_reserved_mb) AS double precision) AS growth_30d_mb, - CASE WHEN b.days_of_data >= 1 - THEN CAST(l.current_reserved_mb - COALESCE(o.reserved_mb, l.current_reserved_mb) AS double precision) / CAST(b.days_of_data AS double precision) - ELSE 0 END AS daily_growth_rate_mb, - CASE WHEN COALESCE(p30.reserved_mb, p7.reserved_mb, o.reserved_mb) > 0 - THEN CAST(l.current_reserved_mb - COALESCE(p30.reserved_mb, p7.reserved_mb, o.reserved_mb) AS double precision) * 100.0 - / CAST(COALESCE(p30.reserved_mb, p7.reserved_mb, o.reserved_mb) AS double precision) - ELSE 0 END AS growth_pct_30d + CAST(p7.reserved_mb AS double precision) AS reserved_mb_7d_ago, + CAST(p30.reserved_mb AS double precision) AS reserved_mb_30d_ago, + CAST(o.reserved_mb AS double precision) AS reserved_mb_oldest, + b.snapshot_7d_time, + b.snapshot_30d_time, + b.earliest_time, + b.latest_time, + b.days_of_data FROM latest l CROSS JOIN boundaries b LEFT JOIN past_7d p7 ON p7.database_name = l.database_name AND p7.schema_name = l.schema_name AND p7.table_name = l.table_name @@ -154,10 +208,15 @@ public static async Task> GetObjectSizeGrowthAsync( reader.IsDBNull(4) ? 0 : reader.GetDouble(4), reader.IsDBNull(5) ? 0 : reader.GetInt64(5), reader.IsDBNull(6) ? 0 : Convert.ToInt32(reader.GetValue(6)), - reader.IsDBNull(7) ? 0 : reader.GetDouble(7), - reader.IsDBNull(8) ? 0 : reader.GetDouble(8), - reader.IsDBNull(9) ? 0 : reader.GetDouble(9), - reader.IsDBNull(10) ? 0 : reader.GetDouble(10))); + /* The baselines stay NULL when the store has none — a missing baseline is not a 0 baseline. */ + reader.IsDBNull(7) ? null : reader.GetDouble(7), + reader.IsDBNull(8) ? null : reader.GetDouble(8), + reader.IsDBNull(9) ? null : reader.GetDouble(9), + reader.IsDBNull(10) ? null : reader.GetDateTime(10), + reader.IsDBNull(11) ? null : reader.GetDateTime(11), + reader.GetDateTime(12), + reader.GetDateTime(13), + Convert.ToInt32(reader.GetValue(14)))); } return rows; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgLoggingAudit.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgLoggingAudit.cs new file mode 100644 index 000000000..c0fd2d0a9 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPgLoggingAudit.cs @@ -0,0 +1,732 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using PerformanceMonitor.Darling.Storage; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The logging-settings audit for a PostgreSQL target (#3607): which of the server's logging GUCs are +/// producing the lines they can, judged facet by facet from the stored pg_settings snapshot, with what +/// each unlocks, the recommended value AND its cost, and the remedy in the syntax the hosting flavour needs. +/// +/// The failure this fixes is the one plan-capture readiness fixed for auto_explain, over the rest +/// of the log. A target with log_lock_waits, log_temp_files, +/// log_autovacuum_min_duration, log_checkpoints, log_connections, +/// log_disconnections and log_min_duration_statement all off looks IDENTICAL to a fully +/// instrumented one from every read this product has — the same counters, the same sampled views — and +/// the operator learns the difference at incident time, when the log they reach for holds nothing. This +/// answers "is this target telling us everything it could" before that moment, and answers it per setting, +/// because each one has a different cost and a different consumer and a single "logging: partial" would tell +/// nobody what to do. +/// +/// It is a READ, not a collector, and the shape follows from what it judges. +/// PgPlanCaptureReadinessCollector persists its facets because judging them means PROBING the target +/// — whether auto_explain is in shared_preload_libraries, whether an auto_explain.* GUC +/// even exists — and that probe is worth a history. Every setting here is a plain core GUC that +/// PgServerConfigCollector already stores hourly with its value, source, unit and context, so the +/// judgment is a pure function over rows the store holds, computed when asked. No new table, no schema rung, +/// nothing for a second collector to disagree with the first about. The snapshot's own +/// collection_time is the audit's captured_at. +/// +/// Four verdicts, and partial is not a lesser instrumented. instrumented +/// means the setting is producing every line it can; off means none; unknown means the setting +/// is not in the stored snapshot and NOTHING is inferred about it; partial means a THRESHOLD is +/// filtering — statements faster than N ms, temp files under N kB, autovacuum runs shorter than N ms — and +/// what falls below it is stated on the row. For two of the settings the threshold is the RECOMMENDED +/// posture: a log_min_duration_statement of 0 logs every statement the server runs, and #2565 +/// measured the capture-everything shape of that mechanism at 31 percent of throughput. So the summary +/// names the off and unknown settings as the actionable ones, and a partial row's +/// cost_note says whether its threshold is the recommendation or a compromise. +/// +/// Consumers are named honestly, "planned" included. The issue's own sequencing note says this +/// audit earns its keep once the log pipeline (#3601) and its first parser families (#3602 temp files, +/// #3603 autovacuum) consume the lines, and none of those ships yet. Each facet therefore names the Darling +/// family that would read its lines and says planned where that is the truth, beside the read that +/// exists today and what it cannot see. The setting is still worth turning on before the consumer lands: +/// the log it fills is the one somebody opens at incident time, whichever tool reads it. +/// +/// Plan capture's own settings are shown, not re-judged. shared_preload_libraries, +/// auto_explain.log_min_duration, log_line_prefix and lc_messages appear in the same +/// snapshot and are listed for completeness with the readiness facet that owns each, because +/// get_pg_plan_capture_readiness already judges them with the trap each one carries (a loaded library +/// capturing nothing at -1, a placeholder GUC on a server that never loaded the module, a translated +/// message catalogue) and a second judgment here would be a second place for those to drift. +/// lc_messages matters to every row above it — the lines these settings produce are English text the +/// parsers match — which is why it is in the list at all. +/// +/// Hosting flavour comes from the snapshot, not from the registry. The store's engine token +/// (MonitoredEngineKind) separates Aurora from everything else, and "everything else" is both +/// self-hosted PostgreSQL and RDS for PostgreSQL — which need OPPOSITE remedies: ALTER SYSTEM plus a +/// reload on one, a parameter group on the other, where ALTER SYSTEM is refused. Any rds.* GUC +/// in the snapshot (rds.extensions, rds.superuser_reserved_connections, … — RDS and Aurora +/// both carry them) is evidence in the same rows this already reads, and the response says which way it +/// decided and on what. +/// +public static class DarlingPgLoggingAudit +{ + /// The verdict vocabulary on the wire — the header says what each means. + public const string Instrumented = "instrumented"; + public const string Partial = "partial"; + public const string Off = "off"; + public const string Unknown = "unknown"; + + /// The tool that judges the plan-capture settings this audit only lists. + public const string ReadinessTool = "get_pg_plan_capture_readiness"; + + /// The GUC. + /// What the snapshot holds, verbatim, or null when the setting is not in it. + /// The unit pg_settings reports for a numeric setting. + /// The compiled-in default, from the same snapshot. + /// Where the value came from. + /// Reload or restart, from the setting's context. + /// One of the four constants above. + /// What telemetry the setting produces, and what this product has INSTEAD today. + /// The Darling family that reads the lines, marked planned where it does not ship. + /// The value to set. + /// What the recommended value costs, and when to deviate from it. + /// The change, in the syntax this server's hosting flavour needs — or why none is needed. + /// Set when the value came from a per-role or per-database override the monitoring + /// connection resolved, so the server-wide value may differ. + /// The file and the running server disagree about this setting, so the value + /// judged here is the RUNNING one and changes at the next restart. + /// The consequence of , spelled out on the row; + /// null when the two agree. + public sealed record Facet( + string Setting, + string? Value, + string? Unit, + string? DefaultValue, + string? Source, + string? ChangeNeeds, + string Verdict, + string Unlocks, + string Consumer, + string Recommended, + string CostNote, + string Remedy, + string? ScopeNote, + bool PendingRestart, + string? RestartNote); + + /// A plan-capture setting shown as observed, with the readiness facet that judges it. + public sealed record ReadinessSetting(string Setting, string? Value, string? Source, string ReadinessFacet); + + /// The snapshot's collection time — the one stamp every row shares. + /// True when the snapshot carries rds.* parameters. + /// What the flavour decision rested on. + /// One per judged setting, in the order an operator reaches for them. + /// The plan-capture settings, listed not judged. + public sealed record Result( + DateTime CapturedAt, + bool Managed, + string HostingEvidence, + IReadOnlyList Facets, + IReadOnlyList JudgedByReadiness); + + /// + /// The settings this audit judges, in the order they are reported: what somebody reaches for FIRST when + /// a server is slow — the statements — then the locks, the spills, the maintenance, the checkpoints, and + /// the connection churn. There is no causal chain between them (unlike readiness, where the library gates + /// the threshold), so the order is the reader's, and the note on the response says so. + /// + public static readonly IReadOnlyList JudgedSettings = new[] + { + "log_min_duration_statement", + "log_lock_waits", + "log_temp_files", + "log_autovacuum_min_duration", + "log_checkpoints", + "log_connections", + "log_disconnections", + }; + + /// The plan-capture settings listed for completeness, with the readiness facet owning each. + public static readonly IReadOnlyList<(string Setting, string ReadinessFacet)> ReadinessSettings = new[] + { + ("shared_preload_libraries", "library_loaded"), + ("auto_explain.log_min_duration", "capture_threshold"), + ("log_line_prefix", "plan_attribution"), + ("lc_messages", "message_locale"), + }; + + /// + /// Judges one server's newest snapshot. The rows are what + /// returned — every non-session setting + /// at the newest collection_time — and MUST be non-empty; an empty snapshot is the tool's + /// empty/not_collected path, not an audit of nothing. + /// + public static Result Audit(IReadOnlyList snapshot) + { + ArgumentNullException.ThrowIfNull(snapshot); + if (snapshot.Count == 0) + { + throw new ArgumentException("An empty snapshot cannot be audited; report it as not collected.", nameof(snapshot)); + } + + var byName = new Dictionary(StringComparer.Ordinal); + foreach (var row in snapshot) + { + /* First wins. The reader excludes session-scoped sources, so two rows for one name at one + collection_time should not happen; if a store ever holds them, the audit must not throw over + a duplicate the collector wrote. */ + byName.TryAdd(row.Name, row); + } + + var capturedAt = snapshot.Max(r => r.CollectionTime); + + /* The hosting flavour, from evidence in the rows rather than from a registry token that cannot + separate RDS from self-hosted. Counted so the response can say how much evidence there was. */ + var rdsParameters = byName.Keys.Count(n => n.StartsWith("rds.", StringComparison.Ordinal)); + var managed = rdsParameters > 0; + var hostingEvidence = managed + ? $"{rdsParameters} rds.* parameter(s) in the snapshot, which only RDS and Aurora carry - remedies are " + + "worded for a parameter group, where ALTER SYSTEM is refused." + : "no rds.* parameter in the snapshot, so this is not RDS or Aurora - remedies are worded as ALTER " + + "SYSTEM plus a reload. If this server IS managed by a provider that hides its own parameters, " + + "translate each remedy to that provider's parameter surface."; + + var deadlockTimeout = Describe(byName, "deadlock_timeout"); + + var facets = new List(JudgedSettings.Count) + { + MinDurationStatement(byName, managed), + LockWaits(byName, managed, deadlockTimeout), + TempFiles(byName, managed), + AutovacuumMinDuration(byName, managed), + Checkpoints(byName, managed), + Connections(byName, managed), + Disconnections(byName, managed), + }; + + var readiness = ReadinessSettings + .Select(s => byName.TryGetValue(s.Setting, out var row) + ? new ReadinessSetting(s.Setting, row.Setting, row.Source, s.ReadinessFacet) + : new ReadinessSetting(s.Setting, null, null, s.ReadinessFacet)) + .ToList(); + + return new Result(capturedAt, managed, hostingEvidence, facets, readiness); + } + + /* ───────────────────────── the facets ───────────────────────── */ + + private static Facet MinDurationStatement( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_min_duration_statement"; + var row = Find(byName, Setting); + var threshold = Threshold(row); + + /* -1 off, 0 everything, N a threshold. The RECOMMENDED state is the threshold, and the verdict is + still partial - the header says why: partial describes the lines, and this row's cost_note says + the filter is the point. */ + var verdict = row is null ? Unknown + : threshold is null ? Unknown + : threshold < 0 ? Off + : threshold == 0 ? Instrumented + : Partial; + + var cost = verdict switch + { + Instrumented => + "0 logs EVERY statement the server runs, with its text. That is the capture-everything shape " + + "#2565 measured for auto_explain at 31 percent of throughput and 772 MB of log in 20 seconds " + + "(pgbench, 8 clients). The statement log at 0 emits one entry per statement exactly as " + + "auto_explain at 0 does - smaller entries with no plan body, but no fewer of them. Move to a " + + "millisecond threshold; the fast statements are the volume and nobody reads them.", + Partial => + $"Statements faster than {threshold} ms write nothing, which is the intended trade: the slow " + + "ones are the question and the fast ones are the volume. This is the recommended posture, " + + "not a gap. To see a SAMPLE of the faster ones without the volume, log_min_duration_sample " + + "with log_statement_sample_rate is the sampling form; it is a separate setting and not judged " + + "here.", + Off => + "Off costs nothing and records nothing: a 40-second statement cancelled by its client leaves " + + "no trace anywhere but this log line. A threshold sized to the workload - well above the " + + "normal statement time, so that only the outliers write - costs one line per outlier.", + _ => UnknownNote(row), + }; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "One LOG line per EXECUTION that ran longer than the threshold, carrying the duration and " + + "the statement text. What this product has instead is pg_stat_statements through " + + "get_pg_top_queries: per-SHAPE aggregates that can say a shape averages 200 ms and " + + "never that one execution took 40 seconds at 03:07 - the log line is the only record " + + "of the individual slow execution.", + Consumer: "PLANNED - no Darling family reads statement-duration lines yet; #3601's log pipeline is " + + "where they would land, and the statement text carries literals, so the same redaction " + + "pass plan capture applies before storage is a precondition of storing them at all. " + + "get_pg_top_queries is the aggregate the lines would sharpen. Until then the line is " + + "what an operator finds in the server log at incident time.", + Recommended: "A millisecond threshold sized to the workload, never 0 - 1000 is a common starting " + + "point on an OLTP workload; lower it as the volume proves tolerable. " + + "auto_explain.log_min_duration is the separate threshold for PLANS and is judged by " + + ReadinessTool + ".", + CostNote: cost, + Remedy: Remedy(Setting, "1000", row, managed, verdict, alreadyRight: verdict == Partial), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet LockWaits( + IReadOnlyDictionary byName, bool managed, + string deadlockTimeout) + { + const string Setting = "log_lock_waits"; + var row = Find(byName, Setting); + var on = Bool(row); + + var verdict = row is null || on is null ? Unknown : on.Value ? Instrumented : Off; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: $"A LOG line whenever a session waits longer than deadlock_timeout ({deadlockTimeout}) for a " + + "lock, naming the waiting process, the lock it wanted and the statement that wanted it - " + + "the ENGINE-recorded record of lock waits. What this product has instead is get_pg_blocking " + + "from pg_blocking, which SAMPLES pg_locks and pg_stat_activity on a cadence: a wait that " + + "starts and ends between two samples is invisible to it and would be in this line.", + Consumer: "PLANNED - #3601 names lock-wait reports among the families the log pipeline would " + + "classify. get_pg_blocking is the sampled read that exists today, and its own response " + + "says how many samples its 'no blocking' rests on.", + Recommended: "on.", + CostNote: verdict == Unknown ? UnknownNote(row) + : "One line per wait longer than deadlock_timeout - negligible on a workload that is not " + + "already lock-bound, and on one that is, the volume is itself the finding. Do NOT lower " + + "deadlock_timeout to make this fire sooner: that is also how often the deadlock " + + "detector runs, and it is a lock-heavy operation in its own right.", + Remedy: Remedy(Setting, "on", row, managed, verdict, alreadyRight: verdict == Instrumented), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet TempFiles( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_temp_files"; + var row = Find(byName, Setting); + var threshold = Threshold(row); + + var verdict = row is null ? Unknown + : threshold is null ? Unknown + : threshold < 0 ? Off + : threshold == 0 ? Instrumented + : Partial; + + var cost = verdict switch + { + Instrumented => + "0 logs EVERY temporary file at the moment it is deleted, including the small ones. On a " + + "workload whose sorts sit just over work_mem that is a line per spill, constantly; if that is " + + "this server, set a kilobyte threshold (10240 is 10 MB) and accept that spills under it are " + + "unseen. On most workloads 0 is cheap and is the recommendation.", + Partial => + $"Temporary files under {threshold} kB write nothing. That is a deliberate trade for a workload " + + "that spills small files constantly; if this server does not, 0 sees everything at little " + + "cost. Per-event attribution works from whatever crosses the line either way.", + Off => + "Off records nothing: the counters say a database spilled 4 GB in an hour and the log says " + + "nothing about which statement did it or when. 0 costs one line per temp file; on a workload " + + "that spills constantly a kilobyte threshold bounds it.", + _ => UnknownNote(row), + }; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "One LOG line per temporary file, with its size and the statement that wrote it - " + + "per-EVENT spill attribution. What this product has instead is the counter shadow of " + + "that: per-database temp_files / temp_bytes deltas (get_pg_database_stats, " + + "get_pg_database_trend) and per-shape temp_blks_* from pg_stat_statements " + + "(get_pg_top_queries). Between them you can know a database spilled and that a shape " + + "spills - never that THIS execution spilled 4 GB at 03:07, which is the question when a " + + "disk fills.", + Consumer: "PLANNED - #3602, per-event temp spill attribution, is the parser family that would read " + + "these lines; its statement text needs plan capture's redaction pass first. " + + "get_pg_database_stats carries the spill FINDING from the counters today.", + Recommended: "0 (every spill), or a kilobyte threshold on a workload that spills small files constantly.", + CostNote: cost, + Remedy: Remedy(Setting, "0", row, managed, verdict, alreadyRight: verdict is Instrumented or Partial), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet AutovacuumMinDuration( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_autovacuum_min_duration"; + var row = Find(byName, Setting); + var threshold = Threshold(row); + + var verdict = row is null ? Unknown + : threshold is null ? Unknown + : threshold < 0 ? Off + : threshold == 0 ? Instrumented + : Partial; + + /* PostgreSQL 15 moved the default from -1 to 600000 (ten minutes). A server sitting on that default + at a positive threshold is the shape worth naming: it sees only the outlier runs. The judgment is + PostgreSQL's OWN - source = 'default' is the server saying nobody set it - and not a text + comparison against boot_val, for the reason DarlingPgServerConfigReader.CurrentConfigSql gives: + an administrator who writes 600000 into postgresql.conf has made a choice that happens to equal + the boot value, and calling that choice a default would attribute it to inaction. */ + var atDefault = row is not null && threshold > 0 + && string.Equals(row.Source, "default", StringComparison.Ordinal); + + var cost = verdict switch + { + Instrumented => + "0 logs every autovacuum and autoanalyze run. The line rate is the rate at which tables get " + + "vacuumed, bounded by autovacuum_max_workers (three by default) each finishing one table before " + + "starting the next - small on most servers, and the cheapest setting on this list for what it " + + "returns. A database with thousands of tiny tables is the exception where a threshold earns " + + "its place.", + Partial => + $"Runs shorter than {threshold} ms write nothing." + + (atDefault + ? " This is PostgreSQL's own default since 15 (ten minutes), and on most tables that is " + + "every run: a cost history that sees only the outliers cannot say what a NORMAL run " + + "costs, which is the baseline the outliers are judged against. 0 is cheap here." + : " Whether that is the right cut depends on what you want the history for: outliers " + + "only, or a baseline of what a normal run costs. 0 is cheap here."), + Off => + "Off records nothing about what any run cost. 0 costs one line per run, and the run rate is " + + "bounded by the worker count, so the volume is small on all but a server with thousands of " + + "tiny tables.", + _ => UnknownNote(row), + }; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "One LOG line per autovacuum or autoanalyze run that took longer than the threshold: " + + "pages and tuples removed, buffer hits and misses, read and write rates, WAL usage and " + + "elapsed time - what the run COST. What this product has instead is pg_stat_user_tables " + + "through get_pg_autovacuum_health: whether autovacuum ran and when, never what it cost, " + + "so a run that takes 40 minutes of I/O in the business peak reads as healthy there.", + Consumer: "PLANNED - #3603, autovacuum per-run cost, is the parser family that would read these " + + "lines and put run history beside get_pg_autovacuum_health. That read is the " + + "whether-not-what read today.", + Recommended: "0 (every run).", + CostNote: cost, + Remedy: Remedy(Setting, "0", row, managed, verdict, alreadyRight: verdict == Instrumented), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet Checkpoints( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_checkpoints"; + var row = Find(byName, Setting); + var on = Bool(row); + + var verdict = row is null || on is null ? Unknown : on.Value ? Instrumented : Off; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "A LOG line per checkpoint with what it did: buffers written, files synced, write, sync " + + "and total time, WAL distance - the CAUSE side of a checkpoint I/O storm, per " + + "checkpoint. What this product has instead is get_pg_write_stats over the checkpointer " + + "counters: timed versus requested and buffers written across a window, never the " + + "duration of one checkpoint.", + Consumer: "PLANNED - #3601's log pipeline. get_pg_write_stats reads the checkpoint counters today.", + Recommended: "on - PostgreSQL 15 made it the default.", + CostNote: verdict == Unknown ? UnknownNote(row) + : "One or two lines per checkpoint, and a checkpoint happens at most every " + + "checkpoint_timeout (five minutes by default) unless something requests one - " + + "negligible. A server reporting off has it set that way in a file or parameter group, " + + "or is on a major before 15 where off was the default.", + Remedy: Remedy(Setting, "on", row, managed, verdict, alreadyRight: verdict == Instrumented), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet Connections( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_connections"; + var row = Find(byName, Setting); + + /* A boolean through PostgreSQL 17 and a STRING LIST in 18 (receipt, authentication, authorization, + setup_durations, all), where on/true/yes/1 still mean all and the empty string means off. Measured + on 18.4: the value is stored verbatim as written - 'on', 'true', '1', 'all', 'receipt,authentication' + - so this reads any non-empty, non-false value as producing lines and shows the value itself. */ + var verdict = row is null ? Unknown + : ConnectionLogging(row.Setting) is bool on ? (on ? Instrumented : Off) + : Unknown; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "A LOG line per connection as it is received, authenticated and authorized - the only " + + "record of connection CHURN and of who a FATAL authentication failure was. What this " + + "product has instead is pg_stat_database's numbackends, a gauge, and its sessions " + + "counter, a cumulative total: neither says who connected when, or that 400 connections " + + "arrived in the minute before the incident.", + Consumer: "PLANNED - #3601 names connection churn among the log pipeline's families.", + Recommended: "on (PostgreSQL 18: 'all', or a list such as receipt,authentication).", + CostNote: verdict == Unknown ? UnknownNote(row) + : "On a POOLED workload connections are rare and this costs nothing. On an unpooled one - a " + + "connection per request - it is a line per request, and that volume is itself the " + + "finding: the pool that is missing. On 18 the list form narrows the lines to the " + + "stages you want.", + Remedy: Remedy(Setting, "on", row, managed, verdict, alreadyRight: verdict == Instrumented), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + private static Facet Disconnections( + IReadOnlyDictionary byName, bool managed) + { + const string Setting = "log_disconnections"; + var row = Find(byName, Setting); + var on = Bool(row); + + var verdict = row is null || on is null ? Unknown : on.Value ? Instrumented : Off; + + return new Facet( + Setting, + row?.Setting, row?.Unit, row?.BootValue, row?.Source, ChangeNeeds(row), + verdict, + Unlocks: "A LOG line per session end WITH the session's duration - the half that turns connection " + + "lines into session lifetimes, so a churning application shows as thousands of " + + "two-second sessions rather than as a connection count that looks stable.", + Consumer: "PLANNED - #3601, beside log_connections.", + Recommended: "on, together with log_connections.", + CostNote: verdict == Unknown ? UnknownNote(row) + : "The same profile as log_connections: one line per session end, nothing on a pooled " + + "workload, a line per request on an unpooled one.", + Remedy: Remedy(Setting, "on", row, managed, verdict, alreadyRight: verdict == Instrumented), + ScopeNote: ScopeNote(row), + PendingRestart: row?.PendingRestart ?? false, + RestartNote: RestartNote(row)); + } + + /* ───────────────────────── shared pieces ───────────────────────── */ + + private const string NotInSnapshot = + "This setting is not in the stored snapshot, so nothing is claimed about it - not inferred from the " + + "default, not inferred from the major version. get_pg_server_config shows what the snapshot holds."; + + /// + /// The two ways a verdict is unknown, told apart on the row: the setting is not in the snapshot at + /// all, or it is there with a value this audit cannot read as the boolean or integer PostgreSQL renders + /// for it. The second is close to unreachable — pg_settings renders well-formed values — but a + /// message that said "not in the snapshot" about a row that is plainly in it would be false, and the + /// contract this type's header states ("unknown means the setting is not in the stored snapshot") is + /// what the first sentence below keeps true by naming the exception. + /// + private static string UnknownNote(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) => row is null + ? NotInSnapshot + : $"This setting IS in the stored snapshot but its value '{row.Setting}' is not one this audit can read " + + "as the boolean or integer PostgreSQL renders for it, so nothing is claimed about it. " + + "get_pg_server_config shows the raw row; if this recurs, the collector's rendering has changed " + + "and the audit's parse needs to learn it."; + + private static DarlingPgLoggingAuditReader.PgLoggingSettingRow? Find( + IReadOnlyDictionary byName, string name) => + byName.TryGetValue(name, out var row) ? row : null; + + /// A setting's value with its unit, for prose — 1000 ms — or a plain statement that + /// the snapshot does not have it. + private static string Describe( + IReadOnlyDictionary byName, string name) + { + var row = Find(byName, name); + if (row?.Setting is null) + { + return "not in the snapshot"; + } + + return string.IsNullOrEmpty(row.Unit) ? row.Setting : $"{row.Setting} {row.Unit}"; + } + + /// + /// The integer a threshold GUC holds. pg_settings.setting renders integers in the setting's base + /// unit with no suffix (600000 with unit = ms), so this is a plain parse; anything else is + /// null and the caller says unknown rather than guessing. + /// + private static long? Threshold(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) + { + /* A block body, not an expression with a property pattern: TsqlConventionGuardTests' member scan + reads a `{ }` pattern as the member's body and stops short, which strands everything after it. */ + var text = row?.Setting; + if (text is null) + { + return null; + } + + return long.TryParse(text.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) + ? value + : null; + } + + /// The spellings PostgreSQL accepts for a boolean GUC, as pg_settings renders them. + private static bool? Bool(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) => BoolText(row?.Setting); + + private static bool? BoolText(string? text) + { + if (text is null) + { + return null; + } + + switch (text.Trim().ToLowerInvariant()) + { + case "on": + case "true": + case "yes": + case "1": + return true; + case "off": + case "false": + case "no": + case "0": + return false; + default: + return null; + } + } + + /// + /// log_connections across the 17/18 boundary: the boolean spellings first, then the 18 list — + /// empty is off, anything else non-empty is a stage list and produces lines. + /// + internal static bool? ConnectionLogging(string? text) + { + if (text is null) + { + return null; + } + + if (BoolText(text) is bool asBool) + { + return asBool; + } + + return text.Trim().Length > 0; + } + + /// Reload or restart, from the GUC's context — the fact get_pg_server_config already + /// exposes as requires_restart_to_change, worded for a remedy. + private static string? ChangeNeeds(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) => row?.Context switch + { + null => null, + "postmaster" => "restart", + "superuser-backend" or "backend" => "reload; applies to connections opened after it", + _ => "reload", + }; + + /// + /// A per-role or per-database override resolved on the MONITORING connection is not the server's value — + /// the limit the readiness collector states for lc_messages, and it holds for every GUC here. A + /// log line is written by the backend that raised it, under THAT backend's resolved value. + /// + private static string? ScopeNote(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) => row?.Source switch + { + "user" or "database" or "database user" => + $"source is '{row.Source}': this value came from a per-role or per-database override that the " + + "monitoring connection resolved, so the server-wide value may differ, and every other backend " + + "logs under its OWN resolved value. get_pg_server_config shows the setting's source; ALTER ROLE " + + "/ ALTER DATABASE ... RESET removes the override.", + _ => null, + }; + + /// + /// pending_restart is the one row where the value judged is provably NOT the value the server will + /// have: postgresql.conf (or ALTER SYSTEM's file) already holds something else and the running server + /// has not restarted. get_pg_server_config reports it loudly for the same reason; here it matters twice + /// over, because the remedy is written against the running value and a restart may deliver the change, + /// or a different one, with no deployment to explain it. Which value the file holds is not in the + /// snapshot - pg_settings does not carry it - so the note says the disagreement exists and where to look, + /// and does not guess the direction. + /// + private static string? RestartNote(DarlingPgLoggingAuditReader.PgLoggingSettingRow? row) + { + /* A block body for the reason Threshold has one: a `{ }` property pattern reads as the member's + body to TsqlConventionGuardTests' scan. */ + if (row is null || !row.PendingRestart) + { + return null; + } + + return "pending_restart is TRUE: the configuration file already holds a different value for this setting " + + "and the running server has not restarted, so the value judged here is the RUNNING one and it " + + "changes at the next restart with no deployment to explain it. pg_settings does not carry the " + + "file's value, so which way it changes is not knowable from here - read the file, or " + + "get_pg_server_config's pending_restart_settings, before acting on this row's remedy."; + } + + /// + /// The change in the hosting flavour's own syntax, or the reason none is needed. alreadyRight is + /// the caller's judgment that the current value is the recommended posture — which for a threshold + /// setting can be a partial verdict — so the remedy does not tell somebody to change a value that + /// is already right. + /// + private static string Remedy( + string setting, string recommendedLiteral, + DarlingPgLoggingAuditReader.PgLoggingSettingRow? row, bool managed, string verdict, bool alreadyRight) + { + if (verdict == Unknown) + { + return row is null + ? "No remedy is offered for a setting the snapshot does not hold: check get_pg_server_config, " + + "and if the collector is running, the next hourly snapshot will carry it." + : "No remedy is offered for a value this audit could not read: get_pg_server_config shows the " + + "raw row, and the remedy depends on what it actually says."; + } + + if (alreadyRight) + { + return "No change needed - the current value is the recommended posture. cost_note says what " + + "it does and does not write."; + } + + var restart = string.Equals(row?.Context, "postmaster", StringComparison.Ordinal); + var perBackend = row?.Context is "superuser-backend" or "backend"; + + if (managed) + { + return $"Set {setting} = {recommendedLiteral} in the DB parameter group and apply it - on Aurora the " + + "CLUSTER parameter group covers every instance and an instance-level group overrides it " + + "per instance. ALTER SYSTEM is refused on RDS and Aurora. " + + (restart + ? "This is a STATIC parameter and needs a reboot." + : "This is a dynamic parameter and applies WITHOUT a reboot" + + (perBackend ? ", to connections opened after it." : ".")); + } + + return $"ALTER SYSTEM SET {setting} = {recommendedLiteral}; SELECT pg_reload_conf(); - " + + (restart + ? "then RESTART: this parameter's context is postmaster and a reload does not apply it." + : "a reload, not a restart" + + (perBackend + ? "; sessions already open keep their value and new connections take the new one." + : ".")); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCacheSchedulerReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCacheSchedulerReader.cs index 6a988e83c..111f30815 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCacheSchedulerReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCacheSchedulerReader.cs @@ -20,7 +20,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// / get_cpu_scheduler_pressure and the viewer's ViewerDataService.PlanCache / /// ViewerDataService.CpuScheduler read. Both are point-in-time snapshot collectors (one/many rows per /// collection, no deltas), so each read is the LATEST snapshot: plan-cache = every (cacheobjtype, objtype) -/// group at the newest collection in the window; cpu-scheduler = the single newest row for the server. STORED +/// group at the newest collection in the window; cpu-scheduler = the single newest row in the window. STORED /// reads, no live monitored-server hit, on the v_plan_cache_stats / v_cpu_scheduler_stats views. /// /// @@ -135,9 +135,14 @@ public static (string Level, string Recommendation) ClassifyPlanCacheBloat(long /* ─────────────────────────── cpu scheduler (latest snapshot) ─────────────────────────── */ /// - /// The single most recent CPU-scheduler snapshot for the server — the Dashboard's - /// get_cpu_scheduler_pressure point-in-time read (the Dashboard tool takes no window, reading - /// report.cpu_scheduler_pressure's TOP 1). $1 server_id. + /// The single most recent CPU-scheduler snapshot for the server IN THE WINDOW — the Dashboard's + /// get_cpu_scheduler_pressure point-in-time read (report.cpu_scheduler_pressure's TOP 1), + /// bounded the way Lite's GetCpuSchedulerSnapshotAsync bounds it (#3541 A10): the Dashboard tool took + /// no window and served the newest row a store ever held, so a server whose scheduler collector died a week + /// ago answered "NORMAL" with a week-old row and nothing in the payload to say so. The window is a SEARCH + /// bound for the newest snapshot, not an aggregate — the tool's hours_back description says exactly + /// that — and the anchor makes "what did the scheduler look like at 03:00 Tuesday" answerable. + /// $1 server_id, $2 window start, $3 window end (naive UTC). /// public const string CpuSchedulerPressureSql = """ SELECT @@ -157,16 +162,18 @@ public static (string Level, string Recommendation) ClassifyPlanCacheBloat(long physical_memory_pressure_warning FROM v_cpu_scheduler_stats WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 ORDER BY collection_time DESC LIMIT 1 """; public static async Task GetCpuSchedulerPressureAsync( - NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) { await using var command = postgres.CreateCommand(CpuSchedulerPressureSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; - DarlingMcpReadParameters.AddInt(command, serverId); + DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (!await reader.ReadAsync(cancellationToken)) { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs index 4c794bb60..373002ece 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingPlanCorrectionReader.cs @@ -42,10 +42,16 @@ internal static class DarlingPlanCorrectionReader { /// /// The engine's automatic plan correction recommendations for one server over the window, newest first. - /// recommendation_name IS NOT NULL drops the enablement-only rows. LIMIT 200 mirrors the Viewer's - /// grid read; the tool applies its own smaller take on top. The four lifecycle times are projected as - /// stored, because the DMV reports them in UTC (see the class remarks). $1 server_id, $2 window start, - /// $3 window end (naive UTC). + /// recommendation_name IS NOT NULL drops the enablement-only rows. The four lifecycle times are + /// projected as stored, because the DMV reports them in UTC (see the class remarks). $1 server_id, + /// $2 window start, $3 window end (naive UTC), $4 row cap. + /// + /// The cap is a PARAMETER, not a literal (#3541 A3). It was LIMIT 200 mirroring the Viewer's + /// grid, with the tool taking limit of those on top — and this table is the one where a fixed row + /// cap most misrepresents a window, because the collector RE-CAPTURES every open recommendation on every + /// cycle. A handful of recommendations on a five-minute cadence fills 200 rows in roughly sixteen hours, + /// so a 168-hour request was answered from its newest sixteen and nothing said so. The tool now passes + /// limit + 1 and publishes the page's actual time reach beside the window it was asked for. /// public const string PlanCorrectionsSql = @" SELECT @@ -79,7 +85,7 @@ FROM plan_correction AND collection_time <= $3 AND recommendation_name IS NOT NULL ORDER BY collection_time DESC, score DESC -LIMIT 200"; +LIMIT $4"; /// /// The latest FORCE_LAST_GOOD_PLAN enablement snapshot per database. The enablement columns repeat on @@ -133,8 +139,10 @@ public sealed record AutomaticTuningRow( string? Reason, DateTime CollectionTime); + /// The newest recommendation rows over the window. Callers detecting + /// truncation pass limit + 1 and read the extra row as the signal. public static async Task> GetPlanCorrectionsAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) { var rows = new List(); await using var command = postgres.CreateCommand(PlanCorrectionsSql); @@ -142,6 +150,7 @@ public static async Task> GetPlanCorrectionsAsync( command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(cap); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs index dd481e38a..c43e404f0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs @@ -38,19 +38,26 @@ internal static class DarlingQueryStoreRegressionReader { /// One regression row - the viewer's ViewerQueryStoreRegressionRow, without the /// display-formatting members. Durations and CPU are ms (converted from the stored microseconds); - /// reads are raw pages; the percents are plain deltas. + /// reads are raw pages; the percents are plain deltas. + /// The three percents are NULLABLE (#3541 A12, contract rule 5). Each divides through + /// NULLIF(baseline, 0), so a query whose baseline side is 0 — no logical reads in every capture + /// before the window, then 50,000 per execution inside it — has no ratio: the SQL returns NULL, and the + /// reader used to coerce it to 0, which published the most dramatic possible I/O regression as + /// io_regression_percent: 0, "no change". NULL stays NULL here and the tool says why. The CPU + /// percent cannot actually arrive NULL (the WHERE gate > 25 drops a NULL comparison), but it is + /// typed like its siblings so the three cannot drift in how they treat a missing denominator. public sealed record RegressionRow( string DatabaseName, long QueryId, double BaselineDurationMs, double RecentDurationMs, - double DurationRegressionPercent, + double? DurationRegressionPercent, double BaselineCpuMs, double RecentCpuMs, - double CpuRegressionPercent, + double? CpuRegressionPercent, double BaselineReads, double RecentReads, - double IoRegressionPercent, + double? IoRegressionPercent, double AdditionalDurationMs, long BaselineExecCount, long RecentExecCount, @@ -233,13 +240,13 @@ public static async Task> GetQueryStoreRegressionsAsync( reader.IsDBNull(1) ? 0 : reader.GetInt64(1), reader.IsDBNull(2) ? 0 : Convert.ToDouble(reader.GetValue(2)), reader.IsDBNull(3) ? 0 : Convert.ToDouble(reader.GetValue(3)), - reader.IsDBNull(4) ? 0 : Convert.ToDouble(reader.GetValue(4)), + reader.IsDBNull(4) ? null : Convert.ToDouble(reader.GetValue(4)), reader.IsDBNull(5) ? 0 : Convert.ToDouble(reader.GetValue(5)), reader.IsDBNull(6) ? 0 : Convert.ToDouble(reader.GetValue(6)), - reader.IsDBNull(7) ? 0 : Convert.ToDouble(reader.GetValue(7)), + reader.IsDBNull(7) ? null : Convert.ToDouble(reader.GetValue(7)), reader.IsDBNull(8) ? 0 : Convert.ToDouble(reader.GetValue(8)), reader.IsDBNull(9) ? 0 : Convert.ToDouble(reader.GetValue(9)), - reader.IsDBNull(10) ? 0 : Convert.ToDouble(reader.GetValue(10)), + reader.IsDBNull(10) ? null : Convert.ToDouble(reader.GetValue(10)), reader.IsDBNull(11) ? 0 : Convert.ToDouble(reader.GetValue(11)), reader.IsDBNull(12) ? 0 : reader.GetInt64(12), reader.IsDBNull(13) ? 0 : reader.GetInt64(13), diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSessionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSessionReader.cs index 731f4f9d8..0c78698f1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSessionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSessionReader.cs @@ -47,7 +47,20 @@ public sealed record ActiveQueryRow( long TotalElapsedTimeMs, string? ElapsedTimeFormatted, long LogicalReads, long Reads, long Writes, string? WaitType, long WaitTimeMs, int BlockingSessionId, int Dop, int ParallelWorkerCount, double GrantedQueryMemoryGb, string? TransactionIsolationLevel, int OpenTransactionCount, - string? LoginName, string? HostName, string? ProgramName, string? QueryText); + string? LoginName, string? HostName, string? ProgramName, string? QueryText) + { + /// Some row in the SAME capture names this session as its blocker (#3541 A13) — the reason a + /// WAITFOR row can be on the page. + public bool IsHeadBlocker { get; init; } + + /// For a victim: its blocker had a row in the same capture at all. False is the idle + /// open-transaction head blocker sys.dm_exec_requests never lists. + public bool BlockerInCapture { get; init; } + + /// For a victim: its blocker also passes the caller's filters, so it is in the population the + /// page is drawn from (it may still be past the page — the tool checks that). + public bool BlockerInPopulation { get; init; } + } /// One waiting-task snapshot row. public sealed record WaitingTaskRow( @@ -110,48 +123,146 @@ public static async Task> GetLatestSessionStatsAsync( /// /// The captured query snapshots over the window — the viewer's LatestQuerySnapshotsSql projected /// to the columns Lite's get_active_queries surfaces, from the base query_snapshots table (the - /// viewer reads base here too). granted_query_memory_gb is numeric(18,2) → double precision. WAITFOR - /// shells are trimmed. $1 server_id, $2/$3 window (naive UTC). + /// viewer reads base here too). granted_query_memory_gb is numeric(18,2) → double precision. + /// $1 server_id, $2/$3 window (naive UTC), $4 row cap, $5 database filter (NULL = all), $6 blocking_only. + /// + /// Every filter is part of the query (#3541 A13). This read used to return the whole window + /// unfiltered and unbounded; the tool then applied database_name and blocking_only in C#, + /// took limit, and published the pre-filter row count as total_snapshots beside the page — + /// so the "total" was of a different population from the rows, and a page could be empty while the + /// window held matches. Now the filters are predicates on the same statement, the population count is + /// COUNT(*) OVER () on the FILTERED rows above the parameterised LIMIT (the #3613 idiom), and + /// the cap is the caller's, fetched at limit + 1 so truncation is observed rather than inferred. + /// + /// Head blockers are never stripped (#3541 A13). The WAITFOR trim exists to drop the idle + /// shells a monitoring session leaves in dm_exec_requests, but the classic head blocker IS a session + /// sitting in WAITFOR with an open transaction — and this read dropped it while its victims' + /// blocking_session_id pointed at the session that was no longer on the page. A row is kept + /// whatever its text when some row in the SAME capture names it as its blocker. Same capture, not same + /// window: session ids are reused, so "any row in the window points at this session id" would resurrect + /// unrelated sessions from other snapshots, which is what the old C# arm did. + /// + /// The two blocker-presence flags. blocker_in_capture: the victim's blocker had a row + /// in the same capture at all — FALSE is the other classic head blocker, a session idle in an open + /// transaction, which sys.dm_exec_requests never lists and so was never captured; the tool says so on the + /// row rather than leaving a dangling id. blocker_in_population: the blocker also passes the + /// caller's own filters (a head blocker in another database under a database_name filter does + /// not) — the caller asked for that database, so the row is honoured and the victim says its blocker was + /// filtered. A blocker that passes both but falls past the page is detected by the tool, which has the + /// page. /// public const string ActiveQueriesSql = """ + WITH window_rows AS ( + SELECT + collection_time, + session_id, + database_name, + status, + cpu_time_ms, + total_elapsed_time_ms, + elapsed_time_formatted, + logical_reads, + reads, + writes, + wait_type, + wait_time_ms, + blocking_session_id, + dop, + parallel_worker_count, + CAST(granted_query_memory_gb AS double precision) AS granted_query_memory_gb, + transaction_isolation_level, + open_transaction_count, + login_name, + host_name, + program_name, + query_text + FROM query_snapshots + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + ), + heads AS ( + /* (capture, session) pairs some victim in the SAME capture points at. */ + SELECT DISTINCT collection_time, blocking_session_id AS session_id + FROM window_rows + WHERE blocking_session_id > 0 + ), + population AS ( + SELECT + w.*, + (h.session_id IS NOT NULL) AS is_head_blocker, + EXISTS ( + SELECT 1 + FROM window_rows b + WHERE b.collection_time = w.collection_time + AND b.session_id = w.blocking_session_id + ) AS blocker_in_capture + FROM window_rows AS w + LEFT JOIN heads AS h + ON h.collection_time = w.collection_time + AND h.session_id = w.session_id + WHERE (w.query_text NOT LIKE 'WAITFOR%' OR h.session_id IS NOT NULL) + AND ($5::text IS NULL OR w.database_name = $5) + AND (NOT $6::boolean OR w.blocking_session_id > 0 OR h.session_id IS NOT NULL) + ) SELECT - collection_time, - session_id, - database_name, - status, - cpu_time_ms, - total_elapsed_time_ms, - elapsed_time_formatted, - logical_reads, - reads, - writes, - wait_type, - wait_time_ms, - blocking_session_id, - dop, - parallel_worker_count, - CAST(granted_query_memory_gb AS double precision), - transaction_isolation_level, - open_transaction_count, - login_name, - host_name, - program_name, - query_text - FROM query_snapshots - WHERE server_id = $1 - AND collection_time >= $2 - AND collection_time <= $3 - AND query_text NOT LIKE 'WAITFOR%' - ORDER BY collection_time DESC, cpu_time_ms DESC + p.collection_time, + p.session_id, + p.database_name, + p.status, + p.cpu_time_ms, + p.total_elapsed_time_ms, + p.elapsed_time_formatted, + p.logical_reads, + p.reads, + p.writes, + p.wait_type, + p.wait_time_ms, + p.blocking_session_id, + p.dop, + p.parallel_worker_count, + p.granted_query_memory_gb, + p.transaction_isolation_level, + p.open_transaction_count, + p.login_name, + p.host_name, + p.program_name, + p.query_text, + p.is_head_blocker, + p.blocker_in_capture, + EXISTS ( + SELECT 1 + FROM population q + WHERE q.collection_time = p.collection_time + AND q.session_id = p.blocking_session_id + ) AS blocker_in_population, + COUNT(*) OVER () AS population_count + FROM population AS p + ORDER BY p.collection_time DESC, p.cpu_time_ms DESC + LIMIT $4 """; - public static async Task> GetActiveQueriesAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + /// The filtered page plus the filtered population's size, from one statement. + public sealed record ActiveQueriesPage(List Rows, long PopulationCount); + + /// + /// The newest snapshots over the window that pass the filters, with the filtered + /// population's count (#3541 A13). Callers detecting truncation pass limit + 1 and read the extra + /// row as the signal. null = every database; + /// keeps victims and the head blockers of victims in the same capture. + /// + public static async Task GetActiveQueriesAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, + string? databaseName = null, bool blockingOnly = false, CancellationToken cancellationToken = default) { var rows = new List(); + long populationCount = 0; await using var command = postgres.CreateCommand(ActiveQueriesSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + DarlingMcpReadParameters.AddInt(command, cap); + DarlingMcpReadParameters.AddNullableText(command, databaseName); + DarlingMcpReadParameters.AddBoolean(command, blockingOnly); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -177,10 +288,16 @@ public static async Task> GetActiveQueriesAsync( reader.IsDBNull(18) ? null : reader.GetString(18), reader.IsDBNull(19) ? null : reader.GetString(19), reader.IsDBNull(20) ? null : reader.GetString(20), - reader.IsDBNull(21) ? null : reader.GetString(21))); + reader.IsDBNull(21) ? null : reader.GetString(21)) + { + IsHeadBlocker = !reader.IsDBNull(22) && reader.GetBoolean(22), + BlockerInCapture = !reader.IsDBNull(23) && reader.GetBoolean(23), + BlockerInPopulation = !reader.IsDBNull(24) && reader.GetBoolean(24), + }); + populationCount = reader.GetInt64(25); } - return rows; + return new ActiveQueriesPage(rows, populationCount); } /* ─────────────────────────── waiting tasks (base table over the window) ─────────────────────────── */ @@ -188,7 +305,11 @@ public static async Task> GetActiveQueriesAsync( /// /// The recently-captured waiting tasks over the window — the base waiting_tasks table (there is no /// v_waiting_tasks view), newest first then longest wait. resource_description is stored but always - /// NULL (the collector no longer collects it). $1 server_id, $2/$3 window (naive UTC). + /// NULL (the collector no longer collects it). $1 server_id, $2/$3 window (naive UTC), $4 row cap. + /// + /// The cap is a PARAMETER, not a literal (#3541 A3). It was LIMIT 500 under a tool that + /// advertised limit, applied it with Take(limit), and then published a bare envelope with no + /// window, no count and no bound — so a caller could not tell thirty tasks from thirty of five thousand. /// public const string WaitingTasksSql = """ SELECT @@ -205,16 +326,19 @@ FROM waiting_tasks AND collection_time <= $3 AND wait_type IS NOT NULL ORDER BY collection_time DESC, wait_duration_ms DESC - LIMIT 500 + LIMIT $4 """; + /// The newest waiting tasks over the window. Callers detecting truncation + /// pass limit + 1 and read the extra row as the signal. public static async Task> GetWaitingTasksAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int cap, CancellationToken cancellationToken = default) { var rows = new List(); await using var command = postgres.CreateCommand(WaitingTasksSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + DarlingMcpReadParameters.AddInt(command, cap); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs index e7bd1d593..62fc6b542 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingStoreMetricsReader.cs @@ -27,10 +27,21 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// per-server daily ingest rate (whole-store daily growth divided by the enabled-server count), computed in /// — pure, so it is unit-tested without a store. /// -/// Plus one read that is not a metric at all: asks whether +/// Plus two reads that are not metrics at all: asks whether /// timescaledb_information.job_history — the route the tool's description sends a maximum question -/// to, since this series cannot answer one — is actually recording (#3175). It qualifies the redirect, so -/// a caller who follows it can tell a census from an empty table. +/// to, since this series cannot answer one — is switched on (#3175), and +/// asks whether the connection doing the asking would SEE its rows and how many it does see (#3574). Together +/// they qualify the redirect, so a caller who follows it can tell a census from an empty table — and can +/// tell an empty table from a table the view is hiding from them. Where that connection is one the view +/// shows nothing (managed mode's mcp role), decodes the +/// OWNER's reading that the hourly sweep persisted into the series, so the block can still carry a +/// measurement and say whose it is. +/// +/// And, since #3582, the reads that let the tool state its own COVERAGE: +/// reconciles the per-object rows of one sweep against that sweep's pg_database_size, +/// reads the catalog facts about each aggregate the series does +/// not carry (compression enabled, which policies exist), and names +/// the biggest relations inside the other row so that number is something a reader can act on. /// internal static class DarlingStoreMetricsReader { @@ -84,11 +95,13 @@ FROM collect.store_metrics /// MAXIMUM question to, because the daily series cannot answer one — is actually recording (#3175). /// /// Why this read exists at all. An empty job_history and a quiet fleet are the same - /// result set. TimescaleDB records nothing there unless - /// timescaledb.enable_job_execution_logging is on, and it defaults OFF, so a maximum over the - /// table returns zero rows on an unhealed store and that reads as "no run exceeded the line". - /// Redirecting a caller to an instrument without telling them whether it is switched on is how the - /// wrong conclusion gets drawn from a correct query. + /// result set. TimescaleDB records a SUCCESSFUL run there only while + /// timescaledb.enable_job_execution_logging is on, and it defaults OFF — a FAILED run's row is + /// written regardless (2.28.1 job_stat_history.c: the failure path logs unconditionally, the + /// success path is gated; measured on a fresh rig with the GUC off, the telemetry job's one failure was + /// the view's one row) — so a maximum over the table on an unhealed store is a census of FAILURES + /// that reads as "no run exceeded the line". Redirecting a caller to an instrument without telling + /// them which runs it is writing is how the wrong conclusion gets drawn from a correct query. /// /// The EFFECTIVE value and its source, never the presence of the managed conf block. /// postgresql.auto.conf is read after postgresql.conf, so an @@ -134,6 +147,111 @@ FROM collect.store_metrics FROM pg_settings WHERE name = $1"; + /// + /// The evidence behind : what THIS connection actually + /// sees in timescaledb_information.job_history, and whether the view would show it anything at all + /// (#3574). The GUC read above answers "is the instrument switched on"; the reader's real question + /// is "will I see its output", and between those two sits a role-membership filter nothing else on + /// this surface mentioned. + /// + /// THE FALSE-QUIET ARM #3175 DID NOT KNOW ABOUT: recording on, rows present, reader filtered. + /// job_history is a security_barrier view whose definition on TimescaleDB 2.28.1 ends with + /// + /// WHERE (pg_catalog.pg_has_role(current_user, + /// (SELECT pg_catalog.pg_get_userbyid(datdba) + /// FROM pg_catalog.pg_database + /// WHERE datname = current_database()), + /// 'MEMBER') IS TRUE + /// OR pg_catalog.pg_has_role(current_user, owner, 'MEMBER') IS TRUE); + /// + /// so a row is visible only to a member of the database owner's role or of the job's owner role + /// (_timescaledb_catalog.bgw_job.owner is regrole NOT NULL DEFAULT current_role — a job + /// belongs to whoever created it, which for every policy this product adds is the service's owner role). + /// The base table is not a back door: pre_install/tables.sql ends with + /// REVOKE ALL ON _timescaledb_internal.bgw_job_stat_history FROM PUBLIC, so the two filtered views + /// (job_history, job_errors) are the only way in. The trap is that the two views a reader + /// checks FIRST are not filtered. timescaledb_information.jobs has no WHERE clause at all and + /// job_stats has none either, so a role that can see all 110 jobs and every one of their + /// total_runs reasonably assumes it can see their history — and reads zero rows, forever, on a + /// store that is recording perfectly. Measured on a production store on 2.28.1: the GUC effective + /// on (sighup context, pending_restart = false), all 110 jobs owned by the service's owner + /// role, two independent reads as the least-privilege admin role counted 0 rows ever, and + /// the owner role's read of the same view returned every hourly run. That zero was declared "unknowable" + /// in a real postmortem before anyone read the view's definition. Recording was never broken and the GUC + /// never lied; the block just never said whose eyes the rows are visible to, and never proved rows exist. + /// + /// THIS READ IS ITSELF SUBJECT TO THE FILTER, and that is why it evaluates the predicate rather + /// than assuming it passes. In managed mode the MCP host connects as the dedicated least-privilege + /// mcp role (DarlingMcpHostService; DarlingManagedRoles grants it SELECT and a few + /// narrow writes, never membership in the owner role) — so on a managed store THIS connection is exactly + /// the kind of reader the view shows nothing to, and a bare count(*) here would have reported + /// rows_observed = 0 on every managed store and manufactured the very contradiction this issue is + /// about. The read therefore also returns current_user and evaluates the view's own two + /// pg_has_role tests for it: membership in the database owner (which sees everything) and, per + /// job, membership in that job's owner. From those the caller knows whether the count that follows is a + /// census, a partial census, or a zero the view produced by construction — and the note says which, in + /// so many words, naming the role. On a bring-your-own store whose connection string is the owner role + /// the count IS the census and the flag becomes self-proving; on a managed store the block says it + /// cannot see, and says who can, which is the sentence that would have ended the postmortem in a minute. + /// + /// The population half rides in the same statement, from the UNFILTERED view. A zero is + /// readable only when the instrument would have caught the event AND the event had a chance to occur, so + /// beside the history count the read takes, from job_stats, how many jobs started a run inside the + /// same window and the newest start it knows of. TimescaleDB writes the history row at job START when + /// the GUC is on (bgw_job_stat_history_mark_start inserts it with finish, pid and outcome NULL and + /// the finish updates it; a failure is written regardless of the GUC), so a job that started inside the + /// window while logging was on has a row with start_time inside the window — no waiting on a + /// finish. recording = on, a reader the predicate admits, zero rows, and jobs that started in the + /// window is the contradiction, and the note calls it one. It does not manufacture certainty about the + /// cause: logging switched on AFTER the last of those starts is the benign shape (the v11 heal lands on a + /// service-owned server start, and nothing before that point was written to recover), and the note says + /// how to settle it — re-read after the next hourly run — rather than pronouncing. + /// + /// A FIXED 24-HOUR WINDOW, not the tool's days_back. Three reasons, in order of + /// weight. The question this answers is CURRENT — is the instrument writing now — and a 30-to-400-day + /// forecasting window would let ten days of rows written after a heal hide a recording that stopped + /// yesterday. Every job this product schedules runs at least daily (CAGG refreshes and the compression + /// tick hourly, retention daily), so any 24-hour window on a live store contains starts, which is what + /// lets the job_stats count prove the population half instead of assuming it. And the view's + /// own Job History Log Retention Policy drops rows after one month by default, so a window past + /// that would count a table the retention job had already trimmed and call the trimming "no rows". + /// $1 is the window start. + /// + /// $1 IS BOUND AS timestamptz WITH Kind = Utc, WHICH IS THE INVERSE OF THIS + /// CODEBASE'S RULE, AND DELIBERATELY SO. Every collector column in the store is naive UTC and the + /// discipline everywhere else is to strip Kind before binding, because a timestamptz parameter against a + /// naive column makes PostgreSQL convert the naive side at the session's TimeZone. These columns are the + /// other way round: bgw_job_stat_history.execution_start and bgw_job_stat.last_start are + /// declared TIMESTAMPTZ in TimescaleDB's own catalog, so here a NAIVE bind would be the bug — the + /// parameter, not the column, would be converted at the session zone and the window would skew by the + /// host's offset. The parameter type is stated explicitly rather than inferred so the intent survives a + /// caller passing a DateTime of the wrong Kind. + /// + /// '-infinity' is TimescaleDB's never-ran sentinel in last_run_started_at (not NULL — + /// the lesson from #1760), so the newest start + /// NULLIFs it away; the window predicate needs no guard because -infinity >= $1 is simply false. + /// IS TRUE on each pg_has_role mirrors the view, whose second test can meet a NULL owner + /// (it LEFT JOINs the job catalog, so a history row whose job has since been deleted has none, and the + /// strict function yields NULL) — a NULL must read as "not a member" rather than poison a count. Here + /// the owner comes from the jobs view and cannot be NULL; the guard is kept so the two predicates + /// stay textually the view's own. + /// + /// The text itself lives on and this is + /// an alias, because the string gained a second consumer with the managed-mode self-proof: the + /// hourly sweep embeds it verbatim in to run it as + /// the OWNER role and persist the answer (the Storage project cannot reference this one). Two copies of + /// a nine-column predicate would drift without erroring; one string cannot. The reasoning stays here, + /// beside the record that interprets the columns. + /// + public const string JobHistoryEvidenceSql = StoreSelfMetrics.JobHistoryEvidenceSql; + + /// The evidence window counts over, in hours. Fixed, not + /// days_back — the paragraph on that constant says why. Published in the response beside the + /// count so the number never travels without its denominator. An alias of the sweep's constant for the + /// reason the SQL is: the owner's persisted count and this connection's live one must be over the same + /// window or the block would compare unlike things. + public const int JobHistoryEvidenceWindowHours = StoreSelfMetrics.JobHistoryEvidenceWindowHours; + /// /// The four distinguishable states of the job_history precondition. Four rather than a bool /// because three of them would otherwise collapse into "not on", and the whole defect being reported @@ -154,8 +272,10 @@ public enum JobExecutionLoggingStatus /// zero-rows paragraph on for the measurement. NotRegistered, - /// Registered and off. job_history is not recording; an empty result from it means - /// the instrument is off, not that nothing happened. + /// Registered and off. job_history records FAILED runs only — TimescaleDB writes a + /// failure's row regardless of this setting and a success's only while it is on — so a result from + /// it is a census of failures, not of runs: an empty one means no failure was recorded, not that + /// nothing happened, and a non-empty one is not a sign the setting is secretly on. Off, /// Registered and on. job_history carries one row per run from the point logging @@ -244,8 +364,725 @@ and reporting that as Unreadable would put a measurement-shaped word on an act o } } + /// + /// Whether produced a reading (#3574). Three states, not a nullable + /// count, for the reason has four: a count this read did not + /// obtain, a count there was nothing to obtain, and a count of zero are three different facts about an + /// empty job_history, and the whole defect class is one of them being read as another. + /// + public enum JobHistoryEvidenceStatus + { + /// The read did not complete. Every evidence field is null and the flag stays a GUC echo + /// — reported as such, never as "zero rows". + Unreadable, + + /// Not attempted, because the GUC read said the view does not exist on this connection + /// (). A plain-PostgreSQL store has no + /// job_history to count, and a failed read of an absent view would report as Unreadable — + /// a fault-shaped word for a store that has no fault. + NotApplicable, + + /// The read completed. The counts are what this connection saw, and + /// says whether what it saw is what is there. + Observed, + } + + /// + /// How much of job_history the view's ownership predicate lets THIS reader see (#3574) — derived + /// from the two pg_has_role facts the read evaluates, never assumed. + /// + public enum JobHistoryVisibility + { + /// Not established: the evidence read did not complete, or there are no jobs to be a + /// member of the owner of. + Unknown, + + /// The reader is a member of neither the database owner nor any job's owner. The view + /// returns it NOTHING by construction, so a zero count here is the filter, not the table. This is + /// the managed-mode mcp role's reading on every store. + None, + + /// The reader is a member of some jobs' owners but not all, and not of the database owner. + /// The count is a census of those jobs only. + Partial, + + /// The reader is a member of the database owner, or of every job's owner. The count is a + /// census — the one state in which a zero says something about recording. + All, + } + + /// + /// One reading of : who read, what the view's predicate lets them + /// see, what they saw, and whether anything happened for them to see (#3574). One value, so a caller + /// cannot take the count and drop the role it was counted through — which is precisely the omission + /// this exists to close. + /// + /// Whether the read completed; every other field is null unless + /// . + /// current_user on the connection that counted. + /// The view's first pg_has_role test, evaluated for + /// this reader: membership in the database owner's role, which sees every row regardless of job owner. + /// Every job in timescaledb_information.jobs — the UNFILTERED view, so + /// this is what any role sees and the denominator the visibility fraction is stated against. + /// The view's second test, evaluated per job: how many jobs' owner + /// roles this reader is a member of. + /// History rows with a start inside the window that the view showed this + /// reader. A census only when is . + /// The newest history start the view showed this reader, over all time — + /// null when it showed none. UTC. + /// Jobs whose job_stats.last_run_started_at falls inside the + /// window — the population half, from the unfiltered view, so it holds whatever the reader's + /// visibility. + /// The newest job start job_stats knows of, over all time — + /// null when no job has ever run. UTC. + public sealed record JobHistoryEvidence( + JobHistoryEvidenceStatus Status, + string? ReaderRole, + bool? ReaderIsDatabaseOwnerMember, + long? JobCount, + long? OwnerMemberJobCount, + long? RowsObserved, + DateTime? NewestRowAt, + long? JobsRunInWindow, + DateTime? NewestRunStartedAt) + { + /// The reading for a connection on which the view does not exist — every field null, + /// status . + public static JobHistoryEvidence NotApplicable { get; } = + new(JobHistoryEvidenceStatus.NotApplicable, null, null, null, null, null, null, null, null); + + /// The reading for a read that did not complete — every field null, status + /// . + public static JobHistoryEvidence Unreadable { get; } = + new(JobHistoryEvidenceStatus.Unreadable, null, null, null, null, null, null, null, null); + + /// + /// How many jobs' history the view lets this reader see: every job when the reader is a member of + /// the database owner (the view's first test short-circuits the second), otherwise the per-job + /// membership count. Null unless observed. + /// + public long? HistoryVisibleJobCount => + Status != JobHistoryEvidenceStatus.Observed ? null + : ReaderIsDatabaseOwnerMember == true ? JobCount + : OwnerMemberJobCount; + + /// + /// The reader's standing under the view's predicate, derived from the two membership facts and the + /// job count. when the read did not complete or there are + /// no jobs — with nothing to be an owner of, "none" and "all" would both be vacuously true, and a + /// note built on either would be inventing a measurement. + /// + public JobHistoryVisibility Visibility + { + get + { + if (Status != JobHistoryEvidenceStatus.Observed || JobCount is not > 0) + { + return JobHistoryVisibility.Unknown; + } + + if (ReaderIsDatabaseOwnerMember == true) + { + return JobHistoryVisibility.All; + } + + return OwnerMemberJobCount switch + { + null or 0 => JobHistoryVisibility.None, + var n when n >= JobCount => JobHistoryVisibility.All, + _ => JobHistoryVisibility.Partial, + }; + } + } + + /// + /// The new finding class (#3574): the GUC says recording, the view admits this reader to every job's + /// history, jobs started runs inside the window, and the reader saw NO rows for them. True only when + /// all four hold — a zero read through a filtered role contradicts nothing, a zero with no runs in + /// the window proves nothing, and a zero with the GUC off is the #3175 arm, not this one. The caller + /// supplies because this record deliberately does not carry the GUC + /// reading; the two are read separately and fail separately. + /// + public bool ContradictsRecording(bool recording) => + recording + && Status == JobHistoryEvidenceStatus.Observed + && Visibility == JobHistoryVisibility.All + && RowsObserved == 0 + && JobsRunInWindow is > 0; + } + + /// + /// Reads over the window ending now and starting + /// ago. Failure-isolated to + /// , independently of the GUC read: the two are separate + /// statements on separate checkouts, so either can fail while the other answers, and a reading that + /// collapsed both into one status would report the GUC as unknown because a count timed out. Takes the + /// GUC reading only to skip the view a plain-PostgreSQL store does not have — the read is not attempted + /// for , and the response says NotApplicable rather + /// than dressing an absent view up as a failed read. No logger, for the reason + /// gives. + /// + public static async Task GetJobHistoryEvidenceAsync( + NpgsqlDataSource postgres, + JobExecutionLoggingReading logging, + CancellationToken cancellationToken = default) + { + if (logging is null) + { + throw new ArgumentNullException(nameof(logging)); + } + + if (logging.Status == JobExecutionLoggingStatus.NotRegistered) + { + return JobHistoryEvidence.NotApplicable; + } + + try + { + await using var command = postgres.CreateCommand(JobHistoryEvidenceSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + + /* Kind = Utc and an EXPLICIT timestamptz, against timestamptz columns — the inverse of every + other bind on this surface, and the paragraph on JobHistoryEvidenceSql says why. Stated rather + than inferred so that a caller's DateTime of another Kind cannot quietly turn this into the + naive bind that would skew the window by the session zone. */ + var windowStartUtc = DateTime.SpecifyKind( + DateTime.UtcNow.AddHours(-JobHistoryEvidenceWindowHours), DateTimeKind.Utc); + command.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.TimestampTz, + Value = windowStartUtc, + }); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + /* A single-row SELECT of scalar subqueries always yields one row; no row is a shape this + read does not understand, and Unreadable is the only honest word for it. */ + return JobHistoryEvidence.Unreadable; + } + + return new JobHistoryEvidence( + JobHistoryEvidenceStatus.Observed, + reader.IsDBNull(0) ? null : reader.GetString(0), + reader.IsDBNull(1) ? null : reader.GetBoolean(1), + reader.IsDBNull(2) ? null : reader.GetInt64(2), + reader.IsDBNull(3) ? null : reader.GetInt64(3), + reader.IsDBNull(4) ? null : reader.GetInt64(4), + reader.IsDBNull(5) ? null : DateTime.SpecifyKind(reader.GetDateTime(5), DateTimeKind.Utc), + reader.IsDBNull(6) ? null : reader.GetInt64(6), + reader.IsDBNull(7) ? null : DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc)); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + /* Same cancellation discipline as the GUC read: a caller's own stop is not a measurement. */ + return JobHistoryEvidence.Unreadable; + } + } + + /// + /// Whether the series held an OWNER's reading of job_history for the block to lean on (#3574). + /// Four states rather than a nullable count, for the reason every other status on this surface has + /// more than two: "the sweep has never recorded one", "it recorded one but could not see", "it + /// recorded one too long ago to speak for now" and "here is the count" are four different facts about + /// the same absent-or-present number. + /// + public enum OwnerJobHistoryEvidenceStatus + { + /// No job_history row in the series. A store whose sweep predates this row, a + /// plain-PostgreSQL store (the arm is TimescaleDB-gated), or a sweep that has not completed since + /// the service started on this build. + Absent, + + /// A row exists, but the sweep's role was itself not admitted to every job's history, so + /// it recorded no count (row_count NULL). The role is named so the reader knows which + /// connection to fix; the population half is still carried. + Filtered, + + /// A row exists and carries a census, but its metric_time is older than + /// . Shown with its age; NOT used for a verdict about now. + Stale, + + /// A fresh census from an admitted role. The one state in which the owner's zero says + /// something about recording. + Observed, + } + + /// + /// How old the owner's job_history row may be and still speak for the present, in hours. The + /// sweep is hourly; over 30 days on one production store the whole-store row's gaps had a mean of + /// 58.6 minutes and a largest-that-happened of 2.47 hours (the + /// paragraph, with its caveat that the maximum is a window artefact). Three hours is the first whole + /// hour past that observed worst case: a row older than this means at least two consecutive sweeps + /// did not land, which the sweep's own Warning line already reports, and a 24-hour count taken that + /// long ago describes a window that no longer overlaps the present enough to adjudicate a + /// contradiction about it — the GUC may have been healed since. The row is still shown when stale, + /// with its age; only the verdict is withheld. + /// + public const int OwnerEvidenceFreshHours = 3; + + /// + /// The OWNER's reading of job_history, decoded from the object_kind = 'job_history' row + /// the hourly sweep persists (#3574) — the managed-mode self-proof. The MCP host reads as the mcp + /// role, which the view's ownership filter shows nothing; the sweep runs as the owner, which it admits. + /// This record carries what the owner saw, WHEN it saw it, and whether that is recent enough to stand + /// beside this connection's own verdict. One value, so the count cannot travel without its instant or + /// its role — the omission #3574 is about, one level down. + /// + /// Whether there is a usable reading; see . + /// The role the sweep counted as (object_name). Null when Absent. + /// The sweep's metric_time, UTC — the instant the count is true of. Null when Absent. + /// How long before nowUtc the sweep ran. Null when Absent. + /// The evidence window the row counted over, from schedule_interval_ms. + /// History rows with a start inside the window, as the owner saw them + /// (row_count). Null when Absent or Filtered — never a plausible zero for a count nobody made. + /// The newest history start the owner had ever seen at the sweep, UTC — + /// metric_time minus the persisted age. Null when the owner had seen none. + /// Jobs whose newest start fell inside the window (total_runs), + /// from the unfiltered job_stats — the population half. + public sealed record OwnerJobHistoryEvidence( + OwnerJobHistoryEvidenceStatus Status, + string? ReaderRole, + DateTime? ObservedAt, + double? AgeHours, + double? WindowHours, + long? RowsObserved, + DateTime? NewestRowAt, + long? JobsRunInWindow) + { + /// The reading when the series holds no job_history row — every field null. + public static OwnerJobHistoryEvidence Absent { get; } = + new(OwnerJobHistoryEvidenceStatus.Absent, null, null, null, null, null, null, null); + + /// + /// The contradiction (#3574), judged from the OWNER's numbers: the GUC says recording NOW, the owner + /// — a reader the view admits to every job's history — saw NO rows over its window, and jobs started + /// runs inside that window. Only for a fresh, admitted reading: a stale row cannot say what the + /// instrument is doing now (the GUC may have been healed since it was taken), and a filtered row + /// made no count. The benign cause and how to settle it are the same as for the connection's own + /// contradiction and the note names them. + /// + public bool ContradictsRecording(bool recording) => + recording + && Status == OwnerJobHistoryEvidenceStatus.Observed + && RowsObserved == 0 + && JobsRunInWindow is > 0; + + /// + /// Decodes the newest job_history row out of the latest-per-object read. Pure, so the + /// mapping the sweep's column overloads define ( class summary) is + /// unit-tested without a store: row_count is the count, total_runs the population, + /// schedule_interval_ms the window, and last_run_duration_ms the newest row's AGE at + /// the sweep, turned back into an instant here so nothing downstream ever sees the overload. + /// is taken rather than read so the staleness verdict is testable. + /// + public static OwnerJobHistoryEvidence FromLatest(IReadOnlyList latest, DateTime nowUtc) + { + if (latest is null) + { + throw new ArgumentNullException(nameof(latest)); + } + + /* The latest read is DISTINCT ON (kind, name), so a store whose owner role was renamed could hold + two job_history rows under two names; the newest sweep's is the one that speaks for now. */ + StoreMetricRow? row = null; + foreach (var candidate in latest) + { + if (candidate.ObjectKind == StoreSelfMetrics.JobHistoryObjectKind + && (row is null || candidate.MetricTime > row.MetricTime)) + { + row = candidate; + } + } + + if (row is null) + { + return Absent; + } + + /* metric_time is naive UTC by the store contract; it is the instant the count is true of. */ + var observedAt = DateTime.SpecifyKind(row.MetricTime, DateTimeKind.Utc); + var ageHours = (DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc) - observedAt).TotalHours; + double? windowHours = row.ScheduleIntervalMs is { } w ? w / 3_600_000.0 : null; + DateTime? newestRowAt = row.LastRunDurationMs is { } age ? observedAt.AddMilliseconds(-age) : null; + + var status = row.RowCount is null ? OwnerJobHistoryEvidenceStatus.Filtered + : ageHours > OwnerEvidenceFreshHours ? OwnerJobHistoryEvidenceStatus.Stale + : OwnerJobHistoryEvidenceStatus.Observed; + + return new OwnerJobHistoryEvidence( + status, + row.ObjectName, + observedAt, + Math.Round(ageHours, 2), + windowHours, + row.RowCount, + newestRowAt, + row.TotalRuns); + } + } + + /* ---------------- #3582: coverage, reconciled ---------------- */ + + /// + /// How far the inventory's rows may miss pg_database_size and still be called reconciled, as a + /// fraction of the database: 1%. Below this the residual is the database directory's non-relation + /// files plus whatever moved between the sweep's statements; above it something is not being + /// attributed to any row and the note says so as a finding. Paired with + /// so a small store is not failed on a fixed overhead. + /// + public const double ReconciliationTolerancePercent = 1.0; + + /// + /// The absolute floor under : 64 MiB. Measured on a fresh + /// PG18/TimescaleDB 2.28.1 rig the residual was 161,471 bytes on a 17 MiB database — exactly + /// pg_database_size minus the sum of every local relation, i.e. pg_internal.init, + /// pg_filenode.map, PG_VERSION and friends — which is already 0.9% of that store and + /// would trip a percent-only bar on anything smaller. A fixed floor sized generously above that + /// overhead lets the percentage do the work where the percentage means something. + /// + public const long ReconciliationToleranceFloorBytes = 64L * 1024 * 1024; + + /// + /// One sweep's inventory, reconciled against its own pg_database_size (#3582). Computed over the + /// rows that share the store row's metric_time — the same sweep — because the latest read takes + /// each object's newest row and a sweep that died half-way leaves older rows behind for the kinds it + /// never reached; summing those against a newer database figure would manufacture a gap. + /// + /// The store row's metric_time — the sweep every figure here comes from. + /// pg_database_size as that sweep recorded it. + /// Byte totals per byte-bearing kind in that sweep, in the sweep's kind + /// order. Kinds with no row in the sweep are absent from the map, never zero. + /// Bytes under NAMED objects: hypertables, continuous aggregates, payload + /// dimensions and named tables. The issue's "inventory covers N%" numerator. + /// Every byte the sweep put in some row: the enumerated bytes plus the + /// two catch-all rows. The reconciliation numerator. + /// The other row's bytes; null when that row is missing from the sweep. + /// The other row's relation count; null likewise. + /// The system row's bytes; null when missing. + /// The system row's relation count; null likewise. + /// DatabaseBytes - AttributedBytes. Expected small and non-zero (the + /// directory's non-relation files, plus movement between statements); can be negative. + /// Latest rows whose metric_time is NOT the store row's — objects the + /// newest sweep did not reach. Non-zero means the sweep is not completing and the note says so. + public sealed record InventoryReconciliation( + DateTime SweepAt, + long DatabaseBytes, + IReadOnlyDictionary BytesByKind, + long EnumeratedBytes, + long AttributedBytes, + long? UnenumeratedBytes, + int? UnenumeratedRelationCount, + long? SystemBytes, + int? SystemRelationCount, + long ResidualBytes, + int StaleRowCount) + { + /// Percent of the database under named objects — the coverage statement. Null on a zero-byte + /// database, never a division by zero dressed as a hundred. + public double? EnumeratedPercent => DatabaseBytes > 0 ? Math.Round(100.0 * EnumeratedBytes / DatabaseBytes, 2) : null; + + /// Percent of the database some row attributes — the reconciliation statement. + public double? AttributedPercent => DatabaseBytes > 0 ? Math.Round(100.0 * AttributedBytes / DatabaseBytes, 2) : null; + + /// Both catch-all rows present in the sweep, so the attribution is complete enough to judge. + /// Without them the residual is the un-enumerated bytes themselves and says nothing. + public bool CatchAllPresent => UnenumeratedBytes is not null && SystemBytes is not null; + + /// The bar the residual is judged against: the larger of the percent and the floor. + public long ToleranceBytes => Math.Max( + (long)Math.Ceiling(DatabaseBytes * ReconciliationTolerancePercent / 100.0), + ReconciliationToleranceFloorBytes); + + /// true when the catch-all rows are present and the residual is inside the bar. A + /// false is a finding: bytes the database holds that no row of the inventory accounts for. + public bool Reconciled => CatchAllPresent && Math.Abs(ResidualBytes) <= ToleranceBytes; + } + + /// + /// The kinds whose total_bytes are bytes under a NAMED object. The catch-all kinds are not here + /// by definition; the store row is the denominator; the job kinds carry no bytes. + /// + private static readonly string[] EnumeratedKinds = + { + StoreSelfMetrics.HypertableObjectKind, + StoreSelfMetrics.ContinuousAggregateObjectKind, + StoreSelfMetrics.DimensionObjectKind, + StoreSelfMetrics.TableObjectKind, + }; + + /// + /// Reconciles the newest sweep's inventory against its own database figure (#3582). Pure. Null when + /// there is no store row to reconcile against — the tool then says coverage is unknown rather than + /// computing a percentage of nothing. Rows from other sweeps are counted, not summed. + /// + public static InventoryReconciliation? ComputeInventory(IReadOnlyList latest) + { + if (latest is null) + { + throw new ArgumentNullException(nameof(latest)); + } + + StoreMetricRow? store = null; + foreach (var row in latest) + { + if (row.ObjectKind == StoreSelfMetrics.StoreObjectKind && row.TotalBytes is not null + && (store is null || row.MetricTime > store.MetricTime)) + { + store = row; + } + } + + if (store is null) + { + return null; + } + + var byKind = new Dictionary(StringComparer.Ordinal); + long? other = null, system = null; + int? otherCount = null, systemCount = null; + var stale = 0; + + foreach (var row in latest) + { + if (row.ObjectKind == StoreSelfMetrics.StoreObjectKind) + { + continue; + } + + if (row.MetricTime != store.MetricTime) + { + stale++; + continue; + } + + if (row.TotalBytes is not { } bytes) + { + continue; + } + + byKind[row.ObjectKind] = byKind.TryGetValue(row.ObjectKind, out var soFar) ? soFar + bytes : bytes; + + if (row.ObjectKind == StoreSelfMetrics.OtherObjectKind) + { + other = bytes; + otherCount = row.ChunkCount; + } + else if (row.ObjectKind == StoreSelfMetrics.SystemObjectKind) + { + system = bytes; + systemCount = row.ChunkCount; + } + } + + long enumerated = 0; + foreach (var kind in EnumeratedKinds) + { + if (byKind.TryGetValue(kind, out var bytes)) + { + enumerated += bytes; + } + } + + var attributed = enumerated + (other ?? 0) + (system ?? 0); + + return new InventoryReconciliation( + store.MetricTime, + store.TotalBytes!.Value, + byKind, + enumerated, + attributed, + other, + otherCount, + system, + systemCount, + store.TotalBytes.Value - attributed, + stale); + } + + /// + /// The catalog facts about each continuous aggregate the series does not carry (#3582), read LIVE at + /// tool time the way #2813 reads retention holds: whether compression is enabled on its + /// materialization, and which of the three policies exist for it, by job id. These are the fields the + /// sibling investigation (#3581) assembled by hand — twenty aggregates, all with compression disabled, + /// holding 57% of a production store — and they are STATE rather than series, which is why they are + /// not persisted (the aggregate row's paragraph on ). + /// + /// timescaledb_information.jobs reports a policy on an aggregate under the aggregate's + /// USER-FACING view name (COALESCE(ca.user_view_schema, ht.schema_name) in the view's own + /// definition), which is what the three correlated lookups join on. The compression predicate carries + /// the 2.18+ columnstore rebrand the rest of the codebase hedges on. hypertable_name in + /// the aggregates view is the aggregate's SOURCE; for a hierarchical aggregate that is another + /// aggregate's materialization under its internal name, so the source is resolved back to that + /// parent's view name where one exists. Readable by the least-privilege mcp role: the + /// information views are granted to PUBLIC (measured on 2.28.1 with a bare LOGIN role). No parameters. + /// + public const string ContinuousAggregateStateSql = @" +SELECT + ca.view_name, + ca.compression_enabled, + ca.materialized_only, + coalesce(parent.view_name, ca.hypertable_name) AS source_name, + (SELECT min(j.job_id) FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_refresh_continuous_aggregate' + AND j.hypertable_schema = ca.view_schema AND j.hypertable_name = ca.view_name) AS refresh_job_id, + (SELECT min(j.job_id) FROM timescaledb_information.jobs j + WHERE (j.proc_name LIKE '%compression%' OR j.proc_name LIKE '%columnstore%') + AND j.hypertable_schema = ca.view_schema AND j.hypertable_name = ca.view_name) AS compression_job_id, + (SELECT min(j.job_id) FROM timescaledb_information.jobs j + WHERE j.proc_name = 'policy_retention' + AND j.hypertable_schema = ca.view_schema AND j.hypertable_name = ca.view_name) AS retention_job_id +FROM timescaledb_information.continuous_aggregates ca +LEFT JOIN timescaledb_information.continuous_aggregates parent + ON parent.materialization_hypertable_schema = ca.hypertable_schema + AND parent.materialization_hypertable_name = ca.hypertable_name"; + + /// One aggregate's live catalog state. Job ids null where no such policy exists. + public sealed record ContinuousAggregateState( + string ViewName, + bool CompressionEnabled, + bool MaterializedOnly, + string? SourceName, + int? RefreshJobId, + int? CompressionJobId, + int? RetentionJobId); + + /// + /// Reads . Failure-isolated to NULL — not an empty list, which + /// would read as "no aggregates" and drop the flags from every aggregate row without a word. Not + /// attempted where the GUC probe said TimescaleDB is not loaded for this database + /// (): the view does not exist there and there are + /// no aggregate rows to decorate. No logger, for the reason gives. + /// + public static async Task?> GetContinuousAggregateStatesAsync( + NpgsqlDataSource postgres, + JobExecutionLoggingReading logging, + CancellationToken cancellationToken = default) + { + if (logging is null) + { + throw new ArgumentNullException(nameof(logging)); + } + + if (logging.Status == JobExecutionLoggingStatus.NotRegistered) + { + return Array.Empty(); + } + + try + { + var states = new List(); + await using var command = postgres.CreateCommand(ContinuousAggregateStateSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + states.Add(new ContinuousAggregateState( + reader.GetString(0), + reader.GetBoolean(1), + reader.GetBoolean(2), + reader.IsDBNull(3) ? null : reader.GetString(3), + reader.IsDBNull(4) ? null : reader.GetInt32(4), + reader.IsDBNull(5) ? null : reader.GetInt32(5), + reader.IsDBNull(6) ? null : reader.GetInt32(6))); + } + + return states; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return null; + } + } + + /// How many of the un-enumerated relations the tool names, largest first. + public const int LargestUnenumeratedLimit = 10; + + /// + /// The biggest relations inside the other row, by name (#3582) — read LIVE at tool time over + /// the SAME census predicate the sweep sums with (the fragments on ), so + /// the list and the number it explains cannot disagree about what counts. Without this the other + /// row is a figure nobody can act on; with it, a growth investigation reads the name of the table that + /// grew instead of running the pg_class census by hand — which is the expedition this whole + /// tool exists to replace. Named schema.relation, the form the table rows use, so a + /// relation the product later names by that kind keeps its name. TimescaleDB variant; readable by the + /// mcp role (pg_total_relation_size needs no privilege on the relation, and the + /// TimescaleDB catalogs are granted to PUBLIC — both measured). $1 the limit. + /// + public const string LargestUnenumeratedSql = $@" +SELECT + n.nspname || '.' || c.relname AS relation, + c.relkind::text, + pg_total_relation_size(c.oid) AS total_bytes +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE {StoreSelfMetrics.CensusRelationPredicateSql} +AND NOT {StoreSelfMetrics.SystemSchemaPredicateSql} +AND NOT {StoreSelfMetrics.NamedRelationPredicateSql} +AND {StoreSelfMetrics.TimescaleInventoriedPredicateSql} +ORDER BY pg_total_relation_size(c.oid) DESC, 1 +LIMIT $1"; + + /// The plain-PostgreSQL variant of : the same census minus + /// the TimescaleDB catalogs it cannot name. On such a store the collector tables lead this list, which + /// is the honest answer (see ). $1 the limit. + public const string LargestUnenumeratedPlainSql = $@" +SELECT + n.nspname || '.' || c.relname AS relation, + c.relkind::text, + pg_total_relation_size(c.oid) AS total_bytes +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE {StoreSelfMetrics.CensusRelationPredicateSql} +AND NOT {StoreSelfMetrics.SystemSchemaPredicateSql} +AND NOT {StoreSelfMetrics.NamedRelationPredicateSql} +ORDER BY pg_total_relation_size(c.oid) DESC, 1 +LIMIT $1"; + + /// One un-enumerated relation, sized live. + public sealed record UnenumeratedRelation(string Relation, string RelKind, long TotalBytes); + + /// + /// Reads the live top-N of un-enumerated relations, choosing the variant by the same signal the other + /// TimescaleDB-only reads use ( means the + /// TimescaleDB catalogs do not exist for this database). Failure-isolated to NULL, not an empty list, + /// for the reason gives. + /// + public static async Task?> GetLargestUnenumeratedAsync( + NpgsqlDataSource postgres, + JobExecutionLoggingReading logging, + CancellationToken cancellationToken = default) + { + if (logging is null) + { + throw new ArgumentNullException(nameof(logging)); + } + + try + { + var rows = new List(); + await using var command = postgres.CreateCommand( + logging.Status == JobExecutionLoggingStatus.NotRegistered ? LargestUnenumeratedPlainSql : LargestUnenumeratedSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + command.Parameters.AddWithValue(LargestUnenumeratedLimit); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new UnenumeratedRelation(reader.GetString(0), reader.GetString(1), reader.GetInt64(2))); + } + + return rows; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + return null; + } + } + /// One object's newest self-metrics row. The four job fields (#2136, V56) are non-null only - /// on background_job rows — every other kind leaves them NULL, as the sweep writes them. + /// on background_job rows and, since #3574, on the job_history row under the column + /// mapping the class summary states — every other kind leaves them NULL, + /// as the sweep writes them. public sealed record StoreMetricRow( string ObjectKind, string ObjectName, diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSystemHealthReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSystemHealthReader.cs index 07ac30db9..3bce88bba 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSystemHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingSystemHealthReader.cs @@ -93,35 +93,78 @@ public static async Task> ReadEventXmlAsync( } /// - /// Whether this server has EVER recorded a system_health event of one type, ignoring any window. - /// Lets an empty parse-on-read result say WHICH kind of nothing it found. Zero significant rows - /// is true both of a healthy window and of a server whose system_health events were never collected, - /// and the two want opposite responses -- widen the window, versus go find out why nothing is being - /// captured. Probes v_system_health_events, the SAME source - /// reads, so it cannot report a server as captured for rows - /// the read itself can never see. Scoped to the event_type because that is the granularity the caller - /// asked about: a server capturing sp_server_diagnostics but no wait_info has not been sampled for - /// waits, whatever its other categories hold. LIMIT 1, so it stops at the first row. - /// $1 server_id, $2 event_type. + /// The newest collection_time at which the system_health collector stored ANY event for this + /// server — the source witness every one of the nine parse-on-read tools publishes (#3541 A12, contract + /// rule 5: zero is a measurement). + /// Eight of the nine tools answered a dead system_health session, or a collector that had + /// never run, with the same empty a healthy quiet window earns, and "no severe errors" from a + /// server nothing was ever read from is a clean bill of health nobody issued. The witness is the + /// events view itself, NOT collection_log: the log records a SUCCESS for a run that read a dead + /// session and stored nothing, which is exactly the shape being mis-reported, whereas a stored event is + /// proof the session was alive and the collector reached it. Windowless and type-less on purpose — it + /// answers "has this server's ring buffer ever been read into the store", which is the question a + /// category with no rows in the window needs answered first; the type-scoped question is + /// . Reads the SAME view the tools read, for the #2484 reason: a + /// probe on another relation could report a source as observed for rows the read itself can never + /// see. + /// Cheap by shape: MAX(collection_time) under server_id = $1 is a backward walk of the + /// (server_id, collection_time) index that stops at the first row, so it rides on the data path + /// of every call and not only on the empty one. Anchored by construction — it names no clock; it is a + /// fact about the store, and a caller anchored in the past receives the store's newest capture, + /// which may be later than its window and is labelled as the collector's, not the window's. + /// $1 server_id. /// - public const string HasAnyEventOfTypeSql = """ - SELECT 1 + public const string LastCaptureSql = """ + SELECT MAX(collection_time) + FROM v_system_health_events + WHERE server_id = $1 + AND event_xml IS NOT NULL + """; + + /// + /// The newest collection_time at which an event of ONE type was stored for this server — the + /// type-scoped half of the witness, run only when a window came back with no events of that type. + /// Separates "this category has fired before, the window is quiet" (widen) from "this category + /// has never fired here while the session IS being read" — which for a rare category (a memory-node + /// OOM, a severe error) is the healthy measurement, not a blind spot. Same view as the read, same + /// event_xml IS NOT NULL guard, same backward index walk with a type filter — it stops at the + /// first match for a type that exists and walks the server's rows for one that never did, the cost the + /// #2484 HasAnyEventOfTypeSql probe this replaces already paid on the same path (that probe + /// answered only yes/no; this one also says WHEN, which is what the message needs). + /// $1 server_id, $2 event_type. + /// + public const string LastCaptureOfTypeSql = """ + SELECT MAX(collection_time) FROM v_system_health_events WHERE server_id = $1 AND event_type = $2 AND event_xml IS NOT NULL - LIMIT 1 """; - /// Runs . - public static async Task HasAnyEventOfTypeAsync( + /// Runs : null when no system_health event of any type has ever + /// been stored for the server. + public static Task GetLastCaptureAsync( + NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) + => ReadNullableTimestampAsync(postgres, LastCaptureSql, serverId, eventType: null, cancellationToken); + + /// Runs : null when no event of + /// has ever been stored for the server. + public static Task GetLastCaptureOfTypeAsync( NpgsqlDataSource postgres, int serverId, string eventType, CancellationToken cancellationToken = default) + => ReadNullableTimestampAsync(postgres, LastCaptureOfTypeSql, serverId, eventType, cancellationToken); + + private static async Task ReadNullableTimestampAsync( + NpgsqlDataSource postgres, string sql, int serverId, string? eventType, CancellationToken cancellationToken) { - await using var command = postgres.CreateCommand(HasAnyEventOfTypeSql); + await using var command = postgres.CreateCommand(sql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; DarlingMcpReadParameters.AddInt(command, serverId); - DarlingMcpReadParameters.AddText(command, eventType); - return await command.ExecuteScalarAsync(cancellationToken) is not null; + if (eventType is not null) + DarlingMcpReadParameters.AddText(command, eventType); + /* MAX over zero rows is one row holding SQL NULL, which Npgsql surfaces as DBNull — the aggregate + never returns no rows, so the null check is on the value rather than on the row. */ + var value = await command.ExecuteScalarAsync(cancellationToken); + return value is DateTime stamp ? stamp : null; } /// Loads the server's latest database_id → database_name map for Severe Errors DB resolution. diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs index 67f163a01..0d9986840 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingTrendReader.cs @@ -30,10 +30,11 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// get_perfmon_trend / get_file_io_trend / get_query_trend / get_query_duration_trend), the store-faithful /// shape Darling's collector-mirror schema serves — the same rule the merged /// follows (Lite's get_query_duration_trend has no Dashboard twin; the other four names + params match the -/// Dashboard, the shape follows Lite where the SKUs diverge). Every SQL string is byte-identical to the -/// viewer's proven Postgres read for that chart (the viewer already ported Lite's DuckDB SQL), so Darling -/// serves one consistent product. Each SQL string is a public const so Darling.Tests can pin the dialect + -/// columns without a live Postgres. +/// Dashboard, the shape follows Lite where the SKUs diverge). Every RAW-tier SQL string is byte-identical to +/// the viewer's proven Postgres read for that chart (the viewer already ported Lite's DuckDB SQL), so Darling +/// serves one consistent product; the hourly-tier twins the retention-routed reads fall to (#2353, #3541 A2) +/// have no viewer counterpart yet and are documented beside their raw originals. Each SQL string is a public +/// const so Darling.Tests can pin the dialect + columns without a live Postgres. /// /// internal static class DarlingTrendReader @@ -41,12 +42,21 @@ internal static class DarlingTrendReader /* ─────────────────────────── result records ─────────────────────────── */ /// One memory-trend point: the four MB metrics per collection (Lite's MemoryTrendPoint - /// minus its always-0 TotalGrantedMb overlay field, which the tool carries as a Lite parity - /// placeholder — see ). + /// minus its TotalGrantedMb overlay field, which the memory_stats source never fills — the tool + /// joins it per point from the grants series (), null with a note + /// naming get_memory_grants where no snapshot aligns (#3529, #3548) — see + /// ). public sealed record MemoryTrendPoint( DateTime CollectionTime, double TotalServerMemoryMb, double TargetServerMemoryMb, double BufferPoolMb, double PlanCacheMb); + /// One memory-grant-trend point: total granted workspace memory summed across every resource + /// pool at one grants collection (#3548) — the series + /// joins onto the memory trend, and the same series the viewer's Memory Overview overlay plots. Its + /// collection_times are the grants collector's OWN stamps, seconds apart from the memory series' even + /// in the same cycle, which is why the join is nearest-match rather than equality. + public sealed record MemoryGrantTrendPoint(DateTime CollectionTime, double TotalGrantedMb); + /// One perfmon-trend point for a single counter: the counter value, the per-interval delta, /// and the wall-clock seconds that delta covers, all summed across the counter's instances at that /// collection (Lite's PerfmonTrendPoint, plus the interval Lite does not carry). @@ -74,7 +84,15 @@ public sealed record FileIoLatencyTrendPoint( /// break; new readers should take ExecutionsPerSecond. /// public sealed record QueryDurationTrendPoint( - DateTime CollectionTime, double Value, long ExecutionCount, double ExecutionsPerSecond); + DateTime CollectionTime, double? Value, long? ExecutionCount, double? ExecutionsPerSecond) + { + /// + /// Whether this point carries a rate at all (#3541 A12). False for the window's first differenced + /// collection — no previous collection to difference against — and for a collection landing in the + /// same second as its predecessor; both have no denominator, and neither is 0. + /// + public bool HasRate => Value.HasValue; + } /// One point of a single query's per-collection history (Lite's QueryStatsHistoryRow, /// the columns get_query_trend surfaces): the interval deltas + DOP spread + the plan hash. Time metrics @@ -176,6 +194,42 @@ public static async Task> GetMemoryTrendAsync( return items; } + /// + /// The memory-grant trend — the viewer's MemoryGrantTrendSql (Lite's + /// GetMemoryGrantTrendAsync): total granted MB across all pools per grants collection over the + /// window, for the join get_memory_trend makes onto the memory series (#3548). $1 server_id, $2/$3 + /// window (naive UTC). + /// + public const string MemoryGrantTrendSql = """ + SELECT + collection_time, + CAST(SUM(granted_memory_mb) AS double precision) AS total_granted_mb + FROM v_memory_grant_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND collection_time <= $3 + GROUP BY collection_time + ORDER BY collection_time + """; + + public static async Task> GetMemoryGrantTrendAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) + { + var items = new List(); + await using var command = postgres.CreateCommand(MemoryGrantTrendSql); + command.CommandTimeout = McpCommandDeadlines.ReadSeconds; + DarlingMcpReadParameters.AddWindow(command, serverId, startUtc, endUtc); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + items.Add(new MemoryGrantTrendPoint( + reader.GetDateTime(0), + reader.IsDBNull(1) ? 0 : reader.GetDouble(1))); + } + + return items; + } + /* ─────────────────────────── perfmon trend ─────────────────────────── */ /// @@ -270,7 +324,9 @@ public static async Task> GetDistinctPerfmonCountersAsync( /// then per-collection average read/write latency (stall-ms / op, delta-stall sums CAST to double /// precision before division) is computed per (collection, database, file). The tool projects /// database_name + the two latencies (file_name rides the grouping but is not surfaced, mirroring Lite's - /// get_file_io_trend field set). $1 server_id, $2/$3 window (naive UTC). + /// get_file_io_trend field set). Rows whose stored sample_interval_seconds is 0 — no delta + /// knowable, a restart — are dropped rather than reported as 0.00 ms (#3540). $1 server_id, $2/$3 + /// window (naive UTC). /// public const string FileIoLatencyTrendSql = """ WITH top_files AS ( @@ -298,6 +354,11 @@ FROM v_file_io_stats f WHERE f.server_id = $1 AND f.collection_time >= $2 AND f.collection_time <= $3 + /* #3540: a stored interval of 0 is the calculator's "no delta knowable" marker (first sighting, + counter reset, a gap past the policy) — the row is dropped so the point is ABSENT rather than the + confident "0.00 ms" a restart used to render. IS DISTINCT FROM 0 keeps pre-V127 rows (NULL: interval + never recorded), which carry on reading exactly as they always did. */ + AND f.sample_interval_seconds IS DISTINCT FROM 0 GROUP BY f.collection_time, f.database_name, f.file_name ORDER BY f.collection_time, f.database_name, f.file_name """; @@ -329,11 +390,24 @@ public static async Task> GetFileIoLatencyTrendAsy /// GetQueryDurationTrendAsync): per collection, the summed delta_elapsed_time (→ ms) and /// delta_execution_count divided by the seconds since the previous collection (the truncate-then- /// diff LAG epoch idiom proven value-identical DuckDB↔Postgres) for an elapsed-ms/sec + executions/sec - /// rate. The first row's LAG is NULL → interval NULL → the CASE yields 0 (Lite's behaviour). Reads the - /// base query_stats table because it projects no text — a read that wanted query_text or - /// query_plan_xml would have to go through v_query_stats to resolve the #1767 payload - /// dimensions. Summed bigints come back as numeric, so the reads Convert tolerantly. $1 server_id, - /// $2/$3 window (naive UTC). + /// rate. Reads the base query_stats table because it projects no text — a read that wanted + /// query_text or query_plan_xml would have to go through v_query_stats to resolve the + /// #1767 payload dimensions. Summed bigints come back as numeric, so the reads Convert tolerantly. + /// $1 server_id, $2/$3 window (naive UTC). + /// + /// The first collection in the window has no rate (#3541 A12, #3540 A8). Its LAG is NULL + /// — there is no previous collection inside the window to difference against — so its rate is + /// unknowable, and the shape this replaced (CASE ... ELSE 0 END, Lite's original behaviour) + /// published that unknowable as a measured 0.0: every duration series began with a fabricated quiet + /// instant, which an agent charting the window read as "idle, then busy" and which dragged every + /// first-bucket average toward zero. Contract rule 5 — zero is a measurement — so the CASE has no ELSE + /// and the rate columns are NULL for that row (and for the degenerate two-collections-in-one-second + /// case, whose denominator is 0 and whose rate is equally undefined). The row is KEPT rather than + /// filtered, deliberately: the collection happened, effective_start is truthfully its instant, + /// and a window holding exactly one collection is "one collection, no rate yet" rather than an empty + /// series the empty ladder would mis-describe as a quiet window. The reader carries the nulls through + /// () and the tool publishes them with the reason. The hourly + /// twin below has no such row: its denominator is the bucket width, known for every bucket. /// public const string QueryDurationTrendSql = """ WITH raw AS @@ -351,16 +425,186 @@ GROUP BY collection_time ) SELECT collection_time, - CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds ELSE 0 END AS elapsed_ms_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS executions_per_second + CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds END AS elapsed_ms_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds END AS executions_per_second FROM raw ORDER BY collection_time """; - /// Runs . - public static Task> GetQueryDurationTrendAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) - => ReadDurationTrendAsync(QueryDurationTrendSql, postgres, serverId, startUtc, endUtc, cancellationToken); + /// + /// The seconds in one hourly-rollup bucket, as the SQL literal the hourly-tier duration trends divide by. + /// A string because a constant interpolated string admits only string constants; pinned equal to + /// by DarlingMcpTrendToolsTests so the two cannot drift. + /// + public const string HourlyBucketSecondsSql = "3600.0"; + + /// + /// The hourly-tier twin of (#3541 A2): the same two per-second rates, + /// read from the query_stats_hourly continuous aggregate for windows whose oldest point the raw + /// tier no longer holds. The rollup carries elapsed_time_sum and execution_count_sum per + /// (server, database, query_hash, sql_handle, hour); summing those across the identity columns per bucket + /// is exactly what the raw read's GROUP BY collection_time does one grain finer, so the two tiers + /// answer the same question at different resolution. bucket is projected as collection_time + /// so the point shape does not change underneath a caller that got a raw answer last time. + /// + /// The denominator is the bucket width, not a LAG. The raw read has to recompute its interval + /// from the gap to the previous collection because a per-sweep row carries no shared interval (the + /// measurement lane's A11a); an hour bucket's width is KNOWN, so this read divides by + /// and every bucket has a real denominator — the LAG idiom's first + /// collection, whose interval is NULL and whose rate is therefore unknowable (published as NULL by the raw + /// read since #3541 A12; as a fabricated 0 before it), does not exist here. The + /// trade is stated in the payload rather than hidden: an hour the collector covered only partly (a + /// service restart mid-hour, a gap past the delta policy) reads LOW, never high, because its summed work is + /// still spread over the full 3,600 seconds. + /// + /// Materialized-only, like every rollup here (#1759): the current hour is never materialized + /// ( is the end_offset) and the previous one + /// lands on the next refresh, so this series trails the clock by up to two hours. $1 server_id, $2/$3 + /// window (naive UTC). + /// + public const string QueryDurationTrendHourlySql = $""" + SELECT + bucket AS collection_time, + SUM(elapsed_time_sum) / 1000.0 / {HourlyBucketSecondsSql} AS elapsed_ms_per_second, + CAST(SUM(execution_count_sum) AS DOUBLE PRECISION) / {HourlyBucketSecondsSql} AS executions_per_second + FROM {TimescaleSupport.QueryStatsHourlyView} + WHERE server_id = $1 + AND bucket >= $2 + AND bucket <= $3 + GROUP BY bucket + ORDER BY bucket + """; + + /// + /// The hourly-tier twin of (#3541 A2), over + /// procedure_stats_hourly. Same shape and same bucket-width denominator as + /// ; see there for why. $1 server_id, $2/$3 window (naive UTC). + /// + public const string ProcedureDurationTrendHourlySql = $""" + SELECT + bucket AS collection_time, + SUM(elapsed_time_sum) / 1000.0 / {HourlyBucketSecondsSql} AS elapsed_ms_per_second, + CAST(SUM(execution_count_sum) AS DOUBLE PRECISION) / {HourlyBucketSecondsSql} AS executions_per_second + FROM {TimescaleSupport.ProcedureStatsHourlyView} + WHERE server_id = $1 + AND bucket >= $2 + AND bucket <= $3 + GROUP BY bucket + ORDER BY bucket + """; + + /// + /// Which tier one unkeyed duration trend read serves from, and the evidence the choice rests on (#3541 A2) + /// — the get_query_trend routing (, #2353) generalized to the reads that + /// have no query key. + /// + /// and are the pair this trend reads at each tier; + /// is the one picked, so a payload or an empty message can say + /// WHAT WAS READ rather than what was asked for. and + /// are kept on the route so the empty branch can say why the tier it read holds nothing — a rollup that + /// has materialized nothing and a rollup whose floor sits above the window's start are different facts + /// with different remedies, and both differ from a quiet server. + /// + /// is whether THIS grain's raw table can have had rows dropped, + /// and it is scoped to the grain rather than to the store on purpose. Retention is armed per raw table by + /// the #1680 gate, which arms a table's purge only once that table's OWN rollup covers everything the + /// table holds — so a grain whose rollup does not exist (plain PostgreSQL, a failed availability probe, or + /// #1664's failure-isolated partial build where one grain's aggregate failed its ensure sweep while the + /// others built) has its purge held paused and its raw rows intact, whatever the other grains' rollups + /// are doing. A store-wide "any rollup exists" test would have told the caller of the un-rolled grain that + /// its rows were dropped and that widening cannot help — the exact false-and-harmful narrative this route + /// exists to remove, wearing the fix's own clothes. Where it is false, raw holds the complete answer and + /// "quiet, widen the window" is honest (#1665); where it is true, the empty branch has to consider that + /// the rows were DROPPED, not absent. + /// + /// It is rollup EXISTENCE, deliberately, not the gate's armed state (RetentionHoldReading.Armed): + /// a rollup can exist while its raw purge is still held pending backfill, so this is an UPPER BOUND on + /// "can have been dropped" — never a claim that rows were. That is the right bound for its one consumer: + /// wherever raw's oldest row is MEASURED (), the measurement decides and this flag is + /// not consulted; it backstops only the unmeasured case, and it errs toward the horizon message (which + /// still names the measured facts it has and the remedy) rather than toward "quiet, widen" — the false + /// direction. Reading the gate's job state per call would cost a catalog round trip on every empty answer + /// to refine a fallback the measurement already makes rare. + /// + /// is the wall clock the age decision was measured against. It rides on + /// the route so the tool that consumes it never names the clock itself: an as_of-anchored tool's + /// only "now" is the anchor it resolved (AsOfWindowAnchorTests pins that as an absolute), and retention's + /// clock — which is NOT the anchor, see — is this reader's concern. + /// + public sealed record DurationTrendRoute( + RetentionTier Tier, string RawTable, string HourlyView, bool HourlyAvailable, TierCoverage Coverage, + bool RawRetentionApplies, DateTime ResolvedAtUtc) + { + /// The payload's source word: raw or hourly, get_query_trend's vocabulary. + public string Source => Tier == RetentionTier.Raw ? "raw" : "hourly"; + + /// The relation the read actually walks. + public string Relation => Tier == RetentionTier.Raw ? RawTable : HourlyView; + + /// + /// Whether the raw table is measured to hold rows at or before . Null + /// when raw's oldest row was not measured (no rollups, probe failed, or the table is empty) — the + /// caller falls back to the retention span, which is the proxy #1759 warns is wrong in the dangerous + /// direction on a held-purge store, so it is used only where nothing was measured. + /// + public bool? RawReaches(DateTime windowStartUtc) => + Coverage.RawOldestUtc is DateTime oldest ? oldest <= windowStartUtc : null; + } + + /// + /// One routed duration-trend answer: the points, the route that produced them, and what the points + /// actually cover — the shape for the unkeyed trends, so all four tiered + /// reads describe themselves with the same three words (source, effective_start, + /// truncated). + /// + public sealed record DurationTrendResult( + List Points, DurationTrendRoute Route, DateTime EffectiveStartUtc, bool Truncated); + + /// + /// Resolves the route for the query-stats duration trend: raw query_stats or + /// query_stats_hourly, by . is the WALL CLOCK, + /// never the window's end — see — and defaults to the real clock, the same + /// way takes it: the tool passes only its anchored window, a test + /// passes a fixed instant to pin the boundary. + /// + public static DurationTrendRoute ResolveQueryDurationTrendRoute( + DateTime startUtc, RollupAvailability rollups, RollupCoverage coverage, DateTime? nowUtc = null) + => ResolveDurationTrendRoute( + "query_stats", TimescaleSupport.QueryStatsHourlyView, TimescaleSupport.QueryStatsDailyView, + rollups.QueryGrainHourly, startUtc, nowUtc ?? DateTime.UtcNow, rollups, coverage); + + /// The procedure-stats twin of . + public static DurationTrendRoute ResolveProcedureDurationTrendRoute( + DateTime startUtc, RollupAvailability rollups, RollupCoverage coverage, DateTime? nowUtc = null) + => ResolveDurationTrendRoute( + "procedure_stats", TimescaleSupport.ProcedureStatsHourlyView, TimescaleSupport.ProcedureStatsDailyView, + rollups.ProcedureGrainHourly, startUtc, nowUtc ?? DateTime.UtcNow, rollups, coverage); + + private static DurationTrendRoute ResolveDurationTrendRoute( + string rawTable, string hourlyView, string dailyView, bool hourlyAvailable, + DateTime startUtc, DateTime nowUtc, RollupAvailability rollups, RollupCoverage coverage) + { + ArgumentNullException.ThrowIfNull(coverage); + + var tierCoverage = coverage.For(hourlyView, dailyView); + return new DurationTrendRoute( + ResolveTier(startUtc, nowUtc, hourlyAvailable, tierCoverage), + rawTable, hourlyView, hourlyAvailable, tierCoverage, + /* Grain-scoped, not rollups != None — see the record's remarks: the arming gate is per table. */ + RawRetentionApplies: hourlyAvailable, + ResolvedAtUtc: nowUtc); + } + + /// + /// Runs the query-stats duration trend down (#3541 A2): the raw read for a + /// window raw can serve, the hourly twin otherwise. Coverage is described by + /// so the payload can say what it served. + /// + public static Task GetQueryDurationTrendAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, DurationTrendRoute route, + CancellationToken cancellationToken = default) + => ReadRoutedDurationTrendAsync( + QueryDurationTrendSql, QueryDurationTrendHourlySql, postgres, serverId, startUtc, endUtc, route, cancellationToken); /* --------------------- procedure + Query Store duration trends (#2484) --------------------- */ @@ -372,6 +616,14 @@ public static Task> GetQueryDurationTrendAsync( /// shows up smeared across however many statements it runs. This charges the whole call to the /// procedure. When both are available, the pair answers "did ad-hoc SQL regress, or did a procedure?" - /// which one series alone never can. $1 server_id, $2/$3 window (naive UTC). + /// #3540 (V128): the interval is the collection's STORED one where the rows have it — MAX + /// over the collection's rows, because a plan first seen in an otherwise steady pass carries 0 beside + /// its siblings' real interval and contributes 0 to the sums; MAX is 0 only when EVERY row was + /// unknowable (a restart), and that 0 becomes NULL through NULLIF so the rates are NULL — an + /// UNRATED point the reader keeps rather than rendering 0.00 ms/sec (#3541 A12; see + /// for why the row stays). NULL (a pre-V128 collection) falls back to + /// the LAG this read always used, whose first row is likewise unrated, never a fabricated 0. No + /// ELSE 0. Verbatim from the viewer's copy apart from the database filter, as before. /// public const string ProcedureDurationTrendSql = """ WITH raw AS @@ -380,7 +632,10 @@ WITH raw AS collection_time, SUM(delta_elapsed_time) / 1000.0 AS total_elapsed_ms, SUM(delta_execution_count) AS total_executions, - extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) AS interval_seconds + CASE WHEN MAX(sample_interval_seconds) IS NULL + THEN extract(epoch FROM (date_trunc('second', collection_time) - date_trunc('second', LAG(collection_time) OVER (ORDER BY collection_time)))) + ELSE NULLIF(MAX(sample_interval_seconds), 0) + END AS interval_seconds FROM procedure_stats WHERE server_id = $1 AND collection_time >= $2 @@ -389,8 +644,8 @@ GROUP BY collection_time ) SELECT collection_time, - CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds ELSE 0 END AS elapsed_ms_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS executions_per_second + CASE WHEN interval_seconds > 0 THEN total_elapsed_ms / interval_seconds END AS elapsed_ms_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds END AS executions_per_second FROM raw ORDER BY collection_time """; @@ -406,7 +661,9 @@ ORDER BY collection_time /// interval start exists for them and none can be reconstructed. The arms split on /// interval_start_time_utc IS NULL, so they partition the rows with no overlap and no gap. /// Rewriting either arm here would make the browser and the desktop viewer disagree about the same - /// hour. $1 server_id, $2/$3 window (naive UTC). + /// hour. The first placed interval in the window carries NULL rates, not 0 — see + /// (#3541 A12); the rollup route's builder applies the same rule to + /// its first bucket. $1 server_id, $2/$3 window (naive UTC). /// #2736: this is now the FALLBACK, not the read. The rank-over-raw below costs the whole /// slab regardless of the window, which exceeds the mcp role's statement_timeout on a large store — /// so on stores with a materialized query_store_stats_corrected_hourly the tool routes through @@ -475,8 +732,8 @@ GROUP BY point_time ) SELECT point_time AS collection_time, - CASE WHEN interval_seconds > 0 THEN total_duration_ms / interval_seconds ELSE 0 END AS duration_ms_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS executions_per_second + CASE WHEN interval_seconds > 0 THEN total_duration_ms / interval_seconds END AS duration_ms_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds END AS executions_per_second FROM raw ORDER BY point_time """; @@ -494,10 +751,30 @@ ORDER BY point_time public static readonly string QueryStoreDurationTrendRollupSql = QueryStoreTrendRouting.BuildRollupTrendSql(withDatabaseFilter: false); - /// Runs . - public static Task> GetProcedureDurationTrendAsync( - NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) - => ReadDurationTrendAsync(ProcedureDurationTrendSql, postgres, serverId, startUtc, endUtc, cancellationToken); + /// Runs the procedure-stats duration trend down (#3541 A2) — the + /// procedure twin of . + public static Task GetProcedureDurationTrendAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, DurationTrendRoute route, + CancellationToken cancellationToken = default) + => ReadRoutedDurationTrendAsync( + ProcedureDurationTrendSql, ProcedureDurationTrendHourlySql, postgres, serverId, startUtc, endUtc, route, cancellationToken); + + /// + /// The shared body of the two routed reads: pick the tier's SQL, read the three-column point shape, and + /// describe what came back. One method so the query and procedure trends cannot drift in how they + /// route, read, or describe coverage. + /// + private static async Task ReadRoutedDurationTrendAsync( + string rawSql, string hourlySql, NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + DurationTrendRoute route, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(route); + + var points = await ReadDurationTrendAsync( + route.Tier == RetentionTier.Raw ? rawSql : hourlySql, postgres, serverId, startUtc, endUtc, cancellationToken); + var (effectiveStart, truncated) = DescribeCoverage(points.Count > 0 ? points[0].CollectionTime : null, startUtc); + return new DurationTrendResult(points, route, effectiveStart, truncated); + } /// /// Runs — the raw-only route, kept for callers that have not @@ -554,11 +831,14 @@ private static async Task> ReadDurationPointsAsync await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - var executionsPerSecond = reader.IsDBNull(2) ? 0 : Convert.ToDouble(reader.GetValue(2)); + /* NULL stays NULL (#3541 A12): no rate for the window's first collection, or for a collection whose + stored interval was unknowable (a restart pass, #3540 V128) — either way the point is kept as + UNRATED rather than dropped or coerced to the fabricated quiet the SQL stopped producing. */ + var executionsPerSecond = reader.IsDBNull(2) ? (double?)null : Convert.ToDouble(reader.GetValue(2)); items.Add(new QueryDurationTrendPoint( reader.GetDateTime(0), - reader.IsDBNull(1) ? 0 : Convert.ToDouble(reader.GetValue(1)), - (long)executionsPerSecond, + reader.IsDBNull(1) ? null : Convert.ToDouble(reader.GetValue(1)), + executionsPerSecond is { } eps ? (long)eps : null, executionsPerSecond)); } @@ -625,6 +905,75 @@ ORDER BY collection_time public static bool ShouldUseRawTier(DateTime startUtc, DateTime nowUtc) => startUtc >= nowUtc - TimescaleSupport.RawRetentionSpan + RawTierMargin; + /// + /// The one tier decision every tiered trend read in this file makes (#3541 A2): get_query_trend's age rule + /// (), degraded to what the store HAS and to what it has MATERIALIZED, in + /// that order — the same three-rung ladder runs for the viewer's tabs, + /// built from the same Storage primitives, with this file's raw margin at the bottom rung instead of the + /// router's one-day one (that margin is #2353's deliberate trade of resolution for purge-independence, and + /// widening it to a day would push every 3-to-4-day window onto the hourly tier for nothing). + /// + /// Availability (#1664). A plain-PostgreSQL store has no continuous aggregates, and a + /// relation named in a statement is resolved at PARSE time — so an age-only rule sent a 168-hour window on + /// such a store to a view that does not exist and the tool answered 42P01. Raw is the right answer there + /// anyway: without the extension nothing ever drops raw, so it holds the complete window. + /// + /// Coverage (#1759). A rollup created WITH NO DATA serves only what a refresh or a + /// backfill has materialized, and its refresh reaches back one + /// from creation — so on a store that pre-existed its rollups and was never backfilled the hourly tier is + /// SHALLOWER than raw, whose purge the arming gate holds paused until the rollup covers it. Routing such a + /// window to the rollup by age alone returned EMPTY while raw held every row. The rule is comparative and + /// only ever moves DOWN on a positive measurement: raw wins only when it is measured to reach further back + /// than the rollup's floor (); unmeasured coverage (nulls) is + /// inert and the age + availability answer stands. Hourly is never abandoned merely because the window + /// starts below its floor — on a healthy store raw keeps four days against the rollup's ninety, and + /// dropping to raw there would return LESS. The part of the window the rollup has not reached is the + /// payload's job to disclose (effective_start, truncated), not routing's job to hide. + /// + /// Pure, for the same reason is: the tiering decision is the whole of + /// the fix, and DarlingQueryTrendTieringTests walks its table without a store. The daily tier is not on + /// this ladder because (seven days) + /// sits far inside ; a window the hourly tier cannot + /// reach by AGE cannot be asked for. + /// + public static RetentionTier ResolveTier(DateTime startUtc, DateTime nowUtc, bool hourlyAvailable, TierCoverage coverage) + { + if (ShouldUseRawTier(startUtc, nowUtc) || !hourlyAvailable) + { + return RetentionTier.Raw; + } + + if (TierCoverage.Covers(coverage.HourlyFloorUtc, startUtc)) + { + return RetentionTier.Hourly; + } + + return TierCoverage.ReachesFurtherBack(coverage.RawOldestUtc, coverage.HourlyFloorUtc) + ? RetentionTier.Raw + : RetentionTier.Hourly; + } + + /// + /// How far past the requested start the first served point may sit before the answer calls itself + /// truncated. Ninety minutes: an hourly bucket can begin up to an hour after a window start that + /// falls mid-hour (bucket >= $start excludes the bucket the start falls inside), and a raw series + /// legitimately opens a collection cadence or two late; anything past that means the tier did not hold the + /// window's head. Extracted from #2353's inline literal so the four tiered reads share one boundary. + /// + public static readonly TimeSpan TruncationSlack = TimeSpan.FromMinutes(90); + + /// + /// What a series actually covers, which is what the caller gets told (#2353): the first served point, or + /// the requested start when nothing came back — an empty result says nothing about coverage, so the + /// requested start stands rather than being narrowed to a window nobody can describe. Truncated is + /// true only on a NON-empty series whose head sits more than after the + /// requested start; an empty series reports its coverage through the empty branch's message instead. + /// + public static (DateTime EffectiveStartUtc, bool Truncated) DescribeCoverage(DateTime? firstPointUtc, DateTime startUtc) + => firstPointUtc is DateTime first + ? (first, first > startUtc + TruncationSlack) + : (startUtc, false); + /// /// Reads a query's history from the tier that can actually serve the window (#2353). /// @@ -644,25 +993,28 @@ public static bool ShouldUseRawTier(DateTime startUtc, DateTime nowUtc) => /// It is deliberately NOT defaulted to : a caller asking for a historical window /// would then have its start measured against its own end, which makes a two-hour window from ten days ago /// look recent and routes it to a tier that dropped those rows six days earlier. + /// + /// and are the store's measured shape + /// (#3541 A2 — see ); the defaults reproduce the age-only #2353 decision for a + /// caller that has not probed, which is what every pre-existing test of this read exercises. /// public static async Task GetQueryHistoryAsync( - NpgsqlDataSource postgres, int serverId, string databaseName, string queryHash, DateTime startUtc, DateTime endUtc, DateTime? nowUtc = null, CancellationToken cancellationToken = default) + NpgsqlDataSource postgres, int serverId, string databaseName, string queryHash, DateTime startUtc, DateTime endUtc, + DateTime? nowUtc = null, bool hourlyAvailable = true, TierCoverage coverage = default, CancellationToken cancellationToken = default) { - var useRaw = ShouldUseRawTier(startUtc, nowUtc ?? DateTime.UtcNow); + var tier = ResolveTier(startUtc, nowUtc ?? DateTime.UtcNow, hourlyAvailable, coverage); - var items = useRaw + var items = tier == RetentionTier.Raw ? await ReadQueryHistoryAsync(postgres, QueryHistorySql, serverId, databaseName, queryHash, startUtc, endUtc, cancellationToken) : await ReadQueryHistoryAsync(postgres, QueryHistoryHourlySql, serverId, databaseName, queryHash, startUtc, endUtc, cancellationToken); - /* What was actually covered, which is what the caller gets told. An empty result says nothing about - coverage, so the requested start stands rather than being narrowed to a window we cannot describe. */ - var effectiveStart = items.Count > 0 ? items[0].CollectionTime : startUtc; + var (effectiveStart, truncated) = DescribeCoverage(items.Count > 0 ? items[0].CollectionTime : null, startUtc); return new QueryHistoryResult( items, - useRaw ? "raw" : "hourly", + tier == RetentionTier.Raw ? "raw" : "hourly", effectiveStart, - Truncated: items.Count > 0 && effectiveStart > startUtc.AddMinutes(90)); + truncated); } private static async Task> ReadQueryHistoryAsync( @@ -788,7 +1140,34 @@ public static Task HasAnyQueryStoreStatAsync( NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) => HasAnySampleAsync(postgres, HasAnyQueryStoreStatSql, serverId, cancellationToken); - /// All five probes share one shape: a scalar that is null when no row qualifies. + /// + /// Whether this server has EVER been sampled as far as a ROUTED duration trend can tell (#3541 A2): the + /// raw probe, OR a row in the hourly rollup the route can reach. The raw probes above are the right + /// question for a raw-tier read, and the wrong one on a rolled table: a server disabled a week ago has no + /// raw rows left (retention dropped them) while its hourly rollup still holds weeks of history, and the + /// raw probe alone would answer "nothing has EVER been stored" — a false statement, not an incomplete one. + /// Probed only when the route says the rollup exists, because the view is parse-time-resolved; the view + /// name comes from , which only ever carries a + /// view constant. + /// + public static async Task HasAnySampleOnRouteAsync( + NpgsqlDataSource postgres, Task rawProbe, DurationTrendRoute route, int serverId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(rawProbe); + ArgumentNullException.ThrowIfNull(route); + + if (await rawProbe) + { + return true; + } + + return route.HourlyAvailable + && await HasAnySampleAsync( + postgres, $"SELECT 1 FROM {route.HourlyView} WHERE server_id = $1 LIMIT 1", serverId, cancellationToken); + } + + /// All the probes share one shape: a scalar that is null when no row qualifies. private static async Task HasAnySampleAsync( NpgsqlDataSource postgres, string sql, int serverId, CancellationToken cancellationToken) { diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs index 011336c6f..e0a45de0f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs @@ -507,15 +507,6 @@ precisely because the dashboard works perfectly. */ var loaded = DarlingWebTls.Load(network.Tls!, plan.Shape); var certificate = loaded.Leaf; - /* #3514: publish the served certificate's expiry to the worker's alert sweep BEFORE - the lifetime gate below, so an already-expired certificate the host is about to - refuse still reaches the operator as a Critical self-alert, not only a log line. - NotAfter is a LOCAL time (see the lifetime check below) — normalize to UTC. */ - _certState.Publish( - new DateTimeOffset(certificate.NotAfter.ToUniversalTime()), - certificate.Subject, - certificate.Thumbprint); - if (plan.Warning is not null) { _logger.LogWarning("Web dashboard TLS: {Warning}", plan.Warning); @@ -529,16 +520,32 @@ place the reason reaches an operator's log. LOCAL DateTimes, and while the implicit DateTime->DateTimeOffset conversion does carry the local offset and would compare correctly, it reads as a UTC value to everyone who follows. Convert where the trap is, not where it detonates. */ - var refusal = DarlingWebTls.LifetimeRefusal( - certificate.NotBefore.ToUniversalTime(), - certificate.NotAfter.ToUniversalTime(), - DateTimeOffset.UtcNow); - if (refusal is not null) + var notBeforeUtc = new DateTimeOffset(certificate.NotBefore.ToUniversalTime()); + var notAfterUtc = new DateTimeOffset(certificate.NotAfter.ToUniversalTime()); + var lifetime = DarlingWebTls.CheckLifetime(notBeforeUtc, notAfterUtc, DateTimeOffset.UtcNow); + + /* #3514: publish the served certificate's facts to the worker's alert sweep BEFORE + acting on the lifetime verdict, so a certificate the host is about to refuse still + reaches the operator as a Critical self-alert, not only a log line. An expired one + needs nothing but its NotAfter — the worker reads the lapse off the clock, as it + must for a certificate that lapses mid-run. A NOT-YET-VALID one (#3517) needs the + VERDICT carried: this host decided once, here, and stays loopback-only on that + decision until its next start, whatever the clock does afterwards. A worker left + to re-derive it from NotBefore would call the dashboard healthy the moment the + date passed — while it is still unreachable. */ + _certState.Publish( + notBeforeUtc, + notAfterUtc, + certificate.Subject, + certificate.Thumbprint, + refusedNotYetValid: lifetime.Status == DarlingWebTls.LifetimeStatus.NotYetValid); + + if (lifetime.Refusal is not null) { loaded.Dispose(); _logger.LogCritical( "Web dashboard TLS certificate cannot be used ({Refusal}) — refusing to expose; binding loopback-only.", - refusal); + lifetime.Refusal); networkMode = false; break; } diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/LatestSnapshot.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/LatestSnapshot.cs new file mode 100644 index 000000000..1b5bb9bc7 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/LatestSnapshot.cs @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; + +namespace PerformanceMonitor.Darling.Service.Mcp; + +/// +/// The rows of one LATEST-snapshot read together with the instant that snapshot was captured (#3541 A10). +/// +/// Why the stamp travels with the rows rather than on them. A latest read is +/// WHERE collection_time = (SELECT MAX(collection_time) ...): every row it returns shares ONE +/// stamp by construction, so the stamp is a property of the snapshot, not of a row. Putting it on each row +/// record would either widen positional records that tests and callers construct by hand, or default it — +/// and a defaulted DateTime on a row that was never stamped is exactly the shape this type exists to +/// make unrepresentable. The reader reads the stamp off the first row it materialises (the SELECT carries +/// the column) and the tool publishes it once, as captured_at. +/// +/// Why it is not a second read. A separate SELECT MAX(collection_time) can disagree with +/// the rows when a collection lands between the two statements — the rows would be one snapshot and the +/// stamp the next. Carrying the column on the row statement makes the two provably the same instant. +/// +/// is null exactly when is empty: there is no snapshot to +/// stamp. A tool must test (or ) before reading the stamp, which is +/// the same branch it already takes to return its unavailable status. +/// +internal sealed class LatestSnapshot +{ + /// The empty snapshot — no rows, no stamp. + public static readonly LatestSnapshot Empty = new(null, new List()); + + public LatestSnapshot(DateTime? capturedAt, List rows) + { + if (rows.Count > 0 && capturedAt is null) + { + throw new ArgumentException("a latest snapshot with rows must carry the instant it was captured", nameof(capturedAt)); + } + + CapturedAt = capturedAt; + Rows = rows; + } + + /// The snapshot's collection_time / capture_time (naive UTC, as stored) — the ONE + /// instant every row in was captured at. Null when there are no rows. + public DateTime? CapturedAt { get; } + + public List Rows { get; } + + public int Count => Rows.Count; + + public bool IsEmpty => Rows.Count == 0; +} + +/// +/// The one arithmetic every stamped latest read shares (#3541 A10). +/// +internal static class LatestSnapshotStamp +{ + /// + /// Whole seconds from a snapshot's stamp to the window's end — the anchor the caller asked for, never the + /// service clock (AsOfWindowAnchorTests: an anchored tool's only "now" is its as_of, and a tool + /// body that names DateTime.UtcNow fails the census). The store stamps naive UTC and the anchor is + /// Kind=Utc; the subtraction ignores Kind, which is correct here because both are UTC instants. + /// Non-negative by construction for a windowed read, whose rows are bounded by + /// collection_time <= window end; clamped at zero anyway so a sub-second precision difference + /// between a microsecond store stamp and a 100 ns anchor can never publish "-0". + /// + public static long AgeSeconds(DateTime capturedAt, DateTime windowEnd) => + Math.Max(0L, (long)Math.Round((windowEnd - capturedAt).TotalSeconds)); +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/WebTlsCertificateState.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/WebTlsCertificateState.cs index 3c524f7b4..b6aa4520c 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/WebTlsCertificateState.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/WebTlsCertificateState.cs @@ -26,6 +26,15 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// on to REFUSE the certificate for lifetime (an already-expired certificate at start), so the worker can /// still raise the Critical the operator needs rather than the fact being lost to a log line. /// +/// Why the snapshot carries the host's refusal DECISION and not only the dates (#3517). Expiry is +/// legitimately the worker's to derive: the certificate lapses while the process runs, the host made no +/// decision about it, and the date against the clock is the whole fact. Not-yet-valid is the opposite shape. +/// The host judged NotBefore against the clock ONCE, at load, refused, and bound loopback-only — and it +/// stays there until the next start no matter what the clock does next. A worker that re-derived that state +/// from NotBefore would declare the dashboard healthy the moment the clock crossed the date, and +/// resolve an alert about a dashboard that is still unreachable. So the host says what it decided +/// () and the worker repeats it until the host says otherwise. +/// /// Thread-safety: one writer (the web host's start path), one reader (the worker's sweep). State is a /// single immutable record reference swapped atomically, so the reader never sees a torn snapshot. Null until /// the host publishes — which it does only when it has a loaded certificate; loopback-only installs, an @@ -34,16 +43,27 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// public sealed class WebTlsCertificateState { - /// A coherent published snapshot of the loaded web-dashboard TLS certificate. - /// is normalized to UTC by the publisher (the X.509 NotAfter is a LOCAL time). - public sealed record Snapshot(DateTimeOffset NotAfterUtc, string Subject, string Thumbprint); + /// A coherent published snapshot of the loaded web-dashboard TLS certificate. + /// and are normalized to UTC by the publisher (the X.509 NotBefore / + /// NotAfter are LOCAL times). is the host's load-time verdict that + /// the certificate's window had not opened yet, so it refused to serve it and the LAN dashboard is + /// loopback-only (#3517) — a standing fact about THIS process's start, true until the host publishes + /// again or clears, not something to re-check against the clock. + public sealed record Snapshot( + DateTimeOffset NotBeforeUtc, + DateTimeOffset NotAfterUtc, + string Subject, + string Thumbprint, + bool RefusedNotYetValid); private volatile Snapshot? _current; /// Publishes the loaded certificate's facts (web host only, at start; also for a certificate the - /// host loaded and then refused for lifetime, so an expired-at-start certificate still surfaces). - public void Publish(DateTimeOffset notAfterUtc, string subject, string thumbprint) => - _current = new Snapshot(notAfterUtc, subject ?? string.Empty, thumbprint ?? string.Empty); + /// host loaded and then refused for lifetime, so an expired-at-start certificate still surfaces and a + /// not-yet-valid one is reported as the refusal it was, #3517). + public void Publish( + DateTimeOffset notBeforeUtc, DateTimeOffset notAfterUtc, string subject, string thumbprint, bool refusedNotYetValid) => + _current = new Snapshot(notBeforeUtc, notAfterUtc, subject ?? string.Empty, thumbprint ?? string.Empty, refusedNotYetValid); /// Clears the published snapshot back to "nothing to watch" (web host only) — called when the /// host STOPS serving TLS: a runtime disable of the dashboard, or a failed/degraded start. Without this diff --git a/Darling/PerformanceMonitor.Darling.Service/SelfAlertDeliveryStampStore.cs b/Darling/PerformanceMonitor.Darling.Service/SelfAlertDeliveryStampStore.cs new file mode 100644 index 000000000..99de27119 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/SelfAlertDeliveryStampStore.cs @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Npgsql; +using NpgsqlTypes; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// Where the two DOCUMENT-class self-alerts — the collector-cost digest (#3443) and the fleet-sweep daily +/// rollup (#3466) — record that a copy was DELIVERED, so their once-a-day gate can outlive the process +/// that fired them (#3580). +/// +/// The question this answers is "was one delivered today", not "did I send one today". Both +/// gates lived in process memory — one ConcurrentDictionary each, one fixed key — and a fresh +/// process has empty dictionaries, so the first tick after ANY start delivered both documents again +/// whatever the previous process had delivered an hour earlier. On the v3.8.0 install night three stores +/// restarted once each and the channel carried six re-announcements among ~23 overnight posts: a quarter +/// of the channel was the same two documents, twice. The same night showed the case any fix must keep: a +/// digest/rollup pair whose delivery had FAILED (a transport fault, not a suppression) was correctly +/// re-attempted after the restart and landed. A failed delivery is not a delivery. So the evaluator writes +/// a stamp here only when the deliverer reports a disposition other than +/// , reads it back before firing, +/// and skips while the stamp is inside the document's interval — restart or not. A failed delivery writes +/// nothing, so the next tick retries, restart or not. +/// +/// The store may throw; the evaluator isolates. The FleetSweepStore shape rather than +/// PgAlertStateStore's catch-inside: the evaluator is the one place that knows what a missing +/// answer means for the gate (fall back to the process-memory gate, warn, count the read into the #3013 +/// swallowed-read census), and putting the catch there means a fake that throws exercises the real +/// fallback path rather than a mirror of it. +/// +public interface ISelfAlertDeliveryStampStore +{ + /// The UTC instant the document keyed was last DELIVERED, or + /// null when no delivery has ever been stamped (a fresh store, or a document that has only ever + /// failed to deliver). + Task GetDeliveredAtUtcAsync(string stateKey, CancellationToken cancellationToken); + + /// Records that the document keyed was delivered at + /// , replacing any earlier stamp. + Task RecordDeliveredAtUtcAsync(string stateKey, DateTime deliveredAtUtc, CancellationToken cancellationToken); +} + +/// +/// over collect.collector_state (V44) — the store's +/// general key/value state table, under a fleet-sentinel server_id and an owner name of its own. No +/// migration rung: the table already holds one short row per (server, collector, key), which is exactly +/// two rows of this shape, and #3580's fix shape names an existing key/value table as the natural home. +/// +/// server_id = 0 — the fleet sentinel the store already uses, in two places. +/// DarlingObservability.FleetServerId is 0 for the fleet-wide retention purge's +/// collection_log run-record, and FleetSweepStore.FleetScopeServerId is 0 for watch items +/// about the fleet rather than one member; both rest on the same fact, that server_ids are FNV-1a hashes +/// of a storage name and no monitored server is registered at 0. These two documents are fleet-level +/// self-alerts (they fire under the synthetic store label, not a server), so they take the same id rather +/// than mint a third convention. The table's server_id is integer NOT NULL and part of the +/// primary key, which is also why a sentinel and not a NULL. +/// +/// collector_name = 'self_alert' — an owner name that is not a collector definition's. +/// The QueryStoreBackfill precedent: a worker that keeps state in this table under its own name, +/// deliberately outside the definitions' StateKeys machinery, so no collector's declared-key read or +/// per-database prune can reach it. Every prune in DarlingCollectorRunner is scoped to a real +/// server_id AND a collector's own name, and the two migration-time deletes (V77, V114) name their +/// collector; nothing retires rows under this name, which is the point — this is state, not facts, and +/// the table carries no retention by design. +/// +/// The stamp is the value, as ISO 8601 round-trip text, and updated_at is the write +/// time. Two different instants: the stamp is the evaluator's clock at the fire (the controllable +/// utcNow seam, so a test can place it), and updated_at is the wall clock the row was written +/// at, the column's meaning on every other row of the table. The stamp travels as text because the column +/// is text; the "O" format round-trips to the tick and carries its Z, so the read side gets +/// back without a SpecifyKind that could lie. A value that does not +/// parse — hand-edited, or written by nothing this build knows — reads as no stamp, with a warning, rather +/// than as a throw: the gate then falls back to memory for that tick, and the next successful delivery +/// overwrites the row. +/// +/// Schema-qualified like the V44 DDL and FleetSweepStore, not bare like the runner's own +/// reads: this store is also handed a scratch database in the live test, where the connection string +/// carries no search path and only the migrator's best-effort ALTER DATABASE would resolve a bare +/// name. Naming the schema costs nothing and removes the dependency. +/// +public sealed class PgSelfAlertDeliveryStampStore : ISelfAlertDeliveryStampStore +{ + /// See the class remarks: the fleet-wide sentinel both existing fleet-scope writers use. + public const int FleetServerId = DarlingObservability.FleetServerId; + + /// See the class remarks: the owner name, deliberately not a collector definition's. + public const string StateCollectorName = "self_alert"; + + /// The digest's stamp key — one fixed row, the document's one fixed in-memory key made + /// durable. + public const string CostDigestStateKey = "digest_delivered_at"; + + /// The fleet-sweep rollup's stamp key. + public const string FleetSweepRollupStateKey = "sweep_rollup_delivered_at"; + + /// The alert pass's own deadline (DarlingAlertReadAdapter.AlertPassCommandTimeoutSeconds), + /// because this read runs inside it: a stamp read that outlives the pass's budget is a stamp read that + /// should have failed toward the memory gate. + internal const int CommandTimeoutSeconds = DarlingAlertReadAdapter.AlertPassCommandTimeoutSeconds; + + internal const string GetSql = @" +SELECT state_value +FROM collect.collector_state +WHERE server_id = $1 +AND collector_name = $2 +AND state_key = $3"; + + internal const string UpsertSql = @" +INSERT INTO collect.collector_state (server_id, collector_name, state_key, state_value, updated_at) +VALUES ($1, $2, $3, $4, $5) +ON CONFLICT (server_id, collector_name, state_key) +DO UPDATE SET state_value = EXCLUDED.state_value, updated_at = EXCLUDED.updated_at"; + + private readonly NpgsqlDataSource _postgres; + private readonly ILogger? _logger; + + public PgSelfAlertDeliveryStampStore(NpgsqlDataSource postgres, ILogger? logger = null) + { + _postgres = postgres ?? throw new ArgumentNullException(nameof(postgres)); + _logger = logger; + } + + public async Task GetDeliveredAtUtcAsync(string stateKey, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(stateKey); + + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + using var command = new NpgsqlCommand(GetSql, connection) { CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = FleetServerId }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = StateCollectorName }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = stateKey }); + + var result = await command.ExecuteScalarAsync(cancellationToken); + if (result is not string text) + { + return null; + } + + if (!DateTime.TryParseExact(text, "O", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var stamp)) + { + _logger?.LogWarning( + "Self-alert delivery stamp {Key} holds '{Value}', which is not a round-trip UTC instant; treating it as no stamp", + stateKey, text); + return null; + } + + /* "O" with its Z parses straight to Kind=Utc; a hand-written value carrying an offset lands as Local + and is converted rather than trusted, so the caller's `now - stamp` is a UTC-to-UTC subtraction + either way. */ + return stamp.Kind == DateTimeKind.Utc ? stamp : stamp.ToUniversalTime(); + } + + public async Task RecordDeliveredAtUtcAsync(string stateKey, DateTime deliveredAtUtc, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(stateKey); + + await using var connection = await _postgres.OpenConnectionAsync(cancellationToken); + using var command = new NpgsqlCommand(UpsertSql, connection) { CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = FleetServerId }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = StateCollectorName }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = stateKey }); + command.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Text, + /* Stamped Utc before formatting so the text always carries Z — a Kind-Unspecified caller value + would otherwise format without a zone and read back as Unspecified. */ + Value = DateTime.SpecifyKind(deliveredAtUtc, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture), + }); + command.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Timestamp, + /* Naive UTC, Kind-Unspecified — the product-wide PG `timestamp` discipline (PgAlertStateStore. + NaiveUtcNow, the runner's own SaveCollectorStateAsync): binding Kind=Utc against a `timestamp` + column does not fail, Npgsql infers timestamptz and PostgreSQL casts it into the SERVER's zone, + so the row lands silently offset while every other timestamp in the store is UTC. */ + Value = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified), + }); + + await command.ExecuteNonQueryAsync(cancellationToken); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/ServiceCommandDeadlines.cs b/Darling/PerformanceMonitor.Darling.Service/ServiceCommandDeadlines.cs index 5f17c5614..f4d871862 100644 --- a/Darling/PerformanceMonitor.Darling.Service/ServiceCommandDeadlines.cs +++ b/Darling/PerformanceMonitor.Darling.Service/ServiceCommandDeadlines.cs @@ -86,7 +86,11 @@ public static class ServiceCommandDeadlines /// /// It happens to land on the same 10 s the alert pass uses. That is two derivations meeting, /// not a number being reused: the alert pass is bounded above by its own 30 s sweep interval and - /// below by a 1,744.9 ms forced-plan read, neither of which appears anywhere above. + /// below by a 1,744.9 ms forced-plan read, neither of which appears anywhere above. (That read has + /// since been re-pathed — #3573 found it streaming the whole fleet's window per server, and the + /// covering index in PgTableTuning made it an Index Only Scan — so its 1,744.9 ms is now the + /// recorded cost of a replaced plan rather than a live figure. Nothing here rested on it, which is + /// the point of this paragraph.) /// public const int CollectionSweepSeconds = 10; @@ -129,7 +133,14 @@ public static class ServiceCommandDeadlines /// deliberately pathological 2,925-row window (the volume a per-query cooldown of zero would produce). /// Those figures are from a local container, so the floor is anchored on the one COLD store read this /// sweep has measured in production instead: #2882's forced-plan read at 1,744.9 ms over ~6.0 GB. - /// A value under ~2 s could fire on a cold read; 5 s carries ~2.9x over it. + /// A value under ~2 s could fire on a cold read; 5 s carries ~2.9x over it. That anchor has since aged + /// in both directions and still holds as a floor: the table it was measured over grew to 23 GB and the + /// read's cold tail reached 10.3 s (#3573) — because its plan walked the entire fleet's two-hour slice and + /// filtered one server out of it, a fleet-width tax that scales with servers, not with this table — and + /// then the covering index in PgTableTuning made it an Index Only Scan over one server's rows. So + /// 1,744.9 ms is the recorded cost of a cold store read under an access path that no longer exists; as + /// a stand-in for "what a cold store read can cost here" it is, if anything, generous, and the + /// arithmetic above is unchanged. /// /// So the CEILING is what fixes this number, not the floor. The pass budget puts it at /// 6 s or less and the floor only rules out the bottom two, which is worth saying plainly rather than diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs index 67e03f56b..10fa21a68 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreConfigProvider.cs @@ -992,11 +992,12 @@ INSERT INTO config_alert_settings ( retention_hold_warn_ratio, retention_hold_critical_ratio, deadlock_warn_per_hour, deadlock_critical_per_hour, pg_deadlock_count_threshold, pg_blocking_count_threshold, - fleet_sweep_enabled, fleet_sweep_interval_minutes) + fleet_sweep_enabled, fleet_sweep_interval_minutes, + self_disk_free_warn_gb) VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44, $45, $46, $47, $48, $49, $50, $51, $52, $53, $54, $55, $56, $57, $58, $59, $60, $61, $62, $63, - $64, $65, $66, $67) + $64, $65, $66, $67, $68) ON CONFLICT (id) DO NOTHING", connection) { CommandTimeout = ServiceCommandDeadlines.BootstrapSeconds }; command.Parameters.AddWithValue(a.Enabled); command.Parameters.AddWithValue(a.CpuEnabled); @@ -1088,6 +1089,10 @@ reports back. */ holds is what get_alert_settings reports back. */ command.Parameters.AddWithValue(a.FleetSweepEnabled); command.Parameters.AddWithValue(a.FleetSweepIntervalMinutes); + /* #3528, bound in the same order the V126 column was appended. Seeded RAW like every sibling: + the floor-at-0 lives on DarlingAlertSettings, so what the store holds is what + get_alert_settings reports back. */ + command.Parameters.AddWithValue(a.SelfDiskFreeWarnGb); await command.ExecuteNonQueryAsync(ct); } @@ -1361,7 +1366,8 @@ internal static string NormalizePlanXmlCompression(string? value) => retention_hold_warn_ratio, retention_hold_critical_ratio, deadlock_warn_per_hour, deadlock_critical_per_hour, pg_deadlock_count_threshold, pg_blocking_count_threshold, - fleet_sweep_enabled, fleet_sweep_interval_minutes + fleet_sweep_enabled, fleet_sweep_interval_minutes, + self_disk_free_warn_gb FROM config_alert_settings WHERE id = 1", connection) { CommandTimeout = ServiceCommandDeadlines.SerialLoopSeconds }; using var reader = await command.ExecuteReaderAsync(ct); if (!await reader.ReadAsync(ct)) @@ -1486,6 +1492,12 @@ but not read here -- or read but not selected -- would silently reset the knob t default on every worker start. */ FleetSweepEnabled = reader.GetBoolean(64), FleetSweepIntervalMinutes = reader.GetInt32(65), + + /* #3528 store-disk-warn GB floor appended (V126) at ordinal 66. Same reachability rule as + every appended knob: ApplyToConfig replaces config.Alerts wholesale, so a column selected + but not read here -- or read but not selected -- would silently reset the floor to the + shipped default on every worker start. */ + SelfDiskFreeWarnGb = reader.GetInt32(66), }; var analysis = new AnalysisConfig { @@ -1733,10 +1745,12 @@ public static EffectiveSchedule ResolveSchedule(string collectorName, int server } /* Sanitize operator-supplied overrides before they drive scheduling / a destructive purge: a - negative frequency or a retention < 1 (0 would invert the purge cutoff and wipe the table) - is treated as "no override" and falls through to the next level. Defense in depth with the - V17 CHECK constraints and the DarlingRetention sink clamp. */ - var frequency = ValidFrequency(perServer?.FrequencyMinutes) ?? ValidFrequency(fleet?.FrequencyMinutes) ?? def.FrequencyMinutes; + negative frequency, a retention < 1 (0 would invert the purge cutoff and wipe the table), or a + delta-family cadence past the gap-policy cap (#3532 — every cycle would exceed + CollectorDeltaCalculator.DefaultMaxGapSeconds, re-baseline, and store a zero delta forever) is + treated as "no override" and falls through to the next level. Defense in depth with the + V17 CHECK constraints, the viewer editor's ValidateSchedule, and the DarlingRetention sink clamp. */ + var frequency = ValidFrequency(collectorName, perServer?.FrequencyMinutes) ?? ValidFrequency(collectorName, fleet?.FrequencyMinutes) ?? def.FrequencyMinutes; var retention = ValidRetention(perServer?.RetentionDays) ?? ValidRetention(fleet?.RetentionDays) ?? def.RetentionDays; /* No override row falls back to the collector's shared default enabled state — true for nearly every collector, but false for an opt-in one like long_query_completions (#1496). Falling back @@ -1831,9 +1845,16 @@ collecting rather than silently collecting nothing on a row full of whitespace. /// cutoff and delete everything, so it degrades to "no override" (fall through to the default). private static int? ValidRetention(int? days) => days is int v && v >= 1 ? v : null; - /// A frequency override is honored only when >= 0 (0 = on-load-only); negative degrades to - /// "no override" so a bad value can't make a collector run every sweep. - private static int? ValidFrequency(int? minutes) => minutes is int v && v >= 0 ? v : null; + /// A frequency override is honored only when >= 0 (0 = on-load-only) and, for a + /// delta-family collector, no slower than + /// (#3532 — a cadence past the delta gap policy fabricates permanent zeros); a bad value degrades to + /// "no override" so it can't make a collector run every sweep or stop measuring. + private static int? ValidFrequency(string collectorName, int? minutes) => + minutes is int v + && v >= 0 + && !(CollectorDeltaCalculator.IsDeltaFamily(collectorName) && v > CollectorDeltaCalculator.MaxDeltaFrequencyMinutes) + ? v + : null; /* ---------------- helpers ---------------- */ diff --git a/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs b/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs index 6bab7fc1a..b82a9ea1e 100644 --- a/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs +++ b/Darling/PerformanceMonitor.Darling.Service/StoreTlsCertificates.cs @@ -50,8 +50,19 @@ internal static Generated Create(string hostName, IPAddress listenIp, int validi var notAfter = notBefore.AddYears(validityYears); using var caKey = RSA.Create(2048); + /* The random mint tag makes every generated root's SUBJECT unique, and that uniqueness is + load-bearing (#3557): Windows silently caches each root a TLS client ever saw into that + user's intermediate-CA store, one entry per rotation, forever. When rotations all share + one subject, that cached pile grows until the chain engine's subject-matched issuer walk + fails outright — "unknown chain building error" from X509Chain.Build, measured at ~50 + cached same-subject roots, killing SslStreamCertificateContext.Create and the + default-trust build a viewer's SslStream runs. (SKI/AKI does NOT prevent it; tested.) + Distinct subjects mean cached copies of other rotations are never candidate issuers for + this chain, so the pile never forms. Distinct names are also honest X.509: each rotation + IS a different CA, and two CAs sharing a DN with different keys is the pathology. */ + var mintTag = Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(4)); var caRequest = new CertificateRequest( - $"CN=PerformanceMonitor Darling store root ({hostName})", caKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + $"CN=PerformanceMonitor Darling store root ({hostName} {mintTag})", caKey, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); /* pathLenConstraint 0: this root may sign end-entity certs only — even with the key discarded, the constraint documents the intent in the certificate itself. */ caRequest.CertificateExtensions.Add(new X509BasicConstraintsExtension(true, true, 0, true)); diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js index 70aefbe97..2e48b625f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js @@ -28,6 +28,7 @@ const ALERT_COLUMNS = [ { key: "alert_time", label: "Time", format: "time" }, { key: "server_name", label: "Server" }, { key: "metric_name", label: "Metric" }, + { key: "severity", label: "Severity", render: (a) => severityCell(a) }, { key: "current_value", label: "Value", format: "num1" }, { key: "threshold_value", label: "Threshold", format: "num1" }, { key: "status", label: "Status", render: (a) => statusCell(a) }, @@ -35,6 +36,26 @@ const ALERT_COLUMNS = [ { key: "triage", label: "Triage", render: (a) => triageCell(a) }, ]; +/* #3539 A8e: the tier the alert FIRED at, as the tool reports it — never re-derived here from the metric name + * (R1). The service reads it off the row's persisted context and falls back to the name only for rows that + * carry none; severity_source says which arm answered, and it rides as the cell's title because the two are + * not equal evidence: "critical" from the row is what the operator was paged with, "critical" from the name + * is what the map says about the name. The tone classes are the status cell's own. */ +const SEVERITY_TONE = { critical: "Critical", warning: "Warning", resolution: "Healthy", info: "Unknown" }; +const SEVERITY_SOURCE_TITLE = { + fired: "The tier this alert fired at, read from the row", + metric_name: "Implied by the metric name; this row carries no fired tier", +}; + +function severityCell(a) { + if (a.severity == null) return el("span", { class: "muted", text: "—" }); + return el("span", { + class: "status-cell sev-" + (SEVERITY_TONE[a.severity] || "Unknown"), + text: String(a.severity), + title: SEVERITY_SOURCE_TITLE[a.severity_source] || null, + }); +} + /* Deep-link into the #2710 triage page for this row — the SAME route the alert webhooks link to, anchored at * this row's own firing instant, so the in-app path and the delivered link land on an identical page. */ function triageCell(a) { diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 44a3e1ba2..62aabe6d8 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -451,10 +451,14 @@ function groupControl() { /* * #3017: the deadlock total's denominator, as a VISIBLE sub-line rather than a tooltip. * - * total_deadlocks comes out of v_deadlocks, which is the SQL Server extended-event capture and nothing else, - * so it is structurally zero on a PostgreSQL fleet — permanently, whatever those clusters do. Zero is also - * exactly what a genuinely quiet SQL Server fleet reports, and the tile could not tell an operator which one - * they were looking at. The API answers that (deadlock_coverage), and this renders it. + * total_deadlocks is each engine's own instrument summed across the fleet — the SQL Server extended-event + * capture, and since #3539 the PostgreSQL server counter differenced over the window — so a server whose + * deadlock-source collector is silent or denied contributes a structural zero. Zero is also exactly what a + * genuinely quiet fleet reports, and the tile could not tell an operator which one they were looking at. + * The API answers that (deadlock_coverage), and this renders it; the note on the tile's title names how many + * of the read servers were counted the counter way (postgres_servers), which is a sub-count of servers_read. + * (Before #3539 a PostgreSQL target was structurally uncounted and this sub-line read "N of M" on any mixed + * fleet; a PostgreSQL fleet whose pg_database_stats collectors run now reads "all".) * * ALWAYS shown when the API reports coverage, including at full coverage, for two reasons. A line that * appeared only on partial coverage would make its ABSENCE the load-bearing signal, which an operator has to @@ -545,9 +549,17 @@ function rollup(d) { function serverCard(c) { const cls = bandClass(c.band); + /* #3528: the band's fold skips Unknown, so a card can read Healthy off one measured metric of six — + say so instead of rendering an unqualified green. The counts are the server's own (R1: read the + pre-computed field, never re-derive); an awaiting card keeps its plain status, which already says + nothing has been measured yet. */ + const coverage = + c.metric_count > 0 && c.measured_metric_count < c.metric_count + ? " · " + c.measured_metric_count + " of " + c.metric_count + " measured" + : ""; const statusLine = c.awaiting_first_collection ? el("div", { class: "status-line awaiting", text: c.status }) - : el("div", { class: "status-line", text: c.status + " · last collect " + localClock(c.last_collection) }); + : el("div", { class: "status-line", text: c.status + " · last collect " + localClock(c.last_collection) + coverage }); return el( "div", @@ -625,16 +637,25 @@ export function metricBands(c) { the reachability signal the card already carries (is_online, the same one that bands the card Offline and titles the header "no recent collection"): when the server is offline the chip reads "Stale" in the neutral Unknown tone instead of a green "OK · N healthy" (the "no recent collection" detail carries the specifics, - and "Stale" is the word the Collection Health tab lands on for these rows once its own floor is crossed). */ + and "Stale" is the word the Collection Health tab lands on for these rows once its own floor is crossed). + + #3539 A6: a REACHABLE server with no collector banded at all (collector_count 0, nothing failing) arrives + with collector_severity "Unknown" from the shared band, and its value reads "n/a" — the chip's word for a + metric with no reading (the Threads chip's) — rather than "OK". R1: the severity is read off the card, not + re-derived here; only the WORD keys on the count, and only so a green word never sits under a grey chip. */ const collectorsStale = c.is_online === false; const collectorsValue = collectorsStale ? "Stale" : c.failed_collector_count > 0 ? fmtInt(c.failed_collector_count) + " failing" - : "OK"; + : c.collector_count > 0 + ? "OK" + : "n/a"; const collectorsDetail = collectorsStale ? "no recent collection" + (c.last_collection ? " · last " + relTime(c.last_collection) : "") - : fmtInt(c.healthy_collector_count) + " healthy · " + fmtInt(c.failed_collector_count) + " failing"; + : c.collector_count > 0 + ? fmtInt(c.healthy_collector_count) + " healthy · " + fmtInt(c.failed_collector_count) + " failing" + : "no collector banded yet"; const collectorsSeverity = collectorsStale ? "Unknown" : c.collector_severity; return el("div", { class: "metric-bands" }, [ diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 80ce815f4..1266aea91 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -1612,6 +1612,21 @@ export const POSTGRES_TABS = [ ctx.label + ", the newest reading of each facet in it; in CAUSAL order rather than alphabetically - fix them top to bottom; the remedy is per facet, and on Aurora/RDS it says which changes need a parameter group and a reboot", "No readiness state collected. Unlike the grids around it an empty panel here is never the healthy answer - the collector writes one row per facet on every run whatever it finds - so this means it has not run for this server, or the window is shorter than its hourly cadence." ), + /* #3607: the rest of the logging surface, directly under plan-capture readiness because it is the + other half of the same onboarding question - is this target telling us everything it could. Judged + from the newest stored pg_server_config snapshot rather than collected, so it takes no window; the + read reports the snapshot time as captured_at. Every setting is shown whatever its + verdict, for the reason the readiness grid shows satisfied facets: a list of only the failures + cannot show that a target IS instrumented. */ + table( + "Logging Settings Audit", + "get_pg_logging_audit", + { server }, + "facets", + PG_LOGGING_AUDIT_COLUMNS, + "newest configuration snapshot; verdict describes the LINES each setting writes - partial means a threshold is filtering and is the recommended posture for log_min_duration_statement; the remedy is worded for this server's hosting flavour", + "No configuration snapshot to audit. pg_server_config runs hourly, so a server registered in the last hour has not reached its first collection - this is the absence of evidence, not a verdict about the server's logging." + ), /* Directly UNDER the query shapes, because that is the question it answers (#2539). A statement whose time makes no sense from its row count usually spilled, and pg_stat_database's temp counters are the only evidence of that we collect — the statement stats themselves cannot see it. The deadlock and @@ -3461,6 +3476,23 @@ const PG_PLAN_CAPTURE_READINESS_COLUMNS = [ { key: "last_observed", label: "Last Seen", format: "time", small: true }, ]; +/* Verdict beside the setting, then the value, then the prose columns widest and last - the same reading + order as the readiness grid above it: scan down setting and verdict to find the row that is off, then + read across for what it unlocks, what it costs, and what to type. remedy is the column the panel exists + for and is worded for the server's hosting flavour by the read, not by this file. */ +const PG_LOGGING_AUDIT_COLUMNS = [ + { key: "setting", label: "Setting", mono: true }, + { key: "verdict", label: "Verdict" }, + { key: "value", label: "Value", mono: true }, + { key: "unit", label: "Unit", small: true }, + { key: "source", label: "Source", small: true }, + { key: "change_needs", label: "Change needs", small: true }, + { key: "unlocks", label: "Unlocks", wrap: true }, + { key: "recommended", label: "Recommended", wrap: true }, + { key: "cost_note", label: "Cost", wrap: true }, + { key: "remedy", label: "Remedy", wrap: true }, +]; + const PG_TOP_QUERY_COLUMNS = [ { key: "queryid", label: "Query ID", mono: true }, { key: "calls", label: "Calls", format: "int" }, diff --git a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs index f5386ffe5..a45618e23 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DailySummarySql.cs @@ -77,8 +77,12 @@ FROM v_dmv_blocking_snapshots GROUP BY 1 ), cpu AS ( - /* Total host CPU = SQL + other-process (NULL on Linux -> 0), matching the alert engine and the - Overview headline; sustained >= 80 samples drive the day's band. */ + /* Total host CPU = SQL + other-process (NULL on Linux -> 0), matching the Overview headline. The + 80 is ServerHealthThresholds.CpuWarningPercent, the card band's Warning bar, restated as a + literal because this is a SQL string and pinned equal by both suites (#3539 A2). Deliberately + NOT the alert engine's configurable CPU threshold: this statement re-counts at read time, so + binding it to a knob would recolour every past day the moment the knob moved. The count feeds + a bar that scales with the window (DailyHealthThresholds.HighCpuCriticalSamplesFor). */ SELECT date_trunc('day', collection_time) AS d, COUNT(*) FILTER (WHERE (sqlserver_cpu_utilization + COALESCE(other_process_cpu_utilization, 0)) >= 80) AS c FROM v_cpu_utilization_stats @@ -87,7 +91,8 @@ GROUP BY 1 ), coll AS ( /* Any run (all statuses) marks the day as collected -> it appears even if every metric is quiet - (a quiet monitored day is Healthy/green, not No-Data/grey). errs feeds the Critical band. */ + (a quiet monitored day is Healthy/green, not No-Data/grey). runs is also the denominator the + error SHARE bands on (#3539 A2); errs alone used to make the day Critical on presence. */ SELECT date_trunc('day', collection_time) AS d, COUNT(*) AS runs, COUNT(*) FILTER (WHERE status = 'ERROR') AS errs @@ -147,7 +152,26 @@ UNION SELECT d FROM alerts /* Peak block wait (ms) from the SAME source the blocking count came from (BPR preferred, DMV-snapshot fallback), so the day-detail blocking reason ('N blocking events (peak block X)') reconciles with the count. 0 when the blocking came from a source without a wait time. */ - COALESCE(CASE WHEN COALESCE(b.c, 0) > 0 THEN b.max_wait_ms ELSE dm.max_wait_ms END, 0) AS peak_block_wait_ms + COALESCE(CASE WHEN COALESCE(b.c, 0) > 0 THEN b.max_wait_ms ELSE dm.max_wait_ms END, 0) AS peak_block_wait_ms, + /* Every collector run in the window (#3539 A2): the denominator that turns collection_errors + into a share. Appended after peak_block_wait_ms so every existing ordinal read stays where it was. */ + COALESCE(cl.runs, 0) AS collection_runs, + /* #3541 A9: how many of the seven per-signal sources hold at least one row for the day — the + retention arm's PRESENCE fact. The spine is a UNION over nine sources aging out at different + horizons, each LEFT JOINed and COALESCEd to zero, so a day the collection log (60 days) or the + alert log (90) still names can have every signal (30) purged and read as measured-zero-Healthy. + The reader judges such a day by this count against the store's retention horizon: zero sources + past the horizon is a purged shell, some sources past it is a day the purge has not reached. + A source counts as present when its grouped CTE produced a row for the day, which for cpu is + "any sample" (the FILTER is inside the aggregate) and for waits is "any positive delta". + Appended LAST, after collection_runs, for the same ordinal reason. */ + (CASE WHEN w.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN q.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN dl.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN b.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN dm.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN cp.d IS NULL THEN 0 ELSE 1 END) + + (CASE WHEN m.d IS NULL THEN 0 ELSE 1 END) AS signal_sources_present FROM day_spine s LEFT JOIN waits w ON w.d = s.d LEFT JOIN queries q ON q.d = s.d diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgIoReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgIoReader.cs index 3695b4618..be8577e52 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgIoReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgIoReader.cs @@ -44,6 +44,18 @@ nothing was read or bytes are simply not measured on this version. */ decimal ExtendBytes, bool ByteCountersTracked); + /// + /// One page of I/O combinations plus the two denominators their shares are taken over (#3541 A7). + /// WindowTotalReads and WindowTotalReadTimeMs are the reads and read time of EVERY + /// (backend_type, object, context) combination that moved in the window, not of the rows on the page. + /// Off the same statement as the rows, as window aggregates over the grouped result before LIMIT, + /// so they cannot drift from them. On the page rather than on : facts about the + /// window, and a per-row copy would widen the Viewer's projection and invite a reader to sum them. + /// WindowTotalReadTimeMs is a sum of stored zeros when track_io_timing is off; the tool + /// already nulls every time figure under that setting, and the total goes with them. + /// + public sealed record PgIoPage(List Rows, long WindowTotalReads, double WindowTotalReadTimeMs); + /// /// Positive-difference-per-interval, summed over the window — the same rule the statement read uses, /// for the same reason: these are cumulative counters, so a plain last-minus-first goes negative @@ -59,7 +71,14 @@ nothing was read or bytes are simply not measured on this version. */ /// This comment previously claimed NULL survived the arithmetic. It does not, and the claim was /// worse than useless: it would have licensed someone to drop the tracked flag believing the NULLs were /// carrying the information. The flag is not belt-and-braces — it is the only discriminator. - /// $1 server_id, $2/$3 window (naive UTC). + /// window_total_reads / window_total_read_time_ms are the whole window's figures, on + /// every row (#3541 A7). SUM(SUM(d_reads)) OVER () is a window aggregate over the GROUPED result, + /// evaluated after GROUP BY / HAVING and before ORDER BY / LIMIT, so it sums every + /// combination that moved rather than the ones the cap admitted. The tool used to divide each row by the + /// sum of the rows it had fetched, so a three-row page summed to 100% of "total" reads by construction. + /// One pass over a result the query has already grouped; no second statement. Appended LAST so the + /// positional reader's existing ordinals stand. + /// $1 server_id, $2/$3 window (naive UTC), $4 row cap. /// public const string PgIoSql = """ WITH differenced AS ( @@ -116,22 +135,40 @@ ORDER BY collection_time coalesce(SUM(d_read_bytes), 0) AS read_bytes, coalesce(SUM(d_write_bytes), 0) AS write_bytes, coalesce(SUM(d_extend_bytes), 0) AS extend_bytes, - bool_or(byte_counters_tracked) AS byte_counters_tracked + bool_or(byte_counters_tracked) AS byte_counters_tracked, + /* #3541 A7: the WINDOW's totals, not the page's - see the remarks. Same on every row. */ + CAST(SUM(coalesce(SUM(d_reads), 0)) OVER () AS bigint) AS window_total_reads, + SUM(coalesce(SUM(d_read_time_ms), 0)) OVER () AS window_total_read_time_ms FROM differenced GROUP BY backend_type, object_type, context /* Anything that moved, ordered by the work that actually costs time. A combination with no - activity in the window is not a finding and would crowd out the ones that are. */ + activity in the window is not a finding and would crowd out the ones that are. The read-count + key is NOT a cosmetic tiebreak: track_io_timing is off by DEFAULT, so on a stock server every + time sum here is zero and the count is the entire ordering — the tool reports which key decided + (#3536), and dropping the second key would make the untracked case effectively unordered. */ HAVING coalesce(SUM(d_reads), 0) + coalesce(SUM(d_writes), 0) + coalesce(SUM(d_extends), 0) + coalesce(SUM(d_hits), 0) > 0 ORDER BY coalesce(SUM(d_read_time_ms), 0) DESC, coalesce(SUM(d_reads), 0) DESC LIMIT $4 """; + /// The rows alone — the WPF Viewer's grid, which has no column for the window totals. public static async Task> GetPgIoAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) => + (await GetPgIoPageAsync(postgres, serverId, startUtc, endUtc, limit, cancellationToken)).Rows; + + /// + /// The paged read: rows, busiest first, and the whole window's reads and read + /// time beside them. The MCP tool asks for limit + 1 so it can OBSERVE truncation rather than infer it. + /// + public static async Task GetPgIoPageAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, CancellationToken cancellationToken = default) { var rows = new List(); + long windowTotalReads = 0; + double windowTotalReadTimeMs = 0; await using var command = postgres.CreateCommand(PgIoSql); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); @@ -146,6 +183,9 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* Identical on every row (OVER () with no partition); the last write wins with the same number. */ + windowTotalReads = reader.IsDBNull(19) ? 0 : reader.GetInt64(19); + windowTotalReadTimeMs = reader.IsDBNull(20) ? 0 : reader.GetDouble(20); rows.Add(new PgIoRow( reader.IsDBNull(0) ? null : reader.GetString(0), reader.IsDBNull(1) ? null : reader.GetString(1), @@ -168,7 +208,7 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC !reader.IsDBNull(18) && reader.GetBoolean(18))); } - return rows; + return new PgIoPage(rows, windowTotalReads, windowTotalReadTimeMs); } /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgKernelStatsReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgKernelStatsReader.cs index f781051bc..dec498d4d 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgKernelStatsReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgKernelStatsReader.cs @@ -47,9 +47,36 @@ public sealed record PgKernelStatRow( bool CounterReset, DateTime CaptureTime); + /// + /// One page of per-query CPU plus the denominator its shares are taken over (#3541 A7). + /// WindowTotalCpuMs is user plus system CPU across EVERY (database, query) series in the + /// window, not across the rows on the page. Off the same statement as the rows, as a window aggregate + /// over the differenced result before LIMIT, so it cannot drift from them. On the page rather than + /// on : a fact about the window, not a statement. + /// + public sealed record PgKernelStatsPage(List Rows, double WindowTotalCpuMs); + /* Newest and oldest per (database, query) in the window, then differenced. The reset arm takes the newest value WHOLE rather than clamping to zero: GREATEST(new - old, 0) reports "no CPU" across a - restart, which reads as a quiet server rather than as a reset. */ + restart, which reads as a quiet server rather than as a reset. + + #3541 A7: window_total_cpu_ms is the WHOLE window's user + system CPU, on every row. A window aggregate + over the differenced result - evaluated before ORDER BY / LIMIT - so it sums every series the window + holds rather than the rows the cap admits. The tool used to divide each row by the sum of the rows it + had fetched, so a three-row page summed to 100% of "total" CPU by construction. + + THE ORDER BY, AND WHY THERE IS A `differenced` LAYER. This read shipped `ORDER BY 3 + 4 DESC`, meant as + "ordinal 3 plus ordinal 4" - user_ms plus system_ms. PostgreSQL reads a bare integer in ORDER BY as an + output-column ordinal, but `3 + 4` is an EXPRESSION, so it is the constant 7, and a constant sort key + orders nothing: the planner dropped it and the rows came back sorted by the tiebreak alone, + (database_name, query_id). Verified with EXPLAIN on PostgreSQL 18 - `ORDER BY 1 + 2 DESC, x` plans as + `Sort Key: x`. So "ranked by total CPU" was never true, and every page this read served under a limit + was the alphabetically-first series rather than the hottest, which the A7 work found the moment a + three-row page's shares came back 60 / 10 / 30. Found by executing the statement, not by reading it; + a dozen string assertions on this SQL passed throughout. An ORDER BY cannot name a select-list alias + inside an expression either, so the CASEs are given names one layer down and the outer query sorts + by `user_ms + system_ms` - which also lets the window aggregate reference them by name instead of + repeating both CASEs. */ public const string PgKernelStatsSql = """ WITH newest AS ( SELECT DISTINCT ON (database_name, query_id) @@ -92,29 +119,54 @@ FROM newest AS n LEFT JOIN oldest AS o ON o.database_name IS NOT DISTINCT FROM n.database_name AND o.query_id = n.query_id + ), + differenced AS ( + SELECT + database_name, + query_id, + CASE WHEN counter_reset THEN n_user ELSE n_user - o_user END AS user_ms, + CASE WHEN counter_reset THEN n_system ELSE n_system - o_system END AS system_ms, + CASE WHEN counter_reset THEN n_reads ELSE n_reads - o_reads END AS read_bytes, + CASE WHEN counter_reset THEN n_writes ELSE n_writes - o_writes END AS write_bytes, + CASE WHEN counter_reset THEN n_majflts ELSE n_majflts - o_majflts END AS major_faults, + counter_reset, + collection_time + FROM paired ) SELECT database_name, query_id, - CASE WHEN counter_reset THEN n_user ELSE n_user - o_user END AS user_ms, - CASE WHEN counter_reset THEN n_system ELSE n_system - o_system END AS system_ms, - CASE WHEN counter_reset THEN n_reads ELSE n_reads - o_reads END AS read_bytes, - CASE WHEN counter_reset THEN n_writes ELSE n_writes - o_writes END AS write_bytes, - CASE WHEN counter_reset THEN n_majflts ELSE n_majflts - o_majflts END AS major_faults, + user_ms, + system_ms, + read_bytes, + write_bytes, + major_faults, counter_reset, - collection_time - FROM paired - ORDER BY 3 + 4 DESC, database_name, query_id + collection_time, + SUM(user_ms + system_ms) OVER () AS window_total_cpu_ms + FROM differenced + ORDER BY user_ms + system_ms DESC, database_name, query_id LIMIT $4 """; + /// The rows alone — the WPF Viewer's grid, which has no column for the window total. public static async Task> GetPgKernelStatsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) => + (await GetPgKernelStatsPageAsync(postgres, serverId, startUtc, endUtc, limit, cancellationToken)).Rows; + + /// + /// The paged read: rows, most CPU first, and the whole window's CPU beside them. + /// The MCP tool asks for limit + 1 so it can OBSERVE truncation rather than infer it. + /// + public static async Task GetPgKernelStatsPageAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(postgres); var rows = new List(); + double windowTotalCpuMs = 0; await using var command = postgres.CreateCommand(PgKernelStatsSql); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); @@ -130,6 +182,8 @@ public static async Task> GetPgKernelStatsAsync( { var userMs = reader.IsDBNull(2) ? 0 : reader.GetDouble(2); var systemMs = reader.IsDBNull(3) ? 0 : reader.GetDouble(3); + /* Identical on every row (OVER () with no partition); the last write wins with the same number. */ + windowTotalCpuMs = reader.IsDBNull(9) ? 0 : Convert.ToDouble(reader.GetValue(9)); rows.Add(new PgKernelStatRow( DatabaseName: reader.IsDBNull(0) ? null : reader.GetString(0), @@ -146,6 +200,6 @@ public static async Task> GetPgKernelStatsAsync( : DateTime.SpecifyKind(reader.GetDateTime(8), DateTimeKind.Utc))); } - return rows; + return new PgKernelStatsPage(rows, windowTotalCpuMs); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgLoggingAuditReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgLoggingAuditReader.cs new file mode 100644 index 000000000..815fa0966 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgLoggingAuditReader.cs @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The newest stored pg_settings snapshot for one server, as the input to the logging-settings audit +/// (get_pg_logging_audit, #3607). +/// +/// This is a READ over pg_server_config, not a collector. Plan-capture readiness +/// (PgPlanCaptureReadinessCollector, #2564) persists its facets because judging them means probing +/// the target — is the library in shared_preload_libraries, does an auto_explain.* GUC exist +/// at all — and the answer is worth a history of its own. Every setting the logging audit judges is a plain +/// core GUC that PgServerConfigCollector (#2658) already stores hourly with its value, source and +/// context, so a second collector would write the same rows under a second name and the two would drift. +/// The judgment happens at read time over the snapshot that is already there, and the snapshot's +/// collection_time is the audit's captured_at (#3541 A10's stamp, selected on the row statement). +/// +/// Anchored on MAX(collection_time), not on a window — the same reason +/// gives: configuration is a state, and an hours +/// filter would answer "this server has no logging configuration" about a server whose hourly collector +/// last ran just outside it. +/// +/// Session-scoped rows are excluded, spelled the same way the config reader spells them. +/// pg_settings is a per-backend view, so a client-sourced row is the collector's own +/// connection. For THIS read the trap is sharper than for the config listing: the monitoring login could +/// carry SET log_min_duration_statement = 0 in its own session, and an audit that read that row would +/// declare the server instrumented while every other backend logs nothing. The list is inline rather than +/// substituted in so the constant stays SQL that DarlingPgReadSqlParsesLiveTests can parse-check — +/// the config reader's header records why. +/// +/// The whole snapshot travels, not just the settings judged. Two reasons. The audit needs +/// evidence of the HOSTING FLAVOUR to word its remedies — ALTER SYSTEM plus a reload on a server +/// somebody administers, a parameter group on RDS/Aurora — and the store's engine token cannot separate RDS +/// from self-hosted (MonitoredEngineKind.Postgres is both). The presence of any rds.* GUC in +/// the snapshot can, and it is in the data already. And the settings list is owned by the judgment code in +/// the service; filtering here would put the list in two places. A snapshot is a few hundred short rows +/// once an hour per server, so this costs nothing the listing read does not already spend. +/// +public static class DarlingPgLoggingAuditReader +{ + /// The GUC name. + /// Its value as the server rendered it — TEXT, units and all, never cast here. + /// The unit pg_settings reports (ms, kB), or null. + /// When a change takes effect: postmaster needs a restart, everything + /// else a reload at most. + /// Where the value came from — default, configuration file, + /// user, database… — which is what separates the server's setting from an override + /// the monitoring role happens to resolve. + /// The compiled-in default, so "PostgreSQL 15 turned this on" is visible + /// without a table of defaults that would rot at every major. + /// The file and the running server disagree about this one. + /// When the snapshot was taken — the audit's captured_at. + public sealed record PgLoggingSettingRow( + string Name, + string? Setting, + string? Unit, + string? Context, + string? Source, + string? BootValue, + bool PendingRestart, + DateTime CollectionTime); + + /* The same two-step anchor as CurrentConfigSql: the newest collection_time for the server, then every + non-session row at that instant. ORDER BY name so the judgment code's lookups and the test fixtures + meet the rows in one stable order. */ + public const string NewestSnapshotSql = """ + SELECT + c.name, + c.setting, + c.unit, + c.context, + c.source, + c.boot_val, + coalesce(c.pending_restart, false), + c.collection_time + FROM pg_server_config AS c + WHERE c.server_id = $1 + AND c.collection_time = ( + SELECT MAX(collection_time) + FROM pg_server_config + WHERE server_id = $1) + AND c.name IS NOT NULL + AND coalesce(c.source, '') NOT IN ('client', 'session', 'override') + ORDER BY c.name + """; + + public static async Task> GetNewestSnapshotAsync( + NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(postgres); + + var rows = new List(); + await using var command = postgres.CreateCommand(NewestSnapshotSql); + command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; + command.Parameters.AddWithValue(serverId); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new PgLoggingSettingRow( + Name: reader.GetString(0), + Setting: reader.IsDBNull(1) ? null : reader.GetString(1), + Unit: reader.IsDBNull(2) ? null : reader.GetString(2), + Context: reader.IsDBNull(3) ? null : reader.GetString(3), + Source: reader.IsDBNull(4) ? null : reader.GetString(4), + BootValue: reader.IsDBNull(5) ? null : reader.GetString(5), + PendingRestart: !reader.IsDBNull(6) && reader.GetBoolean(6), + /* Stored naive-UTC, read back as UTC — the readiness reader's convention. */ + CollectionTime: DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))); + } + + return rows; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgPlanCaptureReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgPlanCaptureReader.cs index 9fee5b34c..e93ffaa06 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgPlanCaptureReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgPlanCaptureReader.cs @@ -11,6 +11,7 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using NpgsqlTypes; namespace PerformanceMonitor.Darling.Storage; @@ -67,14 +68,26 @@ FROM pg_plan_capture WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 + /* The optional queryid pin, IN the SQL rather than filtered client-side over a page (#3533): the + ranking is by total duration, so a cheap-but-wanted query sits arbitrarily far below the top and + no page size makes it reachable — filtering a fetched page turned "ranked low" into "was never + captured". NULL leaves the read as the top-duration page. */ + AND ($4::bigint IS NULL OR query_id = $4) GROUP BY query_id, plan_hash ORDER BY sum(duration_ms) DESC - LIMIT $4 + LIMIT $5 """; + public static Task> GetPgPlanCaptureAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) => + GetPgPlanCaptureAsync(postgres, serverId, startUtc, endUtc, limit, queryId: null, cancellationToken); + + /// Pins the read to one statement's plans, server-side, over the whole window. + /// Null returns the top page by total duration instead. public static async Task> GetPgPlanCaptureAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, - CancellationToken cancellationToken = default) + long? queryId, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(postgres); @@ -85,6 +98,13 @@ public static async Task> GetPgPlanCaptureAsync( /* SpecifyKind(Unspecified) at the BIND, the convention every PostgreSQL read here follows. */ command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + /* Typed explicitly rather than through AddWithValue: DBNull carries no type for Npgsql to infer, + so an untyped null fails at bind time — the same reason the trend reader's TextOrNull exists. */ + command.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Bigint, + Value = (object?)queryId ?? DBNull.Value, + }); command.Parameters.AddWithValue(limit); await using var reader = await command.ExecuteReaderAsync(cancellationToken); diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgSessionStatesReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgSessionStatesReader.cs index e095de1d8..89c9b5425 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgSessionStatesReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgSessionStatesReader.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Npgsql; @@ -383,9 +384,56 @@ public sealed record LongRunningSessionRow( /// sys.dm_exec_requests — a table of requests actually executing, where an idle session has no /// row at all. /// - /// $1 server_id, $2 threshold (ms), $3 recency floor (naive UTC), $4 row limit. + /// The noise opt-outs, and which SQL Server sibling each mirrors (#3539). SQL Server's + /// CheckLongRunningQueriesAsync reads sys.dm_exec_requests through five switchable noise + /// filters plus an unconditional session_id > 50; this read had none, so autovacuum at + /// minute 31, a nightly pg_dump, or a manual VACUUM on a large relation paged with a mute as + /// the only remedy — and the mute is weaker here than on SQL Server, because this table stores no query + /// text (see the collector) and so a mute rule cannot match a statement. What the row DOES carry is + /// backend_type, application_name and the whitelisted command_tag, which are the + /// three handles below. + /// + /// Non-client backends (unconditional): backend_type <> 'client backend' — + /// autovacuum workers, walsenders (streaming replication, pg_basebackup), logical replication + /// workers, background workers. Mirrors SQL Server's unconditional session_id > 50: a system + /// process is not a query. NULL-safe in the INCLUDING direction — backend_type is in the + /// privileged column set and comes back NULL without pg_monitor, and dropping every row on a + /// redacted target would make the alert silently never fire exactly where the collector has already + /// stamped state_is_redacted. + /// Maintenance statements (unconditional): command_tag in VACUUM, + /// ANALYZE, REINDEX, CLUSTER — a manual vacuum of a large table runs for an hour by + /// design, and the alert asks about QUERIES. SQL Server has no statement-shape sibling because it needs + /// none: its row carries the text and an operator mutes ALTER INDEX by pattern; here the tag is + /// the only handle, so the exclusion has to live in the read. CREATE is deliberately NOT in the + /// list: the tag cannot tell CREATE INDEX CONCURRENTLY (maintenance) from CREATE TABLE AS + /// SELECT (a query), and the honest side of that ambiguity is to report — the incident line shows + /// the tag so a reader can see which it was. + /// Dump and restore utilities (the {0} placeholder, on the SHARED + /// longRunningQueryExcludeBackups switch): application_name in pg_dump, + /// pg_dumpall, pg_restore, pg_basebackup — the names libpq's + /// fallback_application_name gives those tools, so a dump's COPY ... TO STDOUT sessions + /// carry them without operator configuration. Mirrors BackupsFilter (BACKUPTHREAD / + /// BACKUPIO) on the SAME knob, the way longRunningQueryEnabled and the threshold are already + /// shared: "do not page me for backups" is one preference, not one per engine. psql is NOT + /// excluded — an operator's ad-hoc statement running long is precisely a long-running query, and SQL + /// Server does not exclude SSMS either. + /// Idle in transaction (unconditional, pre-existing): its own condition — see the + /// paragraph above. + /// + /// The other three SQL Server switches have no honest PostgreSQL reading and are not faked: + /// sp_server_diagnostics and XE_LIVE_TARGET_TVF name SQL Server internals with no + /// counterpart, and WAITFOR's twin (pg_sleep) is invisible here because the row carries + /// no text and SELECT pg_sleep(...) tags as SELECT. CDC's nearest relative — logical + /// replication workers — is already out through backend_type. excludedDatabases is applied + /// after the read, exactly as the SQL Server adapter applies it. + /// + /// $1 server_id, $2 threshold (ms), $3 recency floor (naive UTC), $4 row limit; {0} is + /// the switchable filter block. A PROPERTY rather than a string field on purpose: the shipped-read + /// parse census (DarlingPgReadSqlParsesLiveTests) parse-checks every static string field on a + /// reader, and a template with a placeholder in it cannot parse. The two RENDERINGS below are the + /// fields, so both texts that can actually reach the store are the ones parse-checked. /// - public const string CurrentLongRunningSessionsSql = """ + public static string CurrentLongRunningSessionsSqlTemplate => """ WITH recent AS ( SELECT max(collection_time) AS latest_capture FROM pg_session_states @@ -406,18 +454,50 @@ JOIN recent AS r WHERE s.server_id = $1 AND s.query_duration_ms >= $2 AND s.is_idle_in_transaction = false + AND coalesce(s.backend_type, 'client backend') = 'client backend' + AND coalesce(s.command_tag, '') NOT IN ('VACUUM', 'ANALYZE', 'REINDEX', 'CLUSTER') + {0} ORDER BY s.query_duration_ms DESC, s.pid LIMIT $4 """; + /// The switchable dump/restore opt-out — the PostgreSQL reading of + /// longRunningQueryExcludeBackups. A constant rather than inline so a test can pin the list and + /// the read can be asserted to include it exactly when the switch is on. + public const string BackupUtilitiesFilter = + "AND coalesce(s.application_name, '') NOT IN ('pg_dump', 'pg_dumpall', 'pg_restore', 'pg_basebackup')"; + + /// The read as it runs with the backups opt-out ON — the shipped default, and what the + /// pre-#3539 constant name meant. Kept under the old name so pins on the shape keep pointing at the + /// text that actually executes on an untouched store. A static readonly field, not a property, + /// so the parse census sees it; is a const, so this + /// initializer cannot read it before it exists. + public static readonly string CurrentLongRunningSessionsSql = BuildCurrentLongRunningSessionsSql(excludeBackups: true); + + /// The read as it runs with the backups opt-out OFF — the other text that can reach the store, + /// held as a field for the same parse-census reason. The host does not read this; it calls + /// with the setting. + public static readonly string CurrentLongRunningSessionsSqlBackupsIncluded = BuildCurrentLongRunningSessionsSql(excludeBackups: false); + + /// Renders for one setting of the shared + /// backups switch. Public so the host's call and a test's pin are the same text. + public static string BuildCurrentLongRunningSessionsSql(bool excludeBackups) => + CurrentLongRunningSessionsSqlTemplate.Replace("{0}", excludeBackups ? BackupUtilitiesFilter : ""); + + /// The shared longRunningQueryExcludeBackups switch — drops the + /// dump/restore utilities' sessions (see the SQL's doc comment). + /// The shared excludedDatabases list, applied after the read + /// case-insensitively exactly as the SQL Server adapter applies it; a row with no database name is + /// kept. Null or empty excludes nothing. public static async Task> GetCurrentLongRunningSessionsAsync( NpgsqlDataSource postgres, int serverId, long thresholdMs, DateTime nowUtc, int recencyMinutes, int limit, + bool excludeBackups, IReadOnlyList? excludedDatabases, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(postgres); var rows = new List(); - await using var command = postgres.CreateCommand(CurrentLongRunningSessionsSql); + await using var command = postgres.CreateCommand(BuildCurrentLongRunningSessionsSql(excludeBackups)); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(thresholdMs); @@ -439,7 +519,27 @@ public static async Task> GetCurrentLongRunningSessi reader.IsDBNull(6) ? -1 : reader.GetInt64(6))); } - return rows; + return FilterExcludedDatabases(rows, excludedDatabases); + } + + /// The excludedDatabases arm, pulled out so it is pinnable without a store: the SQL + /// Server adapter's exact rule (ordinal-ignore-case on the name; a row with no database name is kept, + /// because an exclusion list names databases and a session on none of them is not on an excluded + /// one). Applied AFTER the row limit, as the SQL Server adapter applies it, so an excluded database's + /// sessions can crowd the cap — the same known shape on both engines rather than a quiet divergence + /// where one engine's cap counts excluded rows and the other's does not. + public static List FilterExcludedDatabases( + List rows, IReadOnlyList? excludedDatabases) + { + if (excludedDatabases is not { Count: > 0 }) + { + return rows; + } + + return rows + .Where(r => string.IsNullOrEmpty(r.DatabaseName) + || !excludedDatabases.Any(e => string.Equals(e, r.DatabaseName, StringComparison.OrdinalIgnoreCase))) + .ToList(); } public static async Task GetPgSessionStatesCaptureCountsAsync( diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgStatementReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgStatementReader.cs index b4afcd3ee..7b3756731 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgStatementReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgStatementReader.cs @@ -52,6 +52,29 @@ the next refresh. Distinguishing "not captured yet" from "" is what stops a call string as the query. */ string? QueryText = null); + /// + /// One page of the top-queries read, plus the denominator its shares are taken over. + /// WindowTotalExecTimeMs is the execution time of EVERY query shape that ran in the window, + /// not of the rows on the page (#3541 A7). It comes off the same statement as the rows — a window + /// aggregate over the grouped result, evaluated before LIMIT cuts it — so it cannot drift from + /// them and costs no second read. It lives on the page rather than on + /// because it is a fact about the window, not about a statement, and a per-row copy would either + /// widen every consumer's projection or invite a reader to sum it. + /// Zero when the window holds no rows: the HAVING admits only shapes that ran, so a + /// zero total and an empty page are the same fact stated twice. + /// + public sealed record PgTopQueriesPage(List Rows, long WindowTotalExecTimeMs); + + /// + /// The row cap the unpaged read binds — the 50 this query carried as a literal LIMIT from the + /// day it was written. The WPF Viewer's top-queries grid is the remaining caller of that read and keeps + /// reading exactly what it always read; the MCP tool binds the caller's own limit through + /// instead, because a tool that advertised a limit up to 1,000 and + /// silently served 50 was the hidden-cap shape #3541 A3 named (this read was not in A3's list; A7 found it + /// because the page-scoped share was computed over those 50). + /// + public const int PgTopQueriesGridRowCap = 50; + /// /// Every counter is reported for the WINDOW, never as a lifetime total. Summing the cumulative /// counters directly would multiply each query's whole history by the number of snapshots in the @@ -76,7 +99,16 @@ string as the query. */ /// (queryid, database_id), which is the grain a "top queries" answer wants. /// max_exec_peakmem_bytes and max_exec_time_ms stay MAX — they are high-water /// marks, not counters, and differencing a high-water mark would be meaningless. - /// $1 server_id, $2/$3 window (naive UTC). + /// window_total_exec_time_ms is the whole window's figure, on every row (#3541 A7). + /// SUM(SUM(delta_total_exec_time_ms)) OVER () is a window aggregate over the GROUPED result: PostgreSQL + /// evaluates window functions after GROUP BY / HAVING and before ORDER BY / LIMIT, + /// so the value is the sum across every shape the window holds, not across the rows the cap lets + /// through. The tool used to divide each row by the sum of the rows it had fetched, which made a + /// three-row page sum to 100% of "total" time by construction; this column is the honest denominator, + /// and it is one extra pass over a result the query has already grouped and is about to sort — no + /// second statement, no way to drift from the rows. Identical on every row, read once off the first. + /// $1 server_id, $2/$3 window (naive UTC), $4 row cap — a PARAMETER, bound to the caller's limit by + /// the page read and to by the unpaged one. /// public const string PgTopQueriesSql = """ WITH differenced AS ( @@ -142,7 +174,10 @@ ELSE CAST(coalesce(SUM(d_orcache_blks_hit), 0) AS bigint) END AS orcache_blks_hi because the grain here is (queryid, database_id) while text is keyed on queryid alone — one text per group by construction, so MAX picks it without widening the GROUP BY. LEFT JOIN, so a queryid whose text has not been captured yet still ranks; it simply reads as null. */ - MAX(t.query_text) AS query_text + MAX(t.query_text) AS query_text, + /* #3541 A7: the WINDOW's total, not the page's - see the remarks. Window aggregate over the + grouped rows, so it is evaluated before LIMIT and is the same on every row. */ + CAST(SUM(SUM(delta_total_exec_time_ms)) OVER () AS bigint) AS window_total_exec_time_ms FROM differenced LEFT JOIN collect.pg_statement_text AS t ON t.server_id = $1 @@ -150,14 +185,29 @@ LEFT JOIN collect.pg_statement_text AS t GROUP BY differenced.queryid, database_id HAVING SUM(delta_total_exec_time_ms) > 0 ORDER BY SUM(delta_total_exec_time_ms) DESC - LIMIT 50 + LIMIT $4 """; + /// + /// The unpaged read the WPF Viewer's grid calls: the same statement at the cap it always carried. Kept + /// with its signature so the Viewer keeps compiling and keeps reading exactly what it read; the window + /// total the statement now also returns is dropped here because the grid has no column for it. + /// public static async Task> GetPgTopQueriesAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) => + (await GetPgTopQueriesPageAsync(postgres, serverId, startUtc, endUtc, PgTopQueriesGridRowCap, cancellationToken)).Rows; + + /// + /// The paged read: rows, heaviest first, and the whole window's execution time + /// beside them. The MCP tool asks for limit + 1 so it can OBSERVE truncation rather than infer it. + /// + public static async Task GetPgTopQueriesPageAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, CancellationToken cancellationToken = default) { var rows = new List(); + long windowTotalExecTimeMs = 0; await using var command = postgres.CreateCommand(PgTopQueriesSql); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); @@ -168,9 +218,13 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC stores; found by the round-2 review. */ command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(limit); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* Same value on every row by construction (OVER () with no partition); reading it on each is + cheaper than a branch and the last write wins with the same number. */ + windowTotalExecTimeMs = reader.IsDBNull(15) ? 0 : reader.GetInt64(15); rows.Add(new PgStatementRow( reader.GetInt64(0), reader.IsDBNull(1) ? 0 : reader.GetInt64(1), @@ -191,6 +245,6 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC reader.IsDBNull(14) ? null : reader.GetString(14))); } - return rows; + return new PgTopQueriesPage(rows, windowTotalExecTimeMs); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgTrendReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgTrendReader.cs index 600c92624..f88ea8229 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgTrendReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgTrendReader.cs @@ -107,6 +107,19 @@ ORDER BY collection_time /// cumulative ones again. They are written on collection, where the previous reading is in hand, and /// re-deriving them here would give a DIFFERENT answer whenever a snapshot is missing — the collector's /// delta spans the gap it actually observed, and a LAG here would span the gap in the stored data. + /// + /// The same reasoning now applies to the INTERVAL (#3540, V128). The collector stores + /// sample_interval_seconds beside the deltas — the span the delta actually accrued over, which + /// is not the gap between stored snapshots either: this collector skips idle rows at the write, so for + /// a statement that ran, went quiet for three passes and ran again, the stored delta spans four + /// intervals and a LAG over the rows it left behind spans one. Per snapshot the interval is MAX + /// over the queryid's rows (one per database/user/toplevel entry) — an entry first seen in an + /// otherwise steady pass carries 0 and contributes 0 to the sums, so MAX is 0 only when every row was + /// unknowable (a restart or a pg_stat_statements_reset()), and that 0 becomes NULL through + /// NULLIF; a NULL rate drops the point in the reader rather than plotting 0.00 calls/sec. NULL + /// (a pre-V128 row that never recorded one) falls back to the LAG this read always used, so history + /// renders as it did. No ELSE 0: the first snapshot of a pre-V128 series is absent rather than + /// a fabricated 0.0. /// public const string QueryDurationTrendSql = """ WITH per_snapshot AS ( @@ -114,7 +127,10 @@ WITH per_snapshot AS ( collection_time, SUM(delta_calls) AS calls, SUM(delta_total_exec_time_ms) AS total_exec_ms, - extract(epoch FROM (collection_time - LAG(collection_time) OVER (ORDER BY collection_time))) AS interval_seconds + CASE WHEN MAX(sample_interval_seconds) IS NULL + THEN extract(epoch FROM (collection_time - LAG(collection_time) OVER (ORDER BY collection_time))) + ELSE NULLIF(MAX(sample_interval_seconds), 0) + END AS interval_seconds FROM pg_statement_stats WHERE server_id = $1 AND queryid = $2 @@ -133,9 +149,9 @@ CASE WHEN coalesce(calls, 0) > 0 THEN coalesce(total_exec_ms, 0)::double precision / calls ELSE NULL END AS mean_exec_ms, + /* NULL, not 0, when the interval is unknowable or absent — the reader drops the point. */ CASE WHEN interval_seconds > 0 THEN coalesce(calls, 0)::double precision / interval_seconds - ELSE 0 END AS calls_per_second FROM per_snapshot ORDER BY collection_time @@ -266,12 +282,21 @@ public static async Task> GetQueryDurationTrendA await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* A NULL rate is an unknowable interval (#3540, V128) — every row of the snapshot stored 0, a + restart or a stats reset — or a pre-V128 first snapshot with nothing to LAG against. The + point is dropped: its calls and time are the calculator's fabricated zeros, and a point that + says "0 calls, 0 ms" at the moment of a restart is the lie this column exists to stop. */ + if (reader.IsDBNull(4)) + { + continue; + } + points.Add(new PgQueryDurationTrendPoint( reader.GetDateTime(0), reader.IsDBNull(1) ? 0 : reader.GetInt64(1), reader.IsDBNull(2) ? 0 : reader.GetDouble(2), reader.IsDBNull(3) ? 0 : reader.GetDouble(3), - reader.IsDBNull(4) ? 0 : reader.GetDouble(4))); + reader.GetDouble(4))); } return points; diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitReader.cs index f194eeb28..58c2c83dc 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitReader.cs @@ -39,6 +39,16 @@ public sealed record PgWaitRow( double TotalWaitTimeMs, double AvgWaitTimeMs); + /// + /// One page of wait events plus the denominator their shares are taken over (#3541 A7). + /// WindowTotalWaitTimeMs is the wait time of EVERY event in the window, not of the rows on the + /// page. Off the same statement as the rows, as a window aggregate over the grouped result before + /// LIMIT, so it cannot drift from them. On the page rather than on : it is + /// a fact about the window, and a per-row copy would invite a reader to sum it. Zero when the page is + /// empty, which is the same fact twice. + /// + public sealed record PgWaitStatsPage(List Rows, double WindowTotalWaitTimeMs); + /// /// Aggregated over the window from the delta columns, not the raw cumulative counters — summing /// cumulative values across snapshots would multiply the whole history by the snapshot count. @@ -51,6 +61,12 @@ public sealed record PgWaitRow( /// caller-supplied limit and then applied it with Take(limit) — so a caller asking for more than 50 /// silently got 50, and every request below that fetched rows only to discard them. Same shape as every /// other read in this store. + /// window_total_wait_time_ms is the whole window's wait time, on every row (#3541 A7). + /// SUM(SUM(delta_wait_time_us)) OVER () is a window aggregate over the GROUPED result, evaluated + /// after GROUP BY / HAVING and before ORDER BY / LIMIT, so it sums every event + /// that accrued time rather than the events the cap admitted. The tool used to divide each row by the sum + /// of the rows it had fetched, so a three-row page summed to 100% of "total" wait by construction. One + /// pass over a result the query has already grouped; no second statement. /// public const string PgWaitStatsSql = """ SELECT @@ -62,7 +78,9 @@ public sealed record PgWaitRow( WHEN SUM(delta_waits) > 0 THEN (SUM(delta_wait_time_us) / 1000.0) / SUM(delta_waits) ELSE 0 - END AS avg_wait_time_ms + END AS avg_wait_time_ms, + /* #3541 A7: the WINDOW's total, not the page's - see the remarks. Same on every row. */ + SUM(SUM(delta_wait_time_us)) OVER () / 1000.0 AS window_total_wait_time_ms FROM pg_wait_stats WHERE server_id = $1 AND collection_time >= $2 @@ -75,11 +93,22 @@ ORDER BY SUM(delta_wait_time_us) DESC LIMIT $4 """; + /// The rows alone — the WPF Viewer's grid, which has no column for the window total. public static async Task> GetPgWaitStatsAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) => + (await GetPgWaitStatsPageAsync(postgres, serverId, startUtc, endUtc, limit, cancellationToken)).Rows; + + /// + /// The paged read: rows, heaviest first, and the whole window's wait time beside + /// them. The MCP tool asks for limit + 1 so it can OBSERVE truncation rather than infer it. + /// + public static async Task GetPgWaitStatsPageAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, CancellationToken cancellationToken = default) { var rows = new List(); + double windowTotalWaitTimeMs = 0; await using var command = postgres.CreateCommand(PgWaitStatsSql); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); @@ -94,6 +123,8 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { + /* Identical on every row (OVER () with no partition); the last write wins with the same number. */ + windowTotalWaitTimeMs = reader.IsDBNull(5) ? 0 : Convert.ToDouble(reader.GetValue(5)); rows.Add(new PgWaitRow( reader.GetString(0), reader.GetString(1), @@ -102,6 +133,6 @@ row falls out of the window and the read silently returns nothing. Hidden by UTC reader.IsDBNull(4) ? 0 : Convert.ToDouble(reader.GetValue(4)))); } - return rows; + return new PgWaitStatsPage(rows, windowTotalWaitTimeMs); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitSamplingReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitSamplingReader.cs index 1f926ea6b..81c6577ac 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitSamplingReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgWaitSamplingReader.cs @@ -52,12 +52,30 @@ public sealed record PgWaitSamplingRow( bool CounterReset, DateTime CaptureTime); + /// + /// One page of sampled waits plus the denominator their shares are taken over (#3541 A7). + /// WindowTotalSamples is the differenced sample count of EVERY (event type, event, query) series + /// in the window, not of the rows on the page. Off the same statement as the rows, as a window aggregate + /// over the joined result before LIMIT, so it cannot drift from them. On the page rather than on + /// : a fact about the window, not a series, and a per-row copy would invite + /// a reader to sum it. + /// + public sealed record PgWaitSamplingPage(List Rows, long WindowTotalSamples); + /* Newest and oldest per key in one pass each, then differenced. Two DISTINCT ON scans rather than a window function because the key is compound and the hypertable is ordered by time - the same idiom every other reader here uses. The key does NOT include database_name: the profile is cluster-wide and the table carries no such - column, deliberately (#2599 is about not inventing that attribution). */ + column, deliberately (#2599 is about not inventing that attribution). + + #3541 A7: window_total_samples is the WHOLE window's differenced sample count, on every row. A window + aggregate over the joined result - PostgreSQL evaluates it before ORDER BY / LIMIT - so it sums every + series the window holds rather than the rows the cap admits. The tool used to divide each row by the + sum of the rows it had fetched, so a three-row page summed to 100% of the samples by construction. + The CASE is repeated inside the SUM rather than referenced by alias because a window function cannot + name a select-list alias of the same level; the two expressions are pinned identical by test. Appended + LAST so the ordinal ORDER BY 4 still names sample_count. */ public const string PgWaitSamplingSql = """ WITH newest AS ( SELECT DISTINCT ON (event_type, event, query_id) @@ -88,7 +106,11 @@ THEN n.sample_count n.profile_period_ms, n.backend_count, (n.sample_count < o.sample_count) AS counter_reset, - n.collection_time + n.collection_time, + SUM(CASE WHEN n.sample_count < o.sample_count + THEN n.sample_count + ELSE n.sample_count - coalesce(o.sample_count, 0) + END) OVER () AS window_total_samples FROM newest AS n LEFT JOIN oldest AS o ON o.event_type IS NOT DISTINCT FROM n.event_type @@ -98,13 +120,67 @@ AND o.event IS NOT DISTINCT FROM n.event LIMIT $4 """; + /// + /// Which instrument is feeding this server's pg_wait_sampling rows (#3604), read from the + /// collector's own state: PgWaitSamplingCollector records a PgWaitInstrument token under + /// collector_state (server_id, 'pg_wait_sampling', 'instrument') on every cycle, whichever arm ran. + /// Off the store rather than re-derived here because the decision was made once at connect and the + /// collector is the only thing that knows which arm its last cycle took; a read guessing from + /// profile_period_ms would be right until an operator set the extension's period to a second. + /// Null when no cycle has recorded one — a store written before #3604, or a server whose collector + /// has not completed a cycle since. The tool says so rather than picking a default. + /// + public const string InstrumentSql = """ + SELECT state_value, updated_at + FROM collector_state + WHERE server_id = $1 + AND collector_name = 'pg_wait_sampling' + AND state_key = 'instrument' + """; + + /// The recorded instrument and when the collector last recorded it (UTC), or null. + public sealed record WaitInstrumentState(string Instrument, DateTime RecordedAtUtc); + + /// Runs . An unrecognised token is returned as-is; the tool decides + /// whether to echo it. + public static async Task GetWaitInstrumentAsync( + NpgsqlDataSource postgres, int serverId, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(postgres); + + await using var command = postgres.CreateCommand(InstrumentSql); + command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; + command.Parameters.AddWithValue(serverId); + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken) || reader.IsDBNull(0)) + { + return null; + } + + return new WaitInstrumentState( + reader.GetString(0), + reader.IsDBNull(1) ? default : DateTime.SpecifyKind(reader.GetDateTime(1), DateTimeKind.Utc)); + } + + /// The rows alone — the WPF Viewer's grid, which has no column for the window total. public static async Task> GetPgWaitSamplingAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, + CancellationToken cancellationToken = default) => + (await GetPgWaitSamplingPageAsync(postgres, serverId, startUtc, endUtc, limit, cancellationToken)).Rows; + + /// + /// The paged read: rows, most-sampled first, and the whole window's sample count + /// beside them. The MCP tool asks for limit + 1 so it can OBSERVE truncation rather than infer it. + /// + public static async Task GetPgWaitSamplingPageAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, int limit, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(postgres); var rows = new List(); + long windowTotalSamples = 0; await using var command = postgres.CreateCommand(PgWaitSamplingSql); command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; command.Parameters.AddWithValue(serverId); @@ -120,6 +196,8 @@ public static async Task> GetPgWaitSamplingAsync( { var samples = reader.IsDBNull(3) ? 0 : reader.GetInt64(3); var periodMs = reader.IsDBNull(4) ? 10 : reader.GetInt32(4); + /* SUM over bigint widens to numeric in PostgreSQL; identical on every row, last write wins. */ + windowTotalSamples = reader.IsDBNull(8) ? 0 : Convert.ToInt64(reader.GetValue(8)); rows.Add(new PgWaitSamplingRow( EventType: reader.IsDBNull(0) ? null : reader.GetString(0), @@ -134,6 +212,6 @@ public static async Task> GetPgWaitSamplingAsync( : DateTime.SpecifyKind(reader.GetDateTime(7), DateTimeKind.Utc))); } - return rows; + return new PgWaitSamplingPage(rows, windowTotalSamples); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgXminReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgXminReader.cs index b611e15ea..c4ea096a9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingPgXminReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingPgXminReader.cs @@ -78,6 +78,52 @@ FROM latest AS l ORDER BY l.xmin_age DESC """; + /// + /// How many times the xmin collector actually CAPTURED in the window — the honest denominator for + /// "what share of the window was this source winning" (#3541 A12, contract rule 5). + /// 's samples counts a source's OWN rows, and the collector + /// writes a row only when something holds the horizon — an unheld capture stores nothing. So a source + /// that held the horizon in 2 of the window's 288 captures had samples = 2, + /// samples_as_winner = 2, and read as winning 100% of the window: a two-minute query rendered as + /// a chronic holder. The denominator has to be every time the collector LOOKED, and only + /// collection_log has that: one row per run INCLUDING the zero-row (healthy, unheld) runs, behind + /// its (server_id, collection_time) index, counting the exact collector whose captures are being + /// fractioned. Runs that stored nothing because they could not look (ERROR / ABANDONED / PERMISSIONS / + /// YIELDED) are excluded: a cycle that did not look is not evidence the horizon was clear. + /// This is the SAME denominator the alert evaluator's horizon arm uses + /// (DarlingPostgresAlertReadAdapter.XminSql's captures CTE, #3537): same table, same + /// collector name, same SUCCESS filter — pinned equal by DarlingPgXminReaderTests so the MCP payload and + /// the alert can never fraction the same window over different denominators. A separate statement rather + /// than a CROSS JOIN onto the holder rows because the viewer renders field for + /// field and this is a window fact, not a row fact. The log write is failure-isolated and can skip a + /// row, so the count may UNDERCOUNT — the payload says so rather than clamping the share. + /// $1 server_id, $2/$3 window (naive UTC). + /// + public const string XminCapturesInWindowSql = """ + SELECT COUNT(*) AS captures_in_window + FROM collection_log + WHERE server_id = $1 + AND collector_name = 'pg_xmin_horizon' + AND collection_time >= $2 + AND collection_time <= $3 + AND status = 'SUCCESS' + """; + + /// Runs . + public static async Task GetXminCapturesInWindowAsync( + NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, + CancellationToken cancellationToken = default) + { + await using var command = postgres.CreateCommand(XminCapturesInWindowSql); + command.CommandTimeout = StorageCommandDeadlines.McpReadSeconds; + command.Parameters.AddWithValue(serverId); + /* Kind-Unspecified at the bind, for the reason GetPgXminHorizonAsync states. */ + command.Parameters.AddWithValue(DateTime.SpecifyKind(startUtc, DateTimeKind.Unspecified)); + command.Parameters.AddWithValue(DateTime.SpecifyKind(endUtc, DateTimeKind.Unspecified)); + var value = await command.ExecuteScalarAsync(cancellationToken); + return value is long count ? count : Convert.ToInt64(value); + } + public static async Task> GetPgXminHorizonAsync( NpgsqlDataSource postgres, int serverId, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken = default) diff --git a/Darling/PerformanceMonitor.Darling.Storage/OversizedPlanBacklog.cs b/Darling/PerformanceMonitor.Darling.Storage/OversizedPlanBacklog.cs index 270c12b33..03a299bdb 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/OversizedPlanBacklog.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/OversizedPlanBacklog.cs @@ -230,12 +230,17 @@ UPDATE collect.oversized_plan_backlog /// (server, query_hash) — optionally narrowed to one database, the shape both the MCP read (database /// optional) and the viewer's (database required) need from one statement. /// - /// Read from the backlog, not joined to the fact table. query_stats does not store - /// the statement offsets — they are read for the delta key and never persisted — so a join from a stored - /// fact row could only match on plan_handle + sql_handle, which for a multi-statement plan - /// is several backlog rows describing DIFFERENT statements' plans. Serving one of those as "the plan for - /// this query" is worse than serving nothing. query_hash on the row keys the fallback at exactly - /// the grain the readers already ask at. + /// Read from the backlog, not joined to the fact table. When this was written + /// query_stats did not store the statement offsets — they were read for the delta key and never + /// persisted — so a join from a stored fact row could only match on plan_handle + sql_handle, + /// which for a multi-statement plan is several backlog rows describing DIFFERENT statements' plans. + /// Serving one of those as "the plan for this query" is worse than serving nothing. V128 (#3540) stores + /// the offsets, so an exact join is POSSIBLE for rows written since; it is not taken here because the + /// readers ask at the query_hash grain (the Dashboard's and viewer's key), which + /// query_hash on the backlog row already serves, and because every row written before V128 + /// carries NULL offsets, so an exact join would go dark on an upgraded store for a raw retention's + /// worth of history. If a reader ever asks at the statement grain, the join is now available to + /// it. /// /// A non-null plan_xml is itself the "this row was capped" test the caller would otherwise /// make against query_plan_xml_bytes: the row exists only because the measurement exceeded the diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index 1cb523dc8..4dbec7958 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -198,6 +198,16 @@ Generated from the collector definition rather than by threading a specific late new Migration(123, "fleet-sweep-state", V123Sql), new Migration(124, "fleet-sweep-cadence-knobs", V124Sql), new Migration(125, "collector-database-scope", V125Sql), + new Migration(126, "self-disk-warn-gb-floor", V126Sql), + new Migration(127, "delta-family-interval-columns", V127Sql), + /* V128 re-emits the payload-resolving v_query_stats (its two new query_stats columns land mid-list, + ahead of the digests, which CREATE OR REPLACE VIEW refuses), so it rides the V121 idiom: V54's + gz pre-add, then every payload column's pre-add, then the regenerated view. MigrationLadderPins + holds the ordering. */ + new Migration(128, "delta-family-interval-completion", + V128Sql + "\n" + V54Sql + "\n" + + PgSchemaGenerator.GenerateQueryStatsPayloadColumnPreAdds() + "\n" + + PgSchemaGenerator.GenerateQueryStatsResolvingView()), }; /// @@ -651,6 +661,199 @@ ALTER TABLE config.config_alert_settings ALTER TABLE config.config_collector_schedules ADD COLUMN IF NOT EXISTS databases text[];"; + /// + /// V126 — the Store Disk Pressure warning's GB floor on the singleton config_alert_settings + /// row (#3528): the self-alert's percent trigger additionally requires free space below this many + /// GB before it fires, an AND qualifier so a large volume at a low percent (400 GB free on a 4 TB + /// store) stops paging CRITICAL. 0 removes the floor and restores the percent-only condition — + /// the pvs_floor_gb composition, deliberately not the target-volume pair's OR, whose GB + /// dimension ADDS fires. + /// + /// The column default IS the shipped constant + /// (DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb — restated as a literal here only because + /// a rung is a SQL string, and pinned equal by SelfDiskWarnGbFloorRungTests). Non-zero on + /// upgrade DELIBERATELY, unlike the V122 knobs: their acceptance was "an untouched store fires + /// exactly where it did", while #3528's is that the untouched firing IS the defect — the issue's + /// own example is a default-configured store paging "act now" with 400 GB of runway. 50 puts the + /// crossover at a 500 GB volume, so any store volume at or under that keeps the exact pre-#3528 + /// percent behaviour. + /// + /// No CHECK enforcing the bound, matching V119/V120/V122/V124: the floor-at-0 is enforced as + /// the update_alert_settings write bound and DarlingAlertSettings' read-side clamp — + /// the raw-in/clamped-out split every knob on this table uses. No reload beacon of its own: V17's + /// statement-level trg_bump_alert_settings already bumps config_service.config_version + /// on any write here. No GRANT: this table carries table-level grants with no column carve. + /// + private const string V126Sql = @" +ALTER TABLE config.config_alert_settings + ADD COLUMN IF NOT EXISTS self_disk_free_warn_gb integer NOT NULL DEFAULT 50;"; + + /// + /// V127 — sample_interval_seconds on the four delta families that persisted their deltas NAKED + /// (#3540): wait_stats, file_io_stats, latch_stats, spinlock_stats. The + /// measurement-layer keystone: the shared delta calculator reports (delta 0, interval 0) when no delta is + /// knowable — first sighting, counter reset, a gap past the measured 3600 s policy — and (0, n) when an + /// interval was genuinely idle, and the interval is the ONLY thing that tells those apart. These four + /// collectors discarded it at the write, so the fabricated zero survived as a measured one and every + /// per-second reader LAG-divided it into a confident 0.00 ms/sec at exactly the moments (restarts) it was + /// unknowable; the file-I/O latency chart rendered "0.00 ms" mid-restart; and the wait-rate window + /// statistic counted the restart collection as a sample. perfmon_stats and query_stats have + /// carried the column from the start and their readers NULLIF(sample_interval_seconds, 0) — this + /// rung gives the other four the same column, in the same integer type, so the same idiom applies. + /// Pinned by DeltaFamilyIntervalColumnsRungTests. + /// + /// Nullable, no DEFAULT, no backfill, matching every column-adding rung on a collector + /// table (V80, V81, V121): the interval a historical row accrued over was never recorded, so NULL is the + /// honest value and a backfilled 0 would stamp every pre-V127 row as "unknowable" and erase 30 days of + /// perfectly good history from every rate chart. Readers treat the three states distinctly: n > 0 + /// is the measured interval; 0 is the calculator's unknowable marker and maps to NULL (the point is + /// absent, never 0.00); NULL is a pre-V127 row and falls back to the LAG-over-collection_time + /// derivation those readers always used, so history keeps rendering exactly as it did. A nullable + /// no-default ADD COLUMN is a catalog-only change in PostgreSQL and TimescaleDB accepts it on a + /// compressed hypertable, so this stays instant on a multi-hundred-GB wait_stats. + /// + /// The passthrough views are refreshed because Postgres freezes a view's SELECT * + /// column list at CREATE (the V14 lesson, restated by V80 and V81): without the four + /// CREATE OR REPLACE VIEW lines every v_* reader would keep seeing the pre-V127 column list + /// forever and the new column would be invisible to the whole read layer. Appending is the one alteration + /// CREATE OR REPLACE VIEW permits, which is exactly what an ADD COLUMN produces. Fresh stores get + /// the column from the generated CREATE TABLE at V4/V10 (the collector definitions carry it now) and + /// this rung's ALTERs no-op there; the view refresh is idempotent either way. + /// + /// What this rung deliberately does NOT do: touch collect.wait_stats_baseline. That + /// continuous aggregate sums delta_wait_time_ms per collection over the raw table, so a restart + /// collection materializes as a total_wait_ms = 0 sample and drags the WaitStats/WaitMsPerSec + /// baselines' mean down (the campaign's A6). The measurement contract says the rollup should aggregate + /// sample_interval_seconds IS DISTINCT FROM 0 rows only — but a continuous aggregate cannot change + /// its defining query in place; the only path is DROP + CREATE + refresh, and the raw wait_stats + /// horizon is operator-editable and typically 30 days against the aggregate's 35-day baseline tier, so + /// a rebuild forfeits materialized baseline history that raw can no longer refill. That is the exact + /// trade #3527 declined for perfmon_baseline (its interval is LAG-derived from the collapsed series + /// for the same reason), and this rung declines it the same way. The follow-up is a NEW aggregate under + /// a new name with the filter baked in, built WITH NO DATA and backfilled by + /// --backfill-rollups, with the old one retired through RetiredBaselineRelations once the + /// new one has 35 days — the #2007 retirement shape, which loses nothing. Until then the baseline + /// provider's magnitude heuristic (the QUALIFY restart signature) is the guard it always was. + /// + private const string V127Sql = @" +ALTER TABLE collect.wait_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.file_io_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.latch_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.spinlock_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; + +/* Postgres FREEZES a view's SELECT * column list at CREATE, so the four passthroughs would keep serving + the pre-V127 column list forever — the V14 lesson, restated by V80 and V81. Appending is the one + alteration CREATE OR REPLACE VIEW permits, which is exactly what an ADD COLUMN produces. */ +CREATE OR REPLACE VIEW collect.v_wait_stats AS SELECT * FROM collect.wait_stats; +CREATE OR REPLACE VIEW collect.v_file_io_stats AS SELECT * FROM collect.file_io_stats; +CREATE OR REPLACE VIEW collect.v_latch_stats AS SELECT * FROM collect.latch_stats; +CREATE OR REPLACE VIEW collect.v_spinlock_stats AS SELECT * FROM collect.spinlock_stats;"; + + /// + /// V128 — the completion of V127 (#3540): sample_interval_seconds on the four delta families + /// V127 left naked — procedure_stats, memory_grant_stats, pg_wait_stats, + /// pg_statement_stats — and the two statement offsets on query_stats that its delta key + /// is made of. After this rung EVERY member of CollectorDeltaCalculator.DeltaFamilyCollectors + /// stores the interval its deltas accrued over, so the calculator's (delta 0, interval 0) "no delta + /// knowable" marker reaches the store from every family and no restart zero reads as a measurement + /// anywhere; Lite.Tests' DeltaFamilyIntervalColumnTests census asserts the set with nothing + /// left on its still-naked list. Same integer type as the six that already carry it, so the + /// one NULLIF(sample_interval_seconds, 0) idiom reads all ten. Pinned by + /// DeltaFamilyIntervalCompletionRungTests. + /// + /// Why five tables in one rung. The repo allows one un-landed rung at a time, and these + /// five changes are one change: every column here exists so the same reader idiom can be applied + /// uniformly (stored interval → NULLIF(…, 0); NULL → the LAG derivation; no ELSE 0), + /// and the offsets exist so the restart seed can rebuild the one delta key the store could not + /// reproduce. Five rungs would have been five fleet schema hops carrying one idea. + /// + /// Nullable, no DEFAULT, no backfill on all six columns, matching V127 and every + /// column-adding rung on a collector table (V80, V81, V121): a historical row never recorded its + /// interval or its offsets, so NULL is the honest value. A backfilled 0 interval would stamp every + /// pre-V128 row "unknowable" and blank 30 days of rate history; a backfilled 0/-1 offset pair would + /// build a delta key nothing will ever present, and the seed would restore baselines under it + /// silently. Readers treat the three interval states distinctly: n > 0 measured; 0 + /// the unknowable marker, mapped to NULL (the point is absent, never 0.00); NULL a pre-V128 + /// row, falling back to the LAG-over-collection_time derivation those readers always used. The seed + /// consumes only rows whose offsets are NOT NULL for keys, and every row for the pass window. A + /// nullable no-default ADD COLUMN is catalog-only in PostgreSQL and TimescaleDB accepts it on a + /// compressed hypertable with continuous aggregates attached (verified live on 2.28.1 against + /// procedure_stats with its hourly/daily aggregates and query_stats with its hourly one), + /// so this stays instant on a multi-hundred-GB store. + /// + /// The offsets' semantics, stated here because this is where the next reader will look. + /// statement_start_offset and statement_end_offset are sys.dm_exec_query_stats's + /// own columns: the statement's position inside its batch text in BYTES of the + /// nvarchar text, not characters — so slicing the text at them divides by two (the + /// collector's SUBSTRING(st.text, (statement_start_offset / 2) + 1, …)), and a reader who + /// forgets the Unicode factor lands halfway into the wrong statement. statement_end_offset = -1 + /// means "to the end of the batch"; (0, -1) is the whole batch. They are stored + /// verbatim as the DMV reports them, -1 included and never normalized to a length, + /// because the collector's delta key is $"{sql_handle}:{start}:{end}:{plan_handle}" over the raw + /// ints and the seed has to spell the same string byte for byte. + /// + /// The PostgreSQL pair's columns are added in TWO places and both are required — the V101 + /// rule. A store's tables come from one of two texts depending on when it was created: a fresh store + /// builds every table from V1's generated schema, walked from the collector catalog, while a store that + /// predates V63/V64 has whatever those rungs built. So the pg_wait_stats and + /// pg_statement_stats CREATE TABLE rungs gain the column for the population that first meets them + /// (PgSchemaGeneratorTests enforces the rung text against the generator, column for column) and + /// THIS rung's ALTER carries the existing one. Neither is redundant: the CREATE is IF NOT EXISTS + /// and never re-runs on a store that already has the table, and the ALTER is ADD COLUMN IF NOT + /// EXISTS and is a no-op wherever the column already exists. The three SQL Server tables need no + /// second site: they are generated at V1 and only ever ALTERed. + /// + /// Two view treatments. v_memory_grant_stats is a SELECT * passthrough + /// and is refreshed here for the V14/V80/V81/V127 reason (Postgres freezes the column list at CREATE; + /// appending is the one alteration CREATE OR REPLACE VIEW permits). procedure_stats, + /// pg_wait_stats and pg_statement_stats have no v_* view (their readers hit the + /// base table; PgSchemaGenerator.AllPassthroughViews pins the set). v_query_stats is + /// the #1767 payload-RESOLVING view, not a passthrough, and the generator emits payload columns BEFORE + /// the trailing digest columns, so the two offsets land mid-list — an alteration + /// CREATE OR REPLACE VIEW refuses. Hence DROP VIEW here and the regenerated resolving + /// definition concatenated after this constant in the ladder entry, exactly as V51 and V121 did, with + /// V54Sql and GenerateQueryStatsPayloadColumnPreAdds() ahead of it so a store climbing + /// from below those rungs has every column the view names (MigrationLadderPins). Plain DROP, + /// no CASCADE: nothing persistent depends on the view. + /// + /// What this rung deliberately does NOT do. It does not touch + /// collect.wait_stats_baseline (V127's stated follow-up is a new aggregate under a new name — + /// an aggregate-plus-retirement operation, not a column, and not this rung). It does not backfill. It + /// does not change procedure_stats_hourly/_daily or query_stats_hourly: those sum + /// deltas, and a fabricated 0 adds nothing to a sum. + /// + private const string V128Sql = @" +ALTER TABLE collect.procedure_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.memory_grant_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.pg_wait_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; +ALTER TABLE collect.pg_statement_stats + ADD COLUMN IF NOT EXISTS sample_interval_seconds integer; + +/* The delta key's two halves that were never stored. BYTE offsets into the batch's nvarchar text + (a character position is offset / 2); statement_end_offset = -1 means ""to the end of the batch""; + stored raw, -1 included, because the key string carries the raw values. */ +ALTER TABLE collect.query_stats + ADD COLUMN IF NOT EXISTS statement_start_offset integer; +ALTER TABLE collect.query_stats + ADD COLUMN IF NOT EXISTS statement_end_offset integer; + +/* Postgres FREEZES a view's SELECT * column list at CREATE, so the passthrough would keep serving the + pre-V128 column list forever — the V14 lesson, restated by V80, V81 and V127. The other three interval + tables have no v_ view. */ +CREATE OR REPLACE VIEW collect.v_memory_grant_stats AS SELECT * FROM collect.memory_grant_stats; + +/* v_query_stats is the payload-RESOLVING view (#1767), and its two new columns land ahead of the digest + columns — mid-list, which CREATE OR REPLACE VIEW refuses. Dropped here; the ladder entry concatenates + the regenerated resolving definition after the pre-adds, the V51/V121 idiom. */ +DROP VIEW IF EXISTS collect.v_query_stats;"; + /// /// V2 — the service's observability store: the servers registry (upserted on every /// successful connect) and the per-run collection_log. Column names deliberately mirror @@ -1815,7 +2018,8 @@ CREATE TABLE IF NOT EXISTS collect.pg_wait_stats ( waits bigint, wait_time_us bigint, delta_waits bigint, - delta_wait_time_us bigint + delta_wait_time_us bigint, + sample_interval_seconds integer ); CREATE INDEX IF NOT EXISTS idx_pg_wait_stats_time @@ -1878,7 +2082,8 @@ CREATE TABLE IF NOT EXISTS collect.pg_statement_stats ( max_exec_peakmem_bytes bigint, delta_calls bigint, delta_total_exec_time_ms bigint, - delta_rows bigint + delta_rows bigint, + sample_interval_seconds integer ); CREATE INDEX IF NOT EXISTS idx_pg_statement_stats_time diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs b/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs index cc88a2be8..f07ad6d7c 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgTableTuning.cs @@ -38,6 +38,12 @@ namespace PerformanceMonitor.Darling.Storage; /// rollover, degrading the Index Only Scan back to heap fetches). Verified: the two panels that timed out at 15 s /// then ran in 139 ms / 514 ms with 0 heap fetches on vacuumed chunks. DarlingStoredPlanReader and ComposeCompiler /// are unchanged — correct as-is, they just needed these indexes + the vacuum state to perform. +/// +/// The alerting pass joined the composer here (#3573), for the same reason and with the same +/// EXPLAIN-backed shape: DarlingAlertReadAdapter.ForcePlanFailuresSql takes the fourth covering index +/// below, an Index Only Scan replacing a plan that streamed the whole fleet's two-hour slice of +/// query_store_stats to keep one server's rows. Its derivation is on the statement itself — including +/// why it is here and not a ladder rung, and why it is a plain CREATE INDEX and not the per-chunk form. /// public static class PgTableTuning { @@ -46,13 +52,30 @@ public static class PgTableTuning conversion uses. */ private const int SetupTimeoutSeconds = 300; + /// + /// The #3573 covering index's name, and the columns it carries in key-then-INCLUDE order. Exposed so the + /// pin (ForcePlanFailuresAccessPathTests) can hold the read's column references against THIS list + /// rather than against a second copy of the statement text, and so the live test can find the index by + /// name in pg_indexes. The first two are the key; the rest are INCLUDE. Every column the read + /// references must appear here or the Index Only Scan silently degrades to the heap plan it replaced. + /// + public const string ForcePlanFailuresIndexName = "idx_query_store_stats_server_time_forcing"; + + public static IReadOnlyList ForcePlanFailuresIndexColumns { get; } = new[] + { + "server_id", "collection_time", + "database_name", "query_id", "plan_id", "force_failure_count", "is_forced_plan", "plan_forcing_type", "last_force_failure_reason", + }; + /// /// The idempotent tuning statements, applied in order, each on its own command (failure-isolated). Three /// COVERING composer indexes (INCLUDE the aggregate columns the Procedures / Queries / Query Store measures /// SUM/AVG, for an Index Only Scan), three (server_id, handle/hash/id, collection_time DESC) lookup - /// indexes for the single-row analyze_*_plan reads (no INCLUDE — one heap fetch is cheap), then the per-table - /// autovacuum-insert override on exactly the four growing tables. Bare collect-qualified names; every - /// identifier is a compile-time constant, never user input, so interpolation is not a concern. + /// indexes for the single-row analyze_*_plan reads (no INCLUDE — one heap fetch is cheap), the #3573 + /// covering index for the alerting pass's forced-plan-failures read (INCLUDE exactly that read's columns, + /// for the same Index Only Scan), then the per-table autovacuum-insert override on exactly the four growing + /// tables. Bare collect-qualified names; every identifier is a compile-time constant, never user input, so + /// interpolation is not a concern. /// public static IReadOnlyList Statements { get; } = new[] { @@ -66,6 +89,78 @@ above. Bounded by raw retention (4 days of chunks), so the build is cheap on any "CREATE INDEX IF NOT EXISTS idx_query_stats_server_hash_time ON collect.query_stats (server_id, query_hash, collection_time DESC)", "CREATE INDEX IF NOT EXISTS idx_query_store_stats_query_hash ON collect.query_store_stats (query_hash, collection_time) INCLUDE (database_name, module_name, execution_count, avg_duration_us, max_duration_us, avg_cpu_time_us, max_cpu_time_us)", "CREATE INDEX IF NOT EXISTS idx_query_store_stats_server_db_query_plan_time ON collect.query_store_stats (server_id, database_name, query_id, plan_id, collection_time DESC)", + /* #3573: the alerting pass's forced-plan-failures read (DarlingAlertReadAdapter.ForcePlanFailuresSql, + WHERE server_id = $1 AND collection_time > $2, a two-hour window, per server, every 30 s pass) outgrew + its 10 s deadline on the largest production store: 1,744.9 ms cold when the deadline was derived over + ~6 GB of query_store_stats, 10.3 s excursions at 23 GB. The live plan named the mechanism — + + Index Scan Backward using _hyper_.._chunk_query_store_stats_collection_time_idx + Index Cond: (collection_time > now() - '02:00:00') + Filter: (server_id = ...) Rows Removed by Filter: 691,058 actual rows: 37,878 + Buffers: shared hit=54,664 read=2,643 + + — the read walks the ENTIRE fleet's two-hour slice through the TimescaleDB default time index and + discards 95% of it to keep one server. Forty-three servers deep, that is the whole slice re-read + forty-three times per pass cycle; warm it is 422 ms, and the excursions are the cold tail whenever + cache pressure evicts a slice ~4x the size it had on measurement day. + + THE INDEX THAT READ WANTED ALREADY EXISTED. V1's generated idx_query_store_stats_time is exactly + (server_id, collection_time), it is on the hypertable and on the very chunk in that plan, and the + planner declined it. Read from the production catalog: server_id's physical correlation is 0.022 + (43 servers interleaved by collection pass) while collection_time's is 0.99999, so the cost model + prices the composite's heap fetches as one random page per tuple — ~50K pages at random_page_cost + 4 — and the perfectly-correlated time index's 54,664-page stream wins on paper at 58,677. It loses + in fact by 11x: forced under random_page_cost = 1.1 the SAME statement took the existing composite + (Bitmap Index Scan, Index Cond on both columns) and touched 5,063 buffers (Heap Blocks: exact=5001) + instead of 57,307, because a server's rows land in one contiguous run per collection pass (~13 + rows a page) that the planner's single correlation statistic cannot see. A second plain composite, + however ordered, would be priced identically and ignored identically — which is why this is not + the (server_id, collection_time DESC) rung the issue first proposed. + + COVERING, so the choice stops depending on the cost model. With every column the read touches in + the key or INCLUDE, the plan is an Index Only Scan whose cost is the index pages for ONE server's + two hours and nothing else — no heap component to misprice, at any random_page_cost and at any + share of the fleet the busiest server grows into. Both uncompressed production chunks read + relallvisible = 100% of relpages (the insert-autovacuum override below is what keeps them there), + so heap fetches for visibility are the newest pass's pages at most. Measured on a PG18 / + TimescaleDB 2.28.1 rig seeded in the production's write pattern: the shipped statement went from + the identical time-index-plus-Filter plan at 1,514 buffers to Index Only Scan, Heap Fetches: 0, + 50 buffers. The INCLUDE list IS the read's column list, deliberately and exactly — a column added + to the read and not to this list silently degrades it back to the heap plan, so + ForcePlanFailuresAccessPathTests pins the two against each other. + + THE COST, stated rather than implied: INCLUDE disables btree deduplication, so this index is + ~86 bytes/row on the rig (V1's deduplicated composite is ~7) — roughly 0.7-0.9 GB per day-chunk on + the largest store's 8-16 M rows/day, against a 4.7-9.4 GB heap per chunk. It is self-limiting: + on 2.28.1 a compressed chunk's uncompressed relation is an empty shell, and CREATE INDEX on the + hypertable builds an 8 KB page for each one (measured: 4 compressed chunks at 8192 bytes each, + the 2 live chunks at 46 MB and 27 MB), so the footprint is the one or two uncompressed chunks and + the compression policy erases the rest a day later. That is also why the owner's "index only the + uncompressed/new chunks" needs no mechanism: it is what the engine does. New chunks inherit the + index at creation; a decompressed chunk fills it and recompression empties it (both measured). + + PLAIN CREATE INDEX, ONE TRANSACTION, deliberately. The build takes a ShareLock on the hypertable + root for its duration — reads proceed, every INSERT into any chunk queues behind it — which is + why this list runs BEFORE collectors start and why this statement belongs here rather than in the + ladder (whose applier wraps each rung in a transaction block, the one place the per-chunk form + below is refused outright). A cancel at SetupTimeoutSeconds rolls the whole build back and the next + start retries with nothing left behind. The per-chunk form, WITH (timescaledb.transaction_per_chunk), + was measured and rejected: it takes the same root ShareLock first, buying no write concurrency here, + and a cancel MID-build — exactly what the command timeout is — commits the chunks built so far, + leaves the parent index indisvalid = false and the live chunk unindexed, after which this very + idempotent statement reports "already exists, skipping" on every start forever. CREATE INDEX + CONCURRENTLY is refused on hypertables ("hypertables do not support concurrent index creation"). + A collision with the compression job on yesterday's chunk makes one of them wait for the other; + if that is this build past its budget, the cancel-and-retry above is the outcome. + + NOT random_page_cost, though it flipped the plan: at 1.1 the composite won by 53,095 to 58,677 for + a server holding ~8% of the fleet's rows, and the ratio scales with that share, so the next-busiest + store or the same one a month on flips back. A store-wide planner setting is also an owner's + ruling for every read at once, not a lane's fix for one. NOT a partial index WHERE is_forced_plan, + which would be a few MB: the read aggregates unforced rows on purpose (a plan's previous sighting + may be its unforced one), so that index needs the statement reshaped and its first-sighting + semantics changed — the cheaper long-run shape, deferred rather than smuggled in. */ + "CREATE INDEX IF NOT EXISTS " + ForcePlanFailuresIndexName + " ON collect.query_store_stats (server_id, collection_time DESC) INCLUDE (database_name, query_id, plan_id, force_failure_count, is_forced_plan, plan_forcing_type, last_force_failure_reason)", "ALTER TABLE collect.procedure_stats SET (" + InsertTuningOptions + ")", "ALTER TABLE collect.query_stats SET (" + InsertTuningOptions + ")", "ALTER TABLE collect.query_store_stats SET (" + InsertTuningOptions + ")", diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTrendRouting.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTrendRouting.cs index 4ac3166f0..dc3293087 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTrendRouting.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreTrendRouting.cs @@ -285,8 +285,12 @@ FROM united ) SELECT point_time AS collection_time, - CASE WHEN interval_seconds > 0 THEN total_duration_ms / interval_seconds ELSE 0 END AS duration_ms_per_second, - CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds ELSE 0 END AS executions_per_second + /* No ELSE: the first united point's LAG is NULL and its rate unknowable, so the rate is NULL — never a + fabricated 0 (#3541 A12). Shared by the MCP reader and the viewer, so both surfaces see the same + first bucket the same way: the MCP payload publishes it as an unrated point, the viewer's chart + reader skips it (a chart has nowhere to draw "unknown"). */ + CASE WHEN interval_seconds > 0 THEN total_duration_ms / interval_seconds END AS duration_ms_per_second, + CASE WHEN interval_seconds > 0 THEN CAST(total_executions AS DOUBLE PRECISION) / interval_seconds END AS executions_per_second FROM rated ORDER BY point_time """; diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index 91cd2df86..f50b18e9e 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 125; + public const int SchemaVersion = 128; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs b/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs index a762b9339..9771dc34b 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StoreSelfMetrics.cs @@ -34,12 +34,52 @@ namespace PerformanceMonitor.Darling.Storage; /// exact row count. The dims are the store's dominant payloads (measured: query_plan_dim alone was 101 GB /// of a 147 GB store, 69%) and invisible to every hypertable-shaped surface because they are deliberately /// PLAIN tables (see ); +/// one row per continuous aggregate (object_kind = 'continuous_aggregate', #3582): the same +/// three facts as a hypertable row, taken from the aggregate's MATERIALIZATION hypertable and reported +/// under the aggregate's user-facing view name. TimescaleDB-only, like the hypertable arm. These rows +/// exist because the hypertable arm structurally cannot see them: timescaledb_information.hypertables +/// ends AND ca.mat_hypertable_id IS NULL (verified against the 2.28.1 view definition), so a +/// materialization is never enumerated there under any name. Measured on the largest production store, +/// the twenty materializations were ~235 GiB of a 415 GiB database — 57% — and the inventory reported +/// none of it; +/// one row per product-owned plain table the walk would otherwise lump (object_kind = 'table', +/// #3582): collect.query_store_text (V74 stores statement text INLINE by design — 15 GiB on that +/// store), collect.query_store_plan_map and config.config_alert_log. Every store shape; +/// two catch-all rows that make the inventory RECONCILABLE against pg_database_size (#3582): +/// object_kind = 'other' sums every user-schema relation none of the rows above accounts for, with +/// the relation count, and object_kind = 'system' does the same for the system catalogs and +/// TimescaleDB's own bookkeeping schemas. With those two, every byte the database directory holds is +/// attributed to some row or is an honest residual, and get_store_metrics can state its own +/// coverage instead of answering "here is the store" for 38% of it; +/// one row carrying the OWNER's reading of timescaledb_information.job_history +/// (object_kind = 'job_history', #3574): how many history rows the sweep's role — which created the +/// jobs and is admitted by the view's ownership filter — saw over the fixed 24-hour evidence window, when +/// the newest one started, and how many jobs started a run in that window. TimescaleDB-only. This is the +/// managed-mode self-proof: the MCP host reads as the least-privilege mcp role, which that filter +/// shows NOTHING, so the tool's own count is zero by construction on every managed store and only this +/// row can make recording a measurement there; /// one summary row (object_kind = 'store'): pg_database_size plus the enabled-server /// count (the fleet reader's WHERE is_enabled registry predicate), so the per-server ingest rate — /// daily growth divided by servers, the number onboarding N primaries multiplies — is derivable from the /// stored series alone. /// /// +/// Which column means what, per kind. The table has one set of nullable columns and every kind +/// leaves the ones it has no use for NULL; the two catch-all kinds and the job_history kind reuse +/// columns whose names were chosen for hypertables and jobs, and the mapping is stated HERE, once, because +/// a raw read of the table has nothing else to go on. chunk_count on an other or +/// system row is the RELATION count the sum spans (the number of physical pieces, which is what it +/// means on a hypertable row too). On a job_history row: object_name is the role that +/// counted; row_count is the history rows with a start inside the window that the view showed that +/// role — NULL when the role was not admitted to every job's history, because a count the filter +/// truncated is not a count; total_runs is the jobs whose newest start falls inside the same window +/// (the unfiltered job_stats population half); schedule_interval_ms is the window's width; +/// and last_run_duration_ms is the AGE of the newest history row at the sweep — milliseconds from +/// its start to metric_time — NOT a run's duration. That last one is the single overload that bends +/// a column's name, and it is taken rather than a migration rung because the store has no timestamp +/// column besides metric_time, a rung was not free when this landed, and the MCP reader decodes it +/// back into an absolute instant before anyone reads it; a dedicated column is the clean follow-up. +/// /// Retention is ONE bounded DELETE inside the same sweep — deliberately no policy machinery. /// collect.store_metrics is a PLAIN table and must stay one: it is not in the collector catalog, so /// 's catalog-driven hypertable conversion and DarlingRetention's catalog @@ -74,7 +114,7 @@ public static class StoreSelfMetrics /// /// Per-statement command timeout for the sweep (#2317) — and, at the worker's call site, the /// budget for the WHOLE sweep via a linked CTS (see SweepStoreSelfMetricsAsync: this sweep is - /// awaited on the main loop, so five sequential per-statement timeouts must not stack). The + /// awaited on the main loop, so the sequential per-statement timeouts must not stack). The /// sizing queries call hypertable_detailed_size across every hypertable (whose inner /// hypertable_local_size is the frame the server log names when it cancels) and /// pg_database_size over the whole @@ -92,6 +132,17 @@ public static class StoreSelfMetrics /// prior year plus headroom. Enforced by the sweep's own DELETE, not a retention policy. public const int RetentionDays = 400; + /// The object_kind of the per-hypertable rows. Named for the reason + /// gives: the MCP reader partitions and sums by kind, and a drifted + /// spelling returns zero rows rather than an error. The value is part of the on-disk contract. + public const string HypertableObjectKind = "hypertable"; + + /// The object_kind of the two payload-dimension rows. See . + public const string DimensionObjectKind = "dimension"; + + /// The object_kind of the per-background-job rows (#2136). See . + public const string BackgroundJobObjectKind = "background_job"; + /// /// The per-hypertable rows — TimescaleDB stores only (the caller gates on the detected flag; the /// timescaledb_information views referenced here do not exist on plain PostgreSQL). @@ -99,14 +150,24 @@ public static class StoreSelfMetrics /// format('%I.%I', ...) from the catalog view's own rows — never user input. /// compressed_*_bytes are NULL for a hypertable with no compressed chunks yet. $1 metric_time /// (naive UTC, one value per run). + /// + /// What this arm cannot see, and why that is not a filter of ours (#3582). Nothing here + /// restricts the walk to collector tables — it enumerates every row the view returns — but the view + /// itself excludes two classes of hypertable: the internal compressed hypertables + /// (compression_state <> 2) and every continuous aggregate's materialization + /// (ca.mat_hypertable_id IS NULL). The first is right — hypertable_detailed_size on a + /// user hypertable already includes its compressed chunk relations, byte-exact (verified on 2.28.1 + /// against pg_total_relation_size over root + chunks + compressed chunks). The second is the gap + /// closes: a materialization is a hypertable that holds + /// real bytes and is enumerated by this view under NO name, internal or otherwise. /// - public const string HypertableInsertSql = @" + public const string HypertableInsertSql = $@" INSERT INTO collect.store_metrics (metric_time, object_name, object_kind, total_bytes, compressed_before_bytes, compressed_after_bytes, chunk_count) SELECT $1, h.hypertable_name, - 'hypertable', + '{HypertableObjectKind}', s.total_bytes, c.before_bytes, c.after_bytes, @@ -123,6 +184,69 @@ LEFT JOIN LATERAL ( FROM chunk_compression_stats(format('%I.%I', h.hypertable_schema, h.hypertable_name)::regclass) ) c ON true"; + /// The object_kind of the per-continuous-aggregate rows (#3582). See + /// for why it is a const. + public const string ContinuousAggregateObjectKind = "continuous_aggregate"; + + /// + /// The per-continuous-aggregate rows (#3582) — TimescaleDB stores only, like the hypertable arm. One + /// row per aggregate in timescaledb_information.continuous_aggregates, under the aggregate's + /// USER-FACING view name (view_name, bare, the way hypertable rows carry hypertable_name), + /// sized through its materialization hypertable: hypertable_detailed_size on + /// materialization_hypertable_schema.materialization_hypertable_name for the total, + /// chunk_compression_stats on the same regclass for the pre/post-compression bytes, and the + /// materialization's chunk count from timescaledb_information.chunks. + /// + /// Why the chunk count is a third lateral and not a column of the second. + /// chunk_compression_stats returns ZERO rows for a hypertable whose compression is not enabled + /// (measured on 2.28.1: a two-chunk materialization with compression off yields no rows at all, and + /// two rows the moment it is enabled), so a count(*) over it would report every uncompressed + /// aggregate as chunkless. The chunks view lists every non-OSM chunk of every hypertable, + /// materializations included, which is also how the hypertables view computes + /// num_chunks for the sibling rows — so the two kinds count chunks the same way. + /// + /// What this row deliberately does NOT carry. compression_enabled and whether a + /// refresh, compression or retention policy exists for the aggregate — the three facts the sibling + /// investigation (#3581) had to assemble by hand — are STATE, not series: they change when an operator + /// changes them and at no other time, and the table has no column that could hold a bool without + /// bending a byte or count column's meaning. They are read LIVE by get_store_metrics from the + /// same two catalog views (DarlingStoreMetricsReader.ContinuousAggregateStateSql), the #2813 + /// precedent for a catalog fact the series does not carry, and joined to these rows by view name. + /// That keeps this table's schema where it is: no migration rung for three flags. + /// + /// Bare view_name has the same exposure the hypertable rows already accept: two aggregates + /// of the same name in different schemas would share one series. Every aggregate this product creates + /// lives in collect. $1 metric_time. + /// + public const string ContinuousAggregateInsertSql = $@" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, total_bytes, compressed_before_bytes, compressed_after_bytes, chunk_count) +SELECT + $1, + ca.view_name, + '{ContinuousAggregateObjectKind}', + s.total_bytes, + c.before_bytes, + c.after_bytes, + n.chunk_count +FROM timescaledb_information.continuous_aggregates ca +LEFT JOIN LATERAL ( + SELECT sum(total_bytes)::bigint AS total_bytes + FROM hypertable_detailed_size(format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass) +) s ON true +LEFT JOIN LATERAL ( + SELECT + sum(before_compression_total_bytes)::bigint AS before_bytes, + sum(after_compression_total_bytes)::bigint AS after_bytes + FROM chunk_compression_stats(format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass) +) c ON true +LEFT JOIN LATERAL ( + SELECT count(*)::integer AS chunk_count + FROM timescaledb_information.chunks ch + WHERE ch.hypertable_schema = ca.materialization_hypertable_schema + AND ch.hypertable_name = ca.materialization_hypertable_name +) n ON true"; + /// /// The background-job rows (#2136) — TimescaleDB stores only, like the hypertable arm (the /// timescaledb_information views do not exist on plain PostgreSQL). The store's own background jobs @@ -138,13 +262,13 @@ FROM chunk_compression_stats(format('%I.%I', h.hypertable_schema, h.hypertable_n /// stable for a job's lifetime, so per-job series continuity holds. schedule_interval_ms /// rides along so "duration vs cadence" — the honest tripwire — is one division. $1 metric_time. /// - public const string BackgroundJobInsertSql = @" + public const string BackgroundJobInsertSql = $@" INSERT INTO collect.store_metrics (metric_time, object_name, object_kind, last_run_duration_ms, schedule_interval_ms, total_runs, total_failures) SELECT $1, j.proc_name || coalesce(' ' || j.hypertable_name, '') || ' [' || j.job_id || ']', - 'background_job', + '{BackgroundJobObjectKind}', (EXTRACT(EPOCH FROM js.last_run_duration) * 1000)::bigint, (EXTRACT(EPOCH FROM j.schedule_interval) * 1000)::bigint, js.total_runs, @@ -166,25 +290,351 @@ INSERT INTO collect.store_metrics SELECT $1, '{PayloadDimensions.QueryTextDimTable}', - 'dimension', + '{DimensionObjectKind}', pg_total_relation_size('collect.{PayloadDimensions.QueryTextDimTable}'), (SELECT count(*) FROM collect.{PayloadDimensions.QueryTextDimTable}) UNION ALL SELECT $1, '{PayloadDimensions.QueryPlanDimTable}', - 'dimension', + '{DimensionObjectKind}', pg_total_relation_size('collect.{PayloadDimensions.QueryPlanDimTable}'), (SELECT count(*) FROM collect.{PayloadDimensions.QueryPlanDimTable})"; + /// The object_kind of the named plain-table rows (#3582). See + /// for why it is a const. + public const string TableObjectKind = "table"; + /// - /// The object_kind the whole-store summary row carries, named ONCE because the string now has - /// six consumers that must never disagree: this sweep writes it (), the + /// config.config_alert_log, schema-qualified, for and the + /// un-enumerated predicate. The alert log has no owning store class with a name constant the way the + /// two Query Store tables do (QueryStoreTextStore.TableName, QueryStorePlanMap.TableName): + /// its writers reach it through the bare name and the session search_path. The V8 schema split + /// placed it in config (PgSchemaGenerator.ConfigTables), which is where it is sized. + /// + public const string AlertLogTable = "config.config_alert_log"; + + /// + /// The named plain-table rows (#3582) — every store shape, like the dimension rows, and in the same + /// shape: pg_total_relation_size (heap + indexes + TOAST) and the exact row count. Three + /// product-owned tables that are neither hypertables nor payload dimensions and were therefore + /// invisible to the inventory: collect.query_store_text, which V74 made an INLINE text store by + /// design (Query Store already de-duplicates statement text one row per statement per database, so + /// there was nothing for a digest dimension to squeeze) and which was 15 GiB on the largest production + /// store; collect.query_store_plan_map, the V72 plan-id-to-digest map; and + /// config.config_alert_log, the alert history and dismissals. They are stable, named, and the + /// product knows them, so the inventory knows them by name instead of lumping them into + /// . + /// + /// Schema-qualified object_name, unlike every other kind. The hypertable, + /// aggregate and dimension rows are bare because their catalogs name them bare and every one lives in + /// collect. This kind spans two schemas, and the un-enumerated census it shares a population + /// with (DarlingStoreMetricsReader.LargestUnenumeratedSql) names relations + /// schema.relation, so a table that moves from that list to this one keeps its name. + /// + /// row_count here is the planner's reltuples ESTIMATE, not a scan, and the two + /// kinds differ on purpose. The dimension arm counts exactly because a dim's heap is small — its + /// bytes live in TOAST, which a count never reads. query_store_text is the opposite shape: V74 + /// stores statement text INLINE, most statements fit a heap page, so the heap IS the 15 GiB and an + /// exact count(*) would be a 15 GiB read every hour, on the same store the CAGG refresh convoy + /// is running on, for a figure whose job is "roughly how many statements have text". reltuples + /// is refreshed by every autovacuum and ANALYZE, is exact enough for that job, and costs one catalog + /// row. It is -1 for a table never vacuumed or analysed (PostgreSQL 14+), which maps to NULL + /// rather than to a count of minus one. The same estimate is used for all three so the kind means one + /// thing. $1 metric_time. + /// + public const string TableInsertSql = $@" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, total_bytes, row_count) +SELECT + $1, + '{QueryStoreTextStore.TableName}', + '{TableObjectKind}', + pg_total_relation_size('{QueryStoreTextStore.TableName}'), + (SELECT CASE WHEN c.reltuples >= 0 THEN c.reltuples::bigint END FROM pg_class c WHERE c.oid = '{QueryStoreTextStore.TableName}'::regclass) +UNION ALL +SELECT + $1, + '{QueryStorePlanMap.TableName}', + '{TableObjectKind}', + pg_total_relation_size('{QueryStorePlanMap.TableName}'), + (SELECT CASE WHEN c.reltuples >= 0 THEN c.reltuples::bigint END FROM pg_class c WHERE c.oid = '{QueryStorePlanMap.TableName}'::regclass) +UNION ALL +SELECT + $1, + '{AlertLogTable}', + '{TableObjectKind}', + pg_total_relation_size('{AlertLogTable}'), + (SELECT CASE WHEN c.reltuples >= 0 THEN c.reltuples::bigint END FROM pg_class c WHERE c.oid = '{AlertLogTable}'::regclass)"; + + /// The object_kind of the user-schema catch-all row (#3582): every relation in a + /// non-system schema that no named row accounts for. See for why + /// it is a const. + public const string OtherObjectKind = "other"; + + /// The object_name of the one row. A singleton per sweep, + /// so the name is a fixed label rather than a relation's name. + public const string OtherObjectName = "un-enumerated relations"; + + /// The object_kind of the system catch-all row (#3582): pg_catalog, + /// information_schema, and TimescaleDB's own schemas minus the chunk relations the hypertable + /// and aggregate rows already size. See for why it is a const. + public const string SystemObjectKind = "system"; + + /// The object_name of the one row. + public const string SystemObjectName = "catalog and TimescaleDB internals"; + + /// + /// The relations the catch-all census considers at all, as a fragment shared by the two sweep variants + /// and the MCP reader's live top-N (#3582): ordinary tables, materialized views, partitioned parents + /// and sequences, sized with pg_total_relation_size so their indexes and TOAST tables ride along + /// under the parent and are never counted twice (which is why relkind i, I and + /// t are not listed). NOT c.relisshared because the shared catalogs + /// (pg_authid, pg_database, ...) live in the cluster's global/ directory and are + /// NOT inside pg_database_size — measured on a fresh 2.28.1 rig, summing them made the census + /// EXCEED the database by 512 KiB, and a reconciliation that starts over 100% is wrong in the direction + /// nobody checks. Aliases c (pg_class) and n (pg_namespace) are the + /// contract every consumer of this fragment supplies. + /// + public const string CensusRelationPredicateSql = @"c.relkind IN ('r', 'm', 'p', 'S') +AND NOT c.relisshared"; + + /// + /// Which side of the user/system line a relation falls on, as a fragment over n.nspname (#3582). + /// System: PostgreSQL's own two schemas, and every schema TimescaleDB creates — the _timescaledb_ + /// family (_catalog, _config, _cache, _internal, _functions) plus + /// its two information schemas. _timescaledb_internal is where chunks live, but chunks are + /// removed from the census before this predicate is applied (), so + /// what remains of it here is TimescaleDB's bookkeeping: bgw_job_stat_history (the + /// job_history table itself), the compressed hypertables' empty roots, and any chunk relation + /// whose catalog row is gone — which is exactly the class of residue a reconciliation should count + /// rather than lose. pg_toast is not named because TOAST relations are relkind = 't' and + /// already excluded by ; their bytes arrive through their + /// parents. starts_with rather than LIKE so the underscore is a character and not a + /// wildcard, with no escape-string dialect to get wrong. + /// + public const string SystemSchemaPredicateSql = + @"(n.nspname IN ('pg_catalog', 'information_schema', 'timescaledb_information', 'timescaledb_experimental') + OR starts_with(n.nspname, '_timescaledb_'))"; + + /// + /// The relations some NAMED row already sizes, every store shape (#3582): the two payload dimensions + /// and the three named plain tables. A relation matched here is never in a catch-all row, or the + /// reconciliation would count it twice. Every name is the SAME compile-time constant the INSERT arm + /// interpolates — the dims through , the tables through their + /// schema-qualified owners' constants — so the census and the rows it excludes cannot drift apart. That + /// is why the comparison is on the concatenated schema.relation rather than on a + /// (schema, relation) tuple: the table constants are compound ("collect.query_store_text") + /// and cannot be split at compile time, and a hand-typed tuple beside them would be exactly the copy + /// this constant exists not to have (review catch on the first cut, which had three). No product + /// relation name contains a dot, so the concatenation is unambiguous. Aliases c and n as + /// on . + /// + public const string NamedRelationPredicateSql = $@"(n.nspname || '.' || c.relname) IN ( + 'collect.{PayloadDimensions.QueryTextDimTable}', + 'collect.{PayloadDimensions.QueryPlanDimTable}', + '{QueryStoreTextStore.TableName}', + '{QueryStorePlanMap.TableName}', + '{AlertLogTable}')"; + + /// + /// The relations the TimescaleDB rows already size, as a fragment (#3582): every hypertable root the + /// hypertables view lists (its chunks and compressed chunks are inside that row's + /// hypertable_detailed_size), every materialization root the continuous_aggregates view + /// names (same), and every chunk relation the chunk catalog knows — which is what keeps the census from + /// re-summing the ~40,000 chunk relations a production store holds and lets the catch-all statement + /// cost a hash anti-join over pg_class rather than a pg_total_relation_size per chunk. + /// Joined by (schema_name, table_name) rather than by casting the catalog's names to + /// regclass: a catalog row whose relation is gone would make the cast RAISE and fail the sweep, + /// where a name join simply matches nothing. The internal compressed hypertables' roots + /// (compression_state = 2, _compressed_hypertable_N) are deliberately NOT excluded here: + /// no named row sizes them, so they fall to the row, which is where an + /// unattributed byte belongs. TimescaleDB-only; the plain-PostgreSQL variant omits it. + /// + public const string TimescaleInventoriedPredicateSql = @"NOT EXISTS ( + SELECT 1 FROM timescaledb_information.hypertables h + WHERE h.hypertable_schema = n.nspname AND h.hypertable_name = c.relname) +AND NOT EXISTS ( + SELECT 1 FROM timescaledb_information.continuous_aggregates ca + WHERE ca.materialization_hypertable_schema = n.nspname AND ca.materialization_hypertable_name = c.relname) +AND NOT EXISTS ( + SELECT 1 FROM _timescaledb_catalog.chunk ch + WHERE ch.schema_name = n.nspname AND ch.table_name = c.relname)"; + + /// + /// The two catch-all rows (#3582), TimescaleDB variant: one INSERT ... SELECT over a census CTE + /// of every relation no named row accounts for, split by into + /// the row (user schemas — the product's own registry and config tables, + /// store_metrics itself, and anything an operator added: the population the product might want + /// to name next) and the row (the catalogs, which on a chunk-heavy store + /// are not small: pg_attribute, pg_statistic and pg_class grow with the chunk + /// count). total_bytes is the pg_total_relation_size sum, chunk_count the relation + /// count. coalesce(..., 0) because an empty bucket is a ZERO, not a missing reading — the + /// reconciliation needs both rows present every sweep so an absent row means the statement did not + /// run, never that there was nothing to count. + /// + /// Why two rows and not one. The issue asked for other; system is what + /// makes other readable. Folded together, the catalogs' bytes would inflate the count of + /// "relations the product should know by name" with sixty pg_catalog tables no product would + /// ever name; left out, they would surface as an unreconciled gap of a percent or more on every store, + /// every hour — a false finding of exactly the shape this work exists to stop. Attributing them to a + /// row of their own is what lets the residual gap be small enough to mean something. + /// + /// The statement touches pg_class once and sizes only the relations that survive the + /// anti-joins — a few hundred at most — so its cost does not follow the chunk count. $1 metric_time. + /// + public const string UnenumeratedInsertSql = $@" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, total_bytes, chunk_count) +WITH census AS ( + SELECT + c.oid, + {SystemSchemaPredicateSql} AS is_system + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE {CensusRelationPredicateSql} + AND NOT {NamedRelationPredicateSql} + AND {TimescaleInventoriedPredicateSql} +) +SELECT $1, '{OtherObjectName}', '{OtherObjectKind}', coalesce(sum(pg_total_relation_size(oid)), 0)::bigint, count(*)::integer +FROM census WHERE NOT is_system +UNION ALL +SELECT $1, '{SystemObjectName}', '{SystemObjectKind}', coalesce(sum(pg_total_relation_size(oid)), 0)::bigint, count(*)::integer +FROM census WHERE is_system"; + + /// + /// The two catch-all rows, plain-PostgreSQL variant (#3582): + /// minus , because the TimescaleDB catalogs it names do + /// not exist there. On such a store the collector tables are ordinary tables and no row enumerates + /// them, so the row holds most of the database and the reader's coverage + /// note says so in those words — an honest low number, not a fault. Per-collector-table rows for plain + /// PostgreSQL would be the natural extension and are not taken here. $1 metric_time. + /// + public const string UnenumeratedPlainInsertSql = $@" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, total_bytes, chunk_count) +WITH census AS ( + SELECT + c.oid, + {SystemSchemaPredicateSql} AS is_system + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE {CensusRelationPredicateSql} + AND NOT {NamedRelationPredicateSql} +) +SELECT $1, '{OtherObjectName}', '{OtherObjectKind}', coalesce(sum(pg_total_relation_size(oid)), 0)::bigint, count(*)::integer +FROM census WHERE NOT is_system +UNION ALL +SELECT $1, '{SystemObjectName}', '{SystemObjectKind}', coalesce(sum(pg_total_relation_size(oid)), 0)::bigint, count(*)::integer +FROM census WHERE is_system"; + + /// + /// The object_kind of the owner's job_history evidence row (#3574). See + /// for why it is a const; the MCP reader filters on it to find the + /// one row that can make recording a measurement in managed mode. + /// + public const string JobHistoryObjectKind = "job_history"; + + /// + /// The evidence read behind recording (#3574), named ONCE here because it now has two consumers + /// that must never disagree about what they count: the MCP reader runs it as the connection asking + /// (DarlingStoreMetricsReader.JobHistoryEvidenceSql is this string), and + /// embeds it verbatim to run it as the sweep's OWNER role and persist + /// the answer. The full reasoning — the view's ownership filter, why the read evaluates the predicate + /// for its own reader instead of counting and assuming, the population half from the unfiltered + /// job_stats, the fixed 24-hour window, and the timestamptz bind — lives on the reader's + /// alias, beside the code that interprets it. It lives HERE for the reason + /// does: the Storage project cannot reference the Service + /// project, and a retyped copy of a nine-column predicate would drift silently. + /// + /// $1 is the window start, timestamptz, bound with Kind = Utc — the columns it is + /// compared to are TimescaleDB's own TIMESTAMPTZ, the inverse of the store's naive-UTC rule. + /// + public const string JobHistoryEvidenceSql = @" +SELECT + current_user::text AS reader_role, + pg_has_role( + current_user, + (SELECT pg_get_userbyid(datdba) FROM pg_database WHERE datname = current_database()), + 'MEMBER') IS TRUE AS reader_is_database_owner_member, + (SELECT count(*) FROM timescaledb_information.jobs) AS job_count, + (SELECT count(*) + FROM timescaledb_information.jobs AS j + WHERE pg_has_role(current_user, j.owner, 'MEMBER') IS TRUE) AS owner_member_job_count, + (SELECT count(*) + FROM timescaledb_information.job_history AS h + WHERE h.start_time >= $1) AS rows_observed, + (SELECT max(h.start_time) FROM timescaledb_information.job_history AS h) AS newest_row_at, + (SELECT count(*) + FROM timescaledb_information.job_stats AS js + WHERE js.last_run_started_at >= $1) AS jobs_run_in_window, + (SELECT max(NULLIF(js.last_run_started_at, '-infinity'::timestamptz)) + FROM timescaledb_information.job_stats AS js) AS newest_run_started_at"; + + /// The evidence window counts over, in hours — shared with + /// the MCP reader for the same reason as the SQL. Fixed, not the tool's days_back: the question + /// is whether the instrument is writing NOW, every job this product schedules runs at least daily, and + /// the view's own retention policy trims rows after a month. + public const int JobHistoryEvidenceWindowHours = 24; + + /// + /// The owner's job_history evidence row (#3574) — TimescaleDB stores only. The sweep runs on + /// the worker's OWNER pool: the role that ran the migrations and created every policy job, and so the + /// role timescaledb_information.job_history's ownership filter admits. It runs + /// as that role and persists what it saw, so that + /// get_store_metrics — which in managed mode reads as the mcp role, a member of neither + /// the database owner nor any job's owner, and is therefore shown NOTHING by construction — can put + /// the owner's count beside its own verdict and say where the number came from. Without this row the + /// managed block is honest but blind: visibility: None, rows_observed: 0, and + /// recording a GUC echo forever. + /// + /// The column mapping is on the class summary; the load-bearing parts are these. + /// row_count is the owner's rows_observed ONLY when the view admits this role to every + /// job's history — the same two tests the reader's Visibility derives All from, + /// evaluated in SQL: database-owner membership, or membership in every job's owner with at least one + /// job to be a member of. Otherwise NULL, because a count the filter truncated is not a count, and the + /// reader must be able to tell "the owner saw zero" from "the sweep's role could not see". Which role + /// that was is object_name. total_runs is the unfiltered population half. + /// last_run_duration_ms is the newest row's AGE at the sweep ($2 - newest_row_at, in + /// milliseconds; NULL when the role saw no row ever), which the reader turns back into an instant. The + /// window width rides in schedule_interval_ms so the row carries its own denominator — computed + /// from the two binds ($2 - $1) rather than restated as a literal, so the row records the window + /// it actually counted over. + /// + /// Two parameters, and $1 is NOT metric_time — the one arm in this sweep where that is so. + /// The embedded evidence SELECT is shared verbatim with the MCP reader and binds its window start as + /// $1; renumbering it for this arm would mean two copies of the predicate, which is the drift this + /// constant exists to prevent. So $1 is the window start (timestamptz, Kind = Utc) and + /// $2 is metric_time (naive UTC). $2::timestamp AT TIME ZONE 'UTC' converts the naive stamp to + /// the instant it denotes so the age subtraction is between two timestamptz values. + /// + public const string JobHistoryInsertSql = $@" +INSERT INTO collect.store_metrics + (metric_time, object_name, object_kind, row_count, total_runs, schedule_interval_ms, last_run_duration_ms) +SELECT + $2, + e.reader_role, + '{JobHistoryObjectKind}', + CASE + WHEN e.reader_is_database_owner_member + OR (e.job_count > 0 AND e.owner_member_job_count >= e.job_count) + THEN e.rows_observed + END, + e.jobs_run_in_window, + (EXTRACT(EPOCH FROM (($2::timestamp AT TIME ZONE 'UTC') - $1)) * 1000)::bigint, + (EXTRACT(EPOCH FROM (($2::timestamp AT TIME ZONE 'UTC') - e.newest_row_at)) * 1000)::bigint +FROM ({JobHistoryEvidenceSql} +) AS e"; + + /// + /// The object_kind the whole-store summary row carries, named ONCE because the string has + /// several consumers that must never disagree: this sweep writes it (), the /// disk-pressure check filters on it (), and - /// DarlingMcpStoreMetricsTools partitions its response by it in four places — the store summary - /// is the one row those reads must separate from the per-hypertable and per-dimension rows. + /// DarlingMcpStoreMetricsTools partitions its response by it — the store summary is the one row + /// those reads must separate from the per-object rows, and since #3582 the denominator every other + /// kind's bytes are reconciled against. /// - /// Why a const and not six literals. A reader filtering on a kind the writer stopped + /// Why a const and not literals. A reader filtering on a kind the writer stopped /// writing returns ZERO ROWS, not an error, and every consumer here maps zero rows to a null or an /// omitted section. So a drifted spelling is indistinguishable from a store that has not swept yet — /// the honest-empty trap, on a value the shipped store already holds 400 days of. The value itself is @@ -274,8 +724,10 @@ INSERT INTO collect.store_metrics /// Why that is accepted here when #3199 rejected the same shape of argument. The growth /// axis is different, and the axis is what made pg_database_size unbounded. The tied group is /// one sweep's output — hypertable rows (70 today), one - /// row per Timescale background job, two dimension rows and this one — so it tracks the COLLECTOR - /// CATALOG, a product constant that moves only when a migration rung adds a hypertable, and every + /// row per continuous aggregate, one row per Timescale background job, two dimension rows, three named + /// plain-table rows, the two catch-all rows, the owner's job_history row and this one — so it + /// tracks the COLLECTOR CATALOG, a product constant that moves only when a migration rung adds a + /// hypertable or an aggregate, and every /// element is a narrow row on a plain table. pg_database_size tracked the store's file count, /// which retention span and ingest rate grow without anything choosing to. A composite /// (object_kind, metric_time DESC) index would make it exact and is the right follow-up; it @@ -303,10 +755,20 @@ DELETE FROM collect.store_metrics WHERE metric_time < $1"; /// - /// One self-metrics run: the hypertable rows (only when — the - /// worker's cached detection), the dimension rows, the store summary - /// row, then the retention DELETE, all stamped with one . Returns the number - /// of metric rows written (the caller logs it at Debug). + /// One self-metrics run: the hypertable, continuous-aggregate, background-job and owner + /// job_history rows (only when — the worker's cached + /// detection), the dimension rows, the named plain-table rows, the two + /// catch-all rows (the TimescaleDB or plain variant, by the same flag), the store summary row, then the + /// retention DELETE, all stamped with one . Returns the number of metric rows + /// written (the caller logs it at Debug). + /// + /// Order matters for the reconciliation, and it is stated rather than relied on. Every + /// sizing statement runs before 's pg_database_size, so the + /// database figure is the NEWEST reading of the run and the per-object rows are at most one sweep's + /// duration older; ingest and retention keep moving underneath, so the residual the MCP reader + /// computes is expected to be small and non-zero, never exactly zero. The catch-all rows run LAST + /// among the sizing statements so the population they sum is the one the named rows were taken from. + /// documents the other property this order carries. /// public static async Task SweepAsync( NpgsqlConnection connection, @@ -330,9 +792,28 @@ public static async Task SweepAsync( hypertables.Parameters.AddWithValue(metricTime); written += await hypertables.ExecuteNonQueryAsync(cancellationToken); + /* #3582: the materializations the hypertables view structurally omits. */ + using var aggregates = new NpgsqlCommand(ContinuousAggregateInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }; + aggregates.Parameters.AddWithValue(metricTime); + written += await aggregates.ExecuteNonQueryAsync(cancellationToken); + using var jobs = new NpgsqlCommand(BackgroundJobInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }; jobs.Parameters.AddWithValue(metricTime); written += await jobs.ExecuteNonQueryAsync(cancellationToken); + + /* #3574: the owner's own reading of job_history, for the managed-mode reader that cannot take + one. $1 is the evidence window start as an EXPLICIT timestamptz with Kind = Utc — the same + bind the MCP reader makes and the inverse of every other bind in this sweep, because the + columns it is compared to are TimescaleDB's own TIMESTAMPTZ (the reader's + JobHistoryEvidenceSql paragraph has the full reasoning). $2 is the sweep's naive stamp. */ + using var history = new NpgsqlCommand(JobHistoryInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }; + history.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.TimestampTz, + Value = DateTime.SpecifyKind(utcNow.AddHours(-JobHistoryEvidenceWindowHours), DateTimeKind.Utc), + }); + history.Parameters.AddWithValue(metricTime); + written += await history.ExecuteNonQueryAsync(cancellationToken); } using (var dimensions = new NpgsqlCommand(DimensionInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }) @@ -341,6 +822,25 @@ public static async Task SweepAsync( written += await dimensions.ExecuteNonQueryAsync(cancellationToken); } + /* #3582: the product-owned plain tables the inventory knows by name. */ + using (var tables = new NpgsqlCommand(TableInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }) + { + tables.Parameters.AddWithValue(metricTime); + written += await tables.ExecuteNonQueryAsync(cancellationToken); + } + + /* #3582: everything else, attributed to a row so the total reconciles. The TimescaleDB variant + removes chunk relations and the roots the hypertable/aggregate rows already size; the plain + variant cannot name those catalogs and has nothing to remove. Last of the sizing statements on + purpose — see the summary. */ + using (var unenumerated = new NpgsqlCommand( + timescaleAvailable ? UnenumeratedInsertSql : UnenumeratedPlainInsertSql, connection) + { CommandTimeout = SweepTimeoutSeconds }) + { + unenumerated.Parameters.AddWithValue(metricTime); + written += await unenumerated.ExecuteNonQueryAsync(cancellationToken); + } + using (var store = new NpgsqlCommand(StoreInsertSql, connection) { CommandTimeout = SweepTimeoutSeconds }) { store.Parameters.AddWithValue(metricTime); diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index fd84c8622..e4b4f6670 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -984,6 +984,36 @@ public static readonly (string CreateSql, string View)[] BaselineAggregates = (CreateMemoryBaselineSql, MemoryBaselineView), }; + /// + /// The seven DAILY continuous aggregates in CREATION order — every one of them hierarchical, sourced from + /// an hourly aggregate rather than from raw, which is why the ensure sweep creates them after + /// and why the order inside this list is load-bearing too: the day-grain + /// corrected daily (#1869) is THREE levels deep (L1 → L2 → + /// ), so L2 must precede its own child. The corrected DAILY + /// is L1's SIBLING rather than the corrected hourly's child — an identity-width hierarchical aggregate is a + /// leaf (see ) — so it carries no ordering requirement + /// against the corrected hourly and simply follows the whole hourly tier like the others. + /// + /// Hoisted out of (#3581) for the reason + /// was hoisted at #3012: the aggregate-compression ensure needs the same + /// list to know which materializations it owns and which tier's compress_after each one takes, and + /// a second hand-kept copy of seven names is a copy that drifts. The ensure sweep, the compression + /// registry () and the tests now read ONE list. + /// + public static readonly (string CreateSql, string View)[] DailyAggregates = + { + (CreateQueryStatsDailySql, QueryStatsDailyView), + (CreateProcedureStatsDailySql, ProcedureStatsDailyView), + (CreateQueryStoreStatsDailySql, QueryStoreStatsDailyView), + (CreateQueryStoreStatsCorrectedDailySql, QueryStoreStatsCorrectedDailyView), + (CreateQueryStatsDbDailySql, QueryStatsDbDailyView), + /* The DAY-grain corrected daily (#1869), THREE levels deep: L1 (an hourly) -> L2 interval_daily -> + daygrain_daily. Both must follow L1 and L2 must precede its own child, which this ordered list + gives — the same requirement the daily tier has, one level longer. */ + (CreateQueryStoreStatsIntervalDailySql, QueryStoreStatsIntervalDailyView), + (CreateQueryStoreStatsDayGrainDailySql, QueryStoreStatsDayGrainDailyView), + }; + public const string QueryStatsHourlyView = "query_stats_hourly"; /// 's procedure_stats sibling. @@ -1372,9 +1402,32 @@ hourly grain the residual is irreducible — an interval genuinely collected in /// (): nothing reads it, so it only has to outlive raw for the /// arming gate and outlive its consumers' refresh windows ( for /// the corrected hourly, for the corrected daily). + /// + /// create_group_indexes = false (#3597) — no per-column index on the materialization, + /// because nothing reads one and every refresh paid for eleven. TimescaleDB's default builds one btree + /// per GROUP BY column, (column, bucket DESC), on a continuous aggregate's materialization hypertable; + /// here that was eleven of them beside the bucket index — twelve indexes on the one materialization in this + /// file whose row count is near-raw. Nothing reads this relation by any of those columns: its three + /// consumers (, + /// , ) + /// refresh over it by bucket range, the coverage probe and the arming gate read min(bucket), + /// and retention drops whole chunks. What the eleven indexes DID do was tax the refresh: the hourly policy + /// re-materializes a bucket by DELETE + INSERT, and every inserted row cost twelve index inserts. Measured + /// on a rig at one tenth of the largest production store's scale (PostgreSQL 18.4 / TimescaleDB 2.28.1, the + /// production pair), EXPLAIN (ANALYZE, BUFFERS, WAL) of one bucket's materialization INSERT + /// (36,121 rows from 216,721 raw): with the group indexes 45.5 MB of WAL over 483,689 records, + /// 1.64 M buffer touches, 6,040 buffers dirtied; with only the bucket index 10.7 MB over 72,735 records, + /// 447 K buffer touches, 56 dirtied — the indexes were 4.3x the WAL and 1.19 M of the buffer touches per + /// bucket. On the rig those touches are memory hits and the wall clock barely moves; on a store whose + /// materialization is 71.5 GiB they are the leaf pages of eleven cold indexes, which is the I/O the issue's + /// alert-read victims were starved by. brings + /// an existing store to the same shape — this option only speaks at CREATE. Scoped to THIS aggregate on + /// purpose: the composer-grain rollups are read by server_id and query_hash through exactly + /// these indexes, and refreshes once a day and was not + /// measured — the option is earned by a measurement, not applied for symmetry. /// public const string CreateQueryStoreStatsIntervalHourlySql = @"CREATE MATERIALIZED VIEW IF NOT EXISTS collect.query_store_stats_interval_hourly -WITH (timescaledb.continuous) AS +WITH (timescaledb.continuous, timescaledb.create_group_indexes = false) AS SELECT server_id, server_name, @@ -2429,9 +2482,18 @@ internal static int RefreshPhaseMinutesFor(IReadOnlyList order, string v /// is not a fleet reading. #3175/#3177 has since given the GUC its own marker, so existing stores /// heal; that does not widen this population, because the read predates the heal. A later census /// could be broader, and would have to say so rather than inherit this one's scope. No SHIPPED read - /// touches job_history (every product surface uses job_stats, - /// deliberately — see ), so the gap is in what an investigation can - /// ask, not in what the product reports. Read at 2026-09-08 01:37Z, so the + /// takes a DURATION from job_history (every product duration surface uses job_stats, + /// deliberately — see ); the one shipped read that touches the view + /// at all is StoreSelfMetrics.JobHistoryEvidenceSql (#3574; the MCP reader's + /// DarlingStoreMetricsReader.JobHistoryEvidenceSql is an alias of that string), a bounded row + /// COUNT that proves the instrument is writing and evaluates the view's ownership filter for its own + /// connection. It has two callers since #3582: the MCP reader runs it as the connection asking, and the + /// hourly self-metrics sweep runs it as the job OWNER's role and persists the count as an + /// object_kind = 'job_history' row — because job_history shows rows only to members of the + /// job's owner or the database owner, a census like this one must be read as such a role, and a zero read + /// as any other role is the filter and not the table. So the gap is in what an investigation can ask, not + /// in what the product reports. + /// Read at 2026-09-08 01:37Z, so the /// window is one that has ENDED and stays true rather than a scope read against a clock a doc comment /// does not have. Each side of the boundary, since a bound is only as good as what it excludes: 304 /// runs at or before it, median 1081.7 s, maximum 13300.7 s; 57 runs after it, @@ -2908,7 +2970,7 @@ public static void LogRefreshCeilingStaleness( } logger.LogWarning( - "TimescaleDB: {View}'s refresh policy last ran {Seconds:F1}s, which is {Over:F1}s ABOVE {Constant} = {Ceiling:F1}s — a constant recorded as a PREFIX MAXIMUM: the largest run its regime had been recorded to make when it was read, which is no bound on the job, so a later run of the same regime joins that population and can exceed it. This run did, which makes the constant stale rather than wrong (#3188): it has to be RE-DERIVED over a population that includes this run (#3182), which is a different repair from re-deriving the compression phase grid (#3035) and is needed whatever band the slot watch puts this reading in. Its per-run history is timescaledb_information.job_history, one row per run, but only where timescaledb.enable_job_execution_logging is on — it is off by default and a store provisioned before that GUC gained its own conf marker reports nothing there until it heals, so an empty result is that gap and not a quiet hour (#3175/#3177). Reported once per constant and then only for a larger run, because what the re-derivation needs is the LARGEST reading and a repeat of one already reported adds nothing.", + "TimescaleDB: {View}'s refresh policy last ran {Seconds:F1}s, which is {Over:F1}s ABOVE {Constant} = {Ceiling:F1}s — a constant recorded as a PREFIX MAXIMUM: the largest run its regime had been recorded to make when it was read, which is no bound on the job, so a later run of the same regime joins that population and can exceed it. This run did, which makes the constant stale rather than wrong (#3188): it has to be RE-DERIVED over a population that includes this run (#3182), which is a different repair from re-deriving the compression phase grid (#3035) and is needed whatever band the slot watch puts this reading in. Its per-run history is timescaledb_information.job_history, one row per run, but only where timescaledb.enable_job_execution_logging is on — it is off by default, and with it off only FAILED runs are written (a failure is logged regardless of the GUC; a success needs it) — so a store provisioned before that GUC gained its own conf marker shows this job's successes there only once it heals, and a result holding no successful runs is that gap and not a quiet hour (#3175/#3177). Reported once per constant and then only for a larger run, because what the re-derivation needs is the LARGEST reading and a repeat of one already reported adds nothing.", view, observedSeconds, observedSeconds - recordedCeilingSeconds, constantName, recordedCeilingSeconds); } @@ -4011,21 +4073,13 @@ corrected Query Store rollups' ordering requirement lives with the list — L1 i precede the corrected view, which is hierarchical from it. (The corrected DAILY is L1's SIBLING, not the hourly's child: an identity-width hierarchical CAGG is a leaf — see CreateQueryStoreStatsCorrectedDailySql.) */ + /* The daily tier comes from DailyAggregates for the same reason (#3581): the aggregate-compression + ensure decides each materialization's compress_after by tier, and it must read the same seven + names this sweep creates rather than a second copy of them. The list carries its own ordering + requirement — L2 before the day-grain daily it feeds — on its declaration. */ var aggregates = HourlyAggregates .Select(a => (CreateSql: a.CreateSql, View: a.View, Hourly: true)) - .Concat(new[] - { - (CreateSql: CreateQueryStatsDailySql, View: QueryStatsDailyView, Hourly: false), - (CreateSql: CreateProcedureStatsDailySql, View: ProcedureStatsDailyView, Hourly: false), - (CreateSql: CreateQueryStoreStatsDailySql, View: QueryStoreStatsDailyView, Hourly: false), - (CreateSql: CreateQueryStoreStatsCorrectedDailySql, View: QueryStoreStatsCorrectedDailyView, Hourly: false), - (CreateSql: CreateQueryStatsDbDailySql, View: QueryStatsDbDailyView, Hourly: false), - /* The DAY-grain corrected daily (#1869), THREE levels deep: L1 (above) -> L2 interval_daily -> - daygrain_daily. Both must follow L1 and L2 must precede its own child, which this ordered sweep - gives — the same requirement the daily tier has, one level longer. */ - (CreateSql: CreateQueryStoreStatsIntervalDailySql, View: QueryStoreStatsIntervalDailyView, Hourly: false), - (CreateSql: CreateQueryStoreStatsDayGrainDailySql, View: QueryStoreStatsDayGrainDailyView, Hourly: false), - }) + .Concat(DailyAggregates.Select(a => (CreateSql: a.CreateSql, View: a.View, Hourly: false))) /* The seven baseline-tier aggregates (#1757; nine until #2007) ride the HOURLY tier: they are sourced from raw like the hourly tier, not hierarchically from another CAGG, so they carry no ordering requirement against the daily tier. Appended from the single BaselineAggregates list so this sweep @@ -4903,264 +4957,1508 @@ being checked. */ return applied; } - /* ─────────────── rollup availability (the plain-PostgreSQL guard, #1664) ─────────────── */ - - /// - /// One catalog round trip answering "which retention rollups exist in THIS store?" — the availability - /// input to . to_regclass - /// needs no table privilege and returns NULL for a missing relation, so this is safe under the viewer's - /// least-privilege role and on any store shape. Column order matches - /// 's constructor. - /// - public static readonly string RollupProbeSql = - "SELECT " + - $"to_regclass('collect.{QueryStatsHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStatsDailyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStatsDbHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStatsDbDailyView}') IS NOT NULL, " + - $"to_regclass('collect.{ProcedureStatsHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{ProcedureStatsDailyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStoreStatsHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStoreStatsDailyView}') IS NOT NULL, " + - /* The corrected Query Store rollups (#1849). A store on an older service has none of them and reads - fall back to the pair above — the same per-tier degrade #1664/#1665 built, which is why these need - no schema migration or version gate: existence IS the probe. */ - $"to_regclass('collect.{QueryStoreStatsIntervalHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStoreStatsCorrectedHourlyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStoreStatsCorrectedDailyView}') IS NOT NULL, " + - /* The day-grain daily and its dedup layer (#1869) — the same existence-is-the-probe degrade, so a - store on a #1849-era service keeps reading the corrected daily and needs no version gate either. */ - $"to_regclass('collect.{QueryStoreStatsIntervalDailyView}') IS NOT NULL, " + - $"to_regclass('collect.{QueryStoreStatsDayGrainDailyView}') IS NOT NULL"; - - /// - /// Detects which continuous-aggregate rollups exist in the store (). On a - /// plain-PostgreSQL store every flag is false — and that is a COMPLETE configuration, not a degraded one: - /// without the extension no retention policy ever drops raw, so the raw tables hold full history and - /// routing everything to raw loses nothing. On a TimescaleDB store the worker's ensure sweep creates the - /// views before any reader can need them; a partially-built store (one aggregate's failure-isolated - /// setup failed) reports exactly what exists, so the router degrades per tier instead of a reader - /// throwing 42P01 at a user (#1664, the gated-live catch on #1661's first cut). - /// - public static async Task DetectRollupsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken = default) + /* ─────────────── continuous-aggregate compression (#3581) ─────────────── */ + + /// + /// The margin a continuous aggregate's compress_after carries ABOVE its refresh policy's + /// start_offset: one raw chunk, . + /// + /// Why the aggregates need a margin at all, measured. A refresh that reaches into a compressed + /// materialization chunk does not fail on 2.28.1 — it decompresses every compressed batch that overlaps the + /// buckets it re-materializes (1,000 rows per server_id segment, measured on the rig: a six-row + /// backdated write into one hourly bucket staged 6,006 rows), rewrites them into the chunk's heap side, and + /// leaves the chunk PARTIAL (catalog status 9) until the next compression run recompresses it. Against an + /// uncompressed chunk the same single-bucket refresh took 8 ms; against the compressed one 29 ms plus a + /// 195 ms recompression on the next policy pass. A forced eight-day refresh went 1,150 ms → 1,514 ms and + /// left 180,000 heap rows for the policy to take back. So a refresh window that overlapped the compressed + /// region would not break anything; it would turn the newest chunk into a permanent decompress-and- + /// recompress churn, once an hour on the hourly tier, forever — which is why the two boundaries are kept + /// APART rather than merely non-failing when they meet. + /// + /// Why one raw chunk, derived rather than chosen. The refresh window reaches back + /// start_offset and then aligns its start DOWN to a bucket boundary, so the furthest it can reach is + /// start_offset plus one bucket. A chunk becomes eligible only when its WHOLE range is older than + /// compress_after. The two regions are therefore disjoint at any chunk width whenever + /// compress_after ≥ start_offset + bucket: 1 day 1 hour for the hourly tier, 4 days for the daily. + /// The daily tier's bucket IS a day, so the floor is a day there already; the hourly tier's is an hour, and + /// an hour is the wrong unit for the gap. Every other boundary in this store moves in days — chunks close at + /// UTC midnight (), raw eligibility flips at UTC midnight + /// () — and a sub-day margin would put the refresh's aligned start and the + /// compression boundary within one scheduling jitter of each other on every run. Taking the margin as one + /// raw chunk, on every tier, puts a whole day between the two on the hourly tier and lands the daily tier on + /// exactly its own floor. The result — 2 days hourly, 4 days daily — is the ruling #3581 was opened for, + /// reached as an expression rather than written down. + /// + /// What this does NOT protect, stated so it is not assumed. Two refreshes reach past every + /// policy window by design: the coverage-gated baseline backfill + /// (, up to back) and the + /// operator's --backfill-rollups verb. Both write below the materialized floor, where there are no + /// chunks to be compressed yet, except for the one chunk the floor sits inside — and that one they handle + /// through the partial-chunk path above, at the measured cost, once. That is a cost, not a hazard, and it + /// is bounded by being a one-time backfill rather than an hourly policy. + /// + public static readonly TimeSpan AggregateCompressMarginSpan = TimeSpan.FromDays(ChunkIntervalDays); + + /// + /// compress_after for every aggregate whose refresh policy is HOURLY — + /// plus , 2 days. That is + /// the six and the seven : thirteen of the + /// twenty, including the two interval-identity layers whose retention is the shortest in the store + /// (, 7 days) — which at 1-day materialization chunks leaves five + /// of those seven days compressed, and at 10-day chunks leaves the tier mostly uncompressed; see the + /// chunk-width paragraph on . + /// + public static readonly TimeSpan HourlyAggregateCompressAfterSpan = HourlyRefreshStartSpan + AggregateCompressMarginSpan; + + /// + /// compress_after for every aggregate whose refresh policy is DAILY — + /// plus , 4 days. That is + /// the seven . Their refresh keeps TimescaleDB's finish-to-start scheduling + /// (), so the instant it runs drifts — which is one more reason the + /// separation between refresh and compression is carried by the WINDOW arithmetic here rather than by + /// where on the clock either job happens to start. + /// + public static readonly TimeSpan DailyAggregateCompressAfterSpan = DailyRefreshStartSpan + AggregateCompressMarginSpan; + + /// The INTERVAL literal of — rendered from the span + /// rather than kept as a second constant, so the statement and the arithmetic cannot disagree. + public static string HourlyAggregateCompressAfter => WholeDaysInterval(HourlyAggregateCompressAfterSpan); + + /// The INTERVAL literal of . + public static string DailyAggregateCompressAfter => WholeDaysInterval(DailyAggregateCompressAfterSpan); + + /// + /// Renders a whole-day span as the PostgreSQL interval literal the policy statements interpolate, and + /// refuses anything else: a compress_after that was not a whole number of days would mean one of + /// its two inputs stopped being one, which is a design change and not a rendering problem. + /// + internal static string WholeDaysInterval(TimeSpan span) { - if (dataSource is null) + if (span <= TimeSpan.Zero || span.Ticks % TimeSpan.TicksPerDay != 0) { - throw new ArgumentNullException(nameof(dataSource)); + throw new ArgumentOutOfRangeException(nameof(span), span, "an aggregate compress_after must be a positive whole number of days"); } - await using var command = dataSource.CreateCommand(RollupProbeSql); - command.CommandTimeout = JobCatalogReadTimeoutSeconds; - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - await reader.ReadAsync(cancellationToken); - return new RollupAvailability( - reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), - reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), - reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), - reader.GetBoolean(11), reader.GetBoolean(12)); + return $"{((long)span.TotalDays).ToString(CultureInfo.InvariantCulture)} days"; } - /* ─────────────── rollup COVERAGE (the un-materialized-history guard, #1759) ─────────────── */ - - /// The hourly rollups' bucket width — named so reads as data. - /// Declared BEFORE and that is not cosmetic: C# runs static field - /// initializers in DECLARATION order, so a list declared above these would capture - /// default(TimeSpan) — zero — for every width, and every backfill bucket count would divide by - /// zero-width buckets. Caught by RollupBackfillTests going red on exactly that. - public static readonly TimeSpan HourlyBucket = TimeSpan.FromHours(1); + /// + /// The aggregate compression policies' cadence — once a day, not 's + /// hour. + /// + /// The hourly tick on the raw hypertables exists because their newest closed chunk is the least + /// compressed data on disk and a chunk that had already aged in was waiting up to half a day for a tick to + /// take it (#1778). Nothing here has that exposure: an aggregate chunk becomes eligible once, at a UTC + /// midnight, a whole after its refresh policy last touched it, and + /// a run that finds it within the day is exactly as good as one that finds it within the hour. What an + /// hourly cadence WOULD do is put twenty more jobs on the hourly phase grid — the thing + /// exists to avoid. + /// + public const string AggregateCompressionScheduleInterval = "1 day"; - /// The daily rollups' bucket width. See on declaration order. - public static readonly TimeSpan DailyBucket = TimeSpan.FromDays(1); + /// twin of , pinned equal + /// by test; compares a live job's cadence against this in + /// seconds, the same way the raw converge does, so 1 day and 24:00:00 never read as a + /// difference. + public static readonly TimeSpan AggregateCompressionScheduleSpan = TimeSpan.FromDays(1); /// - /// Every rollup view in probe order, with the two DIFFERENT relations it is measured against. One list, so - /// the coverage probe's column order, 's constructor and - /// cannot drift into disagreeing. + /// The segmentby column for every aggregate — the same server_id the raw hypertables use + /// (), for the same reason: every read of these relations filters + /// server_id first (the viewer's month-scale trend reads are WHERE server_id = $1 AND bucket + /// BETWEEN, and the retrieval indexes lead with it), so one segment IS one server's rows and a + /// compressed read touches only the segment it asked for. Measured on the rig against the trend read's own + /// SQL over : the compressed chunks are read through a + /// ColumnarScan whose index condition is exactly server_id = $1 with the bucket range as a + /// vectorized filter over the batches' min/max metadata — 15,179 shared buffers before compression and + /// 3,522 after for the same 30-day window, 11.2 ms → 6.6 ms. /// - /// RawTable and Source are not the same question, and conflating them was #1798. - /// RawTable is where a READ falls back to when this rollup cannot answer a window — always the raw - /// hypertable, because that is the only relation holding per-sweep rows. Source is what this rollup - /// is BUILT FROM, and therefore the most history it can ever contain: raw for the hourlies, but the HOURLY - /// VIEW for every daily, since all four dailies are hierarchical continuous aggregates - /// (time_bucket('1 day', bucket) FROM collect.<x>_hourly). + /// Every registered aggregate groups by it — asserted per definition by test from the shipped + /// CREATE text through , not assumed — so this is a property of the + /// registry rather than a constant that happens to be true today. + /// + public const string AggregateCompressionSegmentBy = "server_id"; + + /// + /// Every continuous aggregate this product owns, paired with the CREATE that defines it and whether its + /// refresh policy is hourly — the registry the compression ensure walks, in the order that ALSO decides each + /// one's hour on the daily band (). /// - /// The distinction decides whether a backfill can ever finish. The #1680 arming gate for an - /// HOURLY-tier retention policy is SOURCE-relative — the daily must cover what the hourly holds — while the - /// backfill verb converged every rollup to RAW's oldest row. On a store whose raw purges are armed, raw is - /// a few days deep and the hourlies legitimately hold weeks, so a daily converged "to raw" stops well short - /// of its hourly and the gate stays correctly held while the verb reports DONE. Worse, a hierarchical daily - /// added AFTER its hourly on such a store enters a hold NOTHING can clear: the pre-raw region exists only - /// in the hourly, and a verb aiming at raw never targets it. + /// Derived from , and + /// — the same three lists the ensure sweep creates from, in the same order + /// — rather than hand-listed, so an aggregate registered for creation is compression-registered the same + /// moment, with its tier decided by which list it came from. There is no fourth list to forget. /// - /// SourceTimeColumn follows from that: raw tables are keyed on collection_time, - /// rollup views on bucket. + /// MUST stay declared after those three lists: static field initializers run in declaration + /// order, and this one reads all three. + /// + public static readonly IReadOnlyList<(string CreateSql, string View, bool Hourly)> AggregateCompressionTargets = + HourlyAggregates.Select(a => (a.CreateSql, a.View, Hourly: true)) + .Concat(DailyAggregates.Select(a => (a.CreateSql, a.View, Hourly: false))) + .Concat(BaselineAggregates.Select(a => (a.CreateSql, a.View, Hourly: true))) + .ToArray(); + + /// + /// Is one of the continuous aggregates whose compression this product owns? Accepts + /// a bare or collect.-qualified name, the way does. /// - /// BucketWidth is carried EXPLICITLY, not inferred (#1849). Until the corrected Query - /// Store rollups existed, "hierarchical" and "daily" were the same fact, so the backfill derived a rollup's - /// bucket width from whether its source time column was bucket. - /// breaks that: it is hierarchical (sourced from L1) but its buckets are HOURS. Inferring would have given - /// its backfill a 24x-too-wide bucket, so every bucket count, slice count and disk estimate for it would - /// have been silently wrong — an under-estimate, which is the one direction the preflight exists to - /// prevent. Ordering still keys on the source column (raw-sourced rollups must be backfilled before the - /// rollups that read them); only the width became its own column. + /// This is the predicate + /// excludes on, and that exclusion is load-bearing rather than tidy. An aggregate's compression job reports + /// the aggregate's USER VIEW as its hypertable (collect / <view> — measured on 2.28.1, + /// the same resolution documents for refresh jobs), so it + /// lands in the raw converge's unscoped proc_name LIKE '%compression%' read, where a cadence other + /// than IS the staleness test (#1778). Without this, the first start + /// after this family exists would retune every one of its once-a-day jobs to an hourly tick — and put all + /// twenty on the hourly phase grid, which is precisely the placement #3581 ruled out. /// - public static readonly (string View, string RawTable, string Source, string SourceTimeColumn, TimeSpan BucketWidth)[] RollupViews = + public static bool IsAggregateCompressionTarget(string? view) { - (QueryStatsHourlyView, "query_stats", "query_stats", "collection_time", HourlyBucket), - (QueryStatsDailyView, "query_stats", QueryStatsHourlyView, "bucket", DailyBucket), - (QueryStatsDbHourlyView, "query_stats", "query_stats", "collection_time", HourlyBucket), - (QueryStatsDbDailyView, "query_stats", QueryStatsDbHourlyView, "bucket", DailyBucket), - (ProcedureStatsHourlyView, "procedure_stats", "procedure_stats", "collection_time", HourlyBucket), - (ProcedureStatsDailyView, "procedure_stats", ProcedureStatsHourlyView, "bucket", DailyBucket), - (QueryStoreStatsHourlyView, "query_store_stats", "query_store_stats", "collection_time", HourlyBucket), - (QueryStoreStatsDailyView, "query_store_stats", QueryStoreStatsHourlyView, "bucket", DailyBucket), + if (string.IsNullOrEmpty(view)) + { + return false; + } - /* The corrected Query Store rollups (#1849). L1 is raw-sourced; BOTH corrected views read L1 — the - daily is L1's second child, not the corrected hourly's, so it converges to L1 like its sibling. */ - (QueryStoreStatsIntervalHourlyView, "query_store_stats", "query_store_stats", "collection_time", HourlyBucket), - (QueryStoreStatsCorrectedHourlyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", HourlyBucket), - (QueryStoreStatsCorrectedDailyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", DailyBucket), + var dot = view.LastIndexOf('.'); + var bare = dot >= 0 ? view[(dot + 1)..] : view; - /* The day-grain daily and its dedup layer (#1869) — L1's THIRD child, and the first rollup in this - list whose own source is itself hierarchical. Both are DAY-bucketed, which is why the explicit - BucketWidth above is what keeps the backfill honest here as well. */ - (QueryStoreStatsIntervalDailyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", DailyBucket), - (QueryStoreStatsDayGrainDailyView, "query_store_stats", QueryStoreStatsIntervalDailyView, "bucket", DailyBucket), - }; + foreach (var target in AggregateCompressionTargets) + { + if (string.Equals(target.View, bare, StringComparison.Ordinal)) + { + return true; + } + } - /// The three raw tables the rollups roll up, in coverage-probe order (deduplicated - /// ). - public static readonly string[] RolledRawTables = { "query_stats", "procedure_stats", "query_store_stats" }; + return false; + } /// - /// How far back each rollup has actually MATERIALIZED, and how far back each raw table still reaches — - /// the input needs to stop routing a window at a rollup that cannot - /// answer it (#1759). - /// - /// The mechanism this exists for: a continuous aggregate created WITH NO DATA over - /// pre-existing history serves ONLY what was materialized. Real-time aggregation cannot rescue it — - /// the watermark is a hard partition (materialized below UNION ALL raw at-or-above), so raw - /// older than the watermark is excluded by construction, not merely un-accelerated. Every rollup's - /// refresh policy only reaches its own start offset back, so on a store that existed before its rollups - /// the materialized span begins at roughly creation minus that offset and NEVER reaches further back on - /// its own. - /// - /// to_regclass-safe by construction, not by guard. A relation named in a statement - /// is resolved at PARSE time, so no in-statement to_regclass test can keep min(bucket) - /// off a view that does not exist. Instead the SQL is BUILT from - /// — a view the round trip just proved - /// absent contributes a literal NULL and is never named. Column count is fixed either way, so - /// the reader's indexing does not depend on the store's shape. - /// - /// min(bucket) is deliberately the SAME expression - /// gates arming on. Routing and arming must agree about what a rollup covers, or the router would - /// serve a window the arming gate considers uncovered (or worse, the reverse). + /// Looks up a registered aggregate by view name. Throws for an unregistered one, for the same reason + /// does: a defaulted tier here would give an aggregate a + /// compress_after nobody derived for it. /// - public static string RollupCoverageProbeSql(RollupAvailability availability) + private static (string CreateSql, string View, bool Hourly) AggregateCompressionTargetFor(string view) { - var columns = RollupViews - .Select(r => availability.Has(r.View) - ? $"(SELECT min(bucket) FROM collect.{r.View})" - : "NULL::timestamp") - /* The raw tables are migration-created and always exist, so they need no availability gate. */ - .Concat(RolledRawTables.Select(t => $"(SELECT min(collection_time) FROM collect.{t})")); + foreach (var target in AggregateCompressionTargets) + { + if (string.Equals(target.View, view, StringComparison.Ordinal)) + { + return target; + } + } - return "SELECT " + string.Join(", ", columns); + throw new ArgumentOutOfRangeException( + nameof(view), + view, + "not a registered continuous aggregate — it is in none of HourlyAggregates, DailyAggregates or BaselineAggregates, so it has no compression tier"); } + /// Which compress_after takes, decided by its refresh tier — the + /// hourly one for every hourly-refreshed aggregate (including the baselines), the daily one for the + /// daily tier. + public static TimeSpan AggregateCompressAfterSpanFor(string view) + => AggregateCompressionTargetFor(view).Hourly ? HourlyAggregateCompressAfterSpan : DailyAggregateCompressAfterSpan; + + /// The INTERVAL literal of . + public static string AggregateCompressAfterFor(string view) + => WholeDaysInterval(AggregateCompressAfterSpanFor(view)); + /// - /// Reads every rollup's materialized floor and every rolled raw table's oldest row - /// (). comes from - /// in the same probe cycle and decides which relations are named at - /// all. + /// The bucket column of one aggregate — the alias its CREATE gives time_bucket(...) — recovered from + /// the shipped text, because that column is the materialization hypertable's time dimension and therefore + /// the orderby the compression settings need. /// - /// NOTE that a DAILY rollup's floor is the day-FLOOR of its oldest hourly bucket, so it can read - /// up to a day earlier than the hourly it is sourced from. That over-claims coverage by at most one - /// bucket, and it is exactly the semantics the arming gate already runs on — matching it is the point. + /// Parsed at parenthesis depth rather than by a regex over AS bucket, so a CREATE that aliased + /// its bucket differently would compress in that column's order instead of in a column that does not exist. + /// Every shipped definition aliases it bucket and a test asserts that per definition; this recovers + /// it anyway so the assertion is about the registry and not about this method agreeing with itself. /// - public static async Task DetectRollupCoverageAsync( - NpgsqlDataSource dataSource, RollupAvailability availability, CancellationToken cancellationToken = default) + internal static string AggregateBucketColumnFor(string createSql) { - if (dataSource is null) + if (createSql is null) { - throw new ArgumentNullException(nameof(dataSource)); + throw new ArgumentNullException(nameof(createSql)); } - await using var command = dataSource.CreateCommand(RollupCoverageProbeSql(availability)); - command.CommandTimeout = JobCatalogReadTimeoutSeconds; - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - if (!await reader.ReadAsync(cancellationToken)) + const string TimeBucket = "time_bucket("; + var start = createSql.IndexOf(TimeBucket, StringComparison.OrdinalIgnoreCase); + if (start < 0) { - return RollupCoverage.Unknown; + throw new ArgumentException("the CREATE has no time_bucket(...) projection, so its bucket column cannot be recovered", nameof(createSql)); } - var floors = new Dictionary(StringComparer.Ordinal); - for (var i = 0; i < RollupViews.Length; i++) + var depth = 0; + var index = start + TimeBucket.Length - 1; + for (; index < createSql.Length; index++) { - if (!await reader.IsDBNullAsync(i, cancellationToken)) + if (createSql[index] == '(') { - floors[RollupViews[i].View] = reader.GetDateTime(i); + depth++; + } + else if (createSql[index] == ')' && --depth == 0) + { + break; } } - var rawOldest = new Dictionary(StringComparer.Ordinal); - for (var i = 0; i < RolledRawTables.Length; i++) + var rest = createSql[(index + 1)..].TrimStart(); + if (!rest.StartsWith("AS ", StringComparison.OrdinalIgnoreCase)) { - var ordinal = RollupViews.Length + i; - if (!await reader.IsDBNullAsync(ordinal, cancellationToken)) - { - rawOldest[RolledRawTables[i]] = reader.GetDateTime(ordinal); - } + throw new ArgumentException("the CREATE's time_bucket(...) projection carries no alias, so its bucket column cannot be recovered", nameof(createSql)); } - return new RollupCoverage(floors, rawOldest); - } + var alias = rest[3..].TrimStart(); + var end = 0; + while (end < alias.Length && (char.IsLetterOrDigit(alias[end]) || alias[end] == '_')) + { + end++; + } - /// - /// Converts every collector table to a hypertable ( scope; - /// per table). Failure-isolated per table: one failed - /// conversion warns and the sweep continues — that table stays a plain PG table, keeps - /// working (COPY and DELETE-based retention are hypertable-agnostic), and is retried on the - /// next service start. Returns the number of tables that converted (or no-op'd) cleanly. - /// - public static async Task ConvertToHypertablesAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) - { - if (connection is null) + if (end == 0) { - throw new ArgumentNullException(nameof(connection)); + throw new ArgumentException("the CREATE's time_bucket(...) alias is empty", nameof(createSql)); } - var converted = 0; - foreach (var schema in HypertableTables) + return alias[..end]; + } + + /// + /// THE DAILY BAND'S MINUTE (#3581) — the last minute of the heaviest refresh's window, stated as a method + /// because a table of minutes cannot be re-derived by the next reader and a method can. + /// + /// What the hour has left, read off the grid. + /// tiles the hour with three bands: the light refreshes start on :00 through :11, the heaviest + /// refresh at , and the raw compression band takes + /// to the end of the hour. No minute is unassigned — the tiling + /// identity TimescaleContinuousAggregateTests holds says so — but two stretches carry no START: the guard + /// after the light band, where the light refreshes are still finishing, and the heaviest refresh's window + /// past its own start. The compression grid deliberately leaves the tail of that window unused + /// (: "the other 6 minutes of the window are past the refresh and are + /// left on the table"), because recovering them for HOURLY policies would size a band for one window + /// against a still-moving ceiling. A once-a-day job is a different trade, and this is the minute it + /// takes. + /// + /// The last minute, not the first clear one, and the difference is 296 seconds. The first + /// minute past the recorded ceiling is plus + /// ceil( / 60) = :30, which is 900 s after + /// the heaviest refresh starts against an 896 s ceiling — a four-second margin. The window's last minute is + /// 1,200 s after it. Both are pinned: this minute is asserted past the ceiling by test in the same way the + /// grid asserts the ceiling inside the window, so a ceiling that grew to meet it goes red rather than + /// quietly putting a daily rewrite into the heaviest refresh's tail. And it is derived from GEOMETRY — + /// start plus width — rather than from the ceiling constant, so re-taking the ceiling (#3182, #3188) moves + /// no job here. + /// + /// What sits on either side, and why neither is a lock hazard. The minute after this one is + /// ' first, where the raw compression policies start; they lock raw + /// chunks and this family locks materialization chunks, so the two cannot queue on each other, and on + /// twenty-three hours of the day those policies find nothing eligible and finish in tens of milliseconds. + /// The next refresh START is the next hour's :00, 25 minutes away — and a compression run cannot + /// block an aggregate's own refresh in any case: compress_chunk on 2.28.1 holds + /// AccessShareLock on the materialization hypertable and escalates only on the CHUNK it is rewriting + /// (ShareLock, ExclusiveLock, then AccessExclusiveLock at the swap — all three observed + /// on the rig inside one run), while the refresh writes the newest chunk, which + /// keeps out of the eligible set. What a run CAN hold up is a + /// reader of exactly the chunk being swapped, for the swap; that is the ordinary cost of compressing and + /// the same one the raw tier pays today. + /// + /// Why not #3174's compression band. That band's width is derived from + /// over , and every minute it + /// takes comes out of . Twenty more members would widen it from + /// 24 minutes to 31 and shrink the window from 21 minutes to 14 — 840 s against the 896 s ceiling, which + /// is the envelope TimescaleSupportTests holds red. #3185 measured what stepping a band by member count + /// does to per-group cost; this is the same lesson from the other side. So this family does not join that + /// band, and its own minute is one the hourly grid already has and does not use. + /// + public static int AggregateCompressionBandMinute => + HeaviestRefreshStartMinute + HeaviestRefreshWindowMinutes - 1; + + /// + /// The hour of the day (UTC) the first registered aggregate compresses at; each later one takes the next + /// hour (). + /// + /// One, and it is the hour AFTER the one that carries the raw tier's daily rewrite. + /// and are both one, and 1-day chunks are + /// epoch-aligned, so every raw hypertable's newest closed chunk becomes eligible at the same UTC midnight and + /// the 00: hour's compression band is the one that does a day's compressing (#3112's midnight band — + /// its largest run, query_store_stats, measured at 552 s from :42). The aggregates' chunks + /// become eligible at the same midnight, for the same epoch-alignment reason, so the earliest hour that both + /// sees the new eligibility and is clear of that rewrite is the next one. Later hours would only add + /// latency to the newest eligible chunk; earlier there is none. + /// + public const int AggregateCompressionBandFirstHour = 1; + + /// The hours in a day, named so the band's fit is an identity against the cadence it tiles rather + /// than a literal 24. + public static int HoursInDailyCadence => (int)AggregateCompressionScheduleSpan.TotalHours; + + /// + /// Which hour of the day (UTC) 's compression policy runs at: one aggregate per + /// hour, in order from . + /// + /// An hour apart rather than a few minutes apart, because distinct STARTS are not the + /// guarantee that was asked for (#3185). Staggering twenty jobs a few minutes apart inside one hour + /// gives twenty distinct starts and nothing about overlap: a run longer than the step overlaps its successor + /// for the rest of its run, and the runs here are minutes — a day's chunk of the largest aggregate is + /// roughly the raw table's, which #3112 measured at 552 s. The hour also has only six minutes that are past + /// every recorded refresh ceiling and on no hourly-grid start (), + /// so the stagger had nowhere to go in any case. Placing one aggregate per hour makes the property "no two + /// aggregates decompress and recompress at once" hold by construction: a run would have to exceed an hour + /// to meet the next, and a run that long is the stuck-job check's business + /// (, 48 hours at this cadence). It also keeps this family inside the + /// background-worker headroom the managed store already sizes — at most one of these jobs runs at a time, + /// so the + 2 over is not re-derived. + /// + /// Keyed on the VIEW's registry position, never on a job id or on a store measurement, for + /// the reason gives: the same configuration has to be reproducible on + /// a store that has never seen this one's ids or sizes. The staging of FIRST runs is by measured size + /// (); the hour is by identity. An operator reading + /// timescaledb_information.jobs therefore sees the same aggregate at the same hour on every + /// store. + /// + /// Throws for an unregistered view ('s reasoning), and for + /// a registry too long for the day — twenty-three hours are available, and a twenty-fourth member would land + /// on the midnight hour exists to avoid. The ensure builds + /// each statement inside its per-aggregate try, so either throw costs one aggregate and names it. + /// + public static int AggregateCompressionBandHourFor(string view) + { + for (var index = 0; index < AggregateCompressionTargets.Count; index++) { - try + if (!string.Equals(AggregateCompressionTargets[index].View, view, StringComparison.Ordinal)) { - using var command = new NpgsqlCommand(CreateHypertableSql(schema), connection) { CommandTimeout = SetupTimeoutSeconds }; - await command.ExecuteNonQueryAsync(cancellationToken); - converted++; + continue; } - catch (Exception ex) when (ex is not OperationCanceledException) + + var hour = AggregateCompressionBandFirstHour + index; + if (hour >= HoursInDailyCadence) { - logger?.LogWarning("Hypertable conversion failed for {Table} — it stays a plain table: {Message}", - schema.TargetTable, ex.Message); + throw new InvalidOperationException( + $"the aggregate compression band has {AggregateCompressionTargets.Count} members from hour {AggregateCompressionBandFirstHour}, which runs past the day — {view} would land on hour {hour}. Re-derive the band (two per hour, or a second minute) rather than wrapping onto the midnight hour"); } + + return hour; } - logger?.LogInformation("TimescaleDB: {Converted}/{Total} collector table(s) are hypertables", - converted, HypertableTables.Count); - return converted; + throw new ArgumentOutOfRangeException( + nameof(view), + view, + "not a registered continuous aggregate — register it in HourlyAggregates, DailyAggregates or BaselineAggregates before giving it a compression policy"); } /// - /// Enables compression and adds the -day background policy on - /// every collector table (both statements per table, failure-isolated per table — a table - /// that failed hypertable conversion warns here too and stays uncompressed). Compressed - /// chunks remain fully queryable: this is Darling's archival tier (see - /// ). Returns the number of tables with a policy in place. + /// One aggregate's compression enablement: the same server_id segmentation the raw tier uses, ordered + /// by the aggregate's own bucket column descending — the read order of every trend query. + /// + /// ALTER MATERIALIZED VIEW rather than ALTER TABLE on the materialization: TimescaleDB + /// resolves the settings onto the materialization hypertable itself, and the statement is idempotent — on a + /// view already compressed it re-states the settings with a NOTICE that existing compressed chunks keep + /// theirs, measured on 2.28.1. The ensure still gates it on the catalog + /// (continuous_aggregates.compression_enabled) so a settled store runs no ALTER at all. Same pre-2.18 + /// vocabulary as , for the same cross-2.x reason. /// - public static async Task ApplyCompressionPolicyAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + public static string EnableAggregateCompressionSql(string view) { - if (connection is null) + var target = AggregateCompressionTargetFor(view); + return $"ALTER MATERIALIZED VIEW collect.{view} SET (timescaledb.compress, timescaledb.compress_segmentby = '{AggregateCompressionSegmentBy}', timescaledb.compress_orderby = '{AggregateBucketColumnFor(target.CreateSql)} DESC')"; + } + + /// + /// The initial_start expression for an aggregate compression job: + /// UTC midnights after the NEXT one, at :. + /// + /// Anchored to the next UTC midnight rather than to "the next occurrence of this hour" so that night + /// k is the same calendar day for every aggregate — which is what makes "one aggregate per night, + /// largest first" () mean consecutive calendar nights rather + /// than a sequence that folds when one aggregate's hour has already passed today and another's has not. + /// Always in the future, so the statement never depends on TimescaleDB's handling of a past anchor; computed + /// in UTC and cast back, for the reason gives — a bare + /// date_trunc('day', now()) truncates in the SESSION time zone. + /// + public static string AggregateCompressionInitialStartSql(int hour, int minute, int nightOffset) + { + if (hour < 0 || hour >= 24 || minute < 0 || minute >= 60 || nightOffset < 0) + { + throw new ArgumentOutOfRangeException(nameof(nightOffset), "the band instant must be a valid hour and minute of the day and a non-negative night"); + } + + return "(date_trunc('day', now() AT TIME ZONE 'UTC') + INTERVAL '1 day'" + + $" + INTERVAL '{nightOffset.ToString(CultureInfo.InvariantCulture)} days'" + + $" + INTERVAL '{hour.ToString(CultureInfo.InvariantCulture)} hours {minute.ToString(CultureInfo.InvariantCulture)} minutes') AT TIME ZONE 'UTC'"; + } + + /// + /// One aggregate's compression policy: its tier's compress_after, the once-a-day cadence, and a + /// FIXED schedule pinned to its hour on the daily band, first running nights + /// after the next UTC midnight. if_not_exists so a restart re-converges; the -1 it returns for a + /// policy the store already has is the quiet skip add_compression_policy is documented to give + /// (), which is why parameter drift on an EXISTING policy is + /// the converge's business () and not this statement's. + /// + public static string AddAggregateCompressionPolicySql(string view, int nightOffset) + => $"SELECT add_compression_policy('collect.{view}', compress_after => INTERVAL '{AggregateCompressAfterFor(view)}', schedule_interval => INTERVAL '{AggregateCompressionScheduleInterval}', if_not_exists => true, initial_start => {AggregateCompressionInitialStartSql(AggregateCompressionBandHourFor(view), AggregateCompressionBandMinute, nightOffset)})"; + + /// + /// Every registered aggregate's compression state in one read: whether compression is enabled on the + /// materialization, its compression job if any (with the four things a converged policy is made of — + /// compress_after, cadence, fixed schedule, and the UTC hour and minute it is pinned to), how large the + /// materialization is, and how many of its chunks would compress on the policy's first run under EACH + /// tier's compress_after — both counted, the caller picks its tier's, because the tier is a C# + /// registry fact and the read should not restate it. + /// + /// Joins the job on EITHER identity, the view or its materialization, for the reason + /// documents: the job catalog resolves an aggregate's + /// jobs back to the user view (measured for compression jobs too — collect / <view>), + /// but the underlying row carries the materialization, and the two schemas are disjoint so no job can + /// match twice. Chunks are keyed by the MATERIALIZATION, which is the only identity + /// timescaledb_information.chunks knows an aggregate by — a count keyed by the view name reads zero + /// forever, silently, which is the shape #3582 recorded for the store-metrics tool. + /// + /// Sizes come from hypertable_size over the materialization — the whole relation, indexes + /// included, because indexes are what compression removes on a chunk — and the hour and minute are + /// extracted in UTC so a session time zone cannot make a correctly pinned job read as stale. + /// + public static string AggregateCompressionStateSql => + $@" +SELECT + ca.view_name, + ca.compression_enabled, + j.job_id, + EXTRACT(EPOCH FROM (j.config->>'compress_after')::interval)::bigint AS compress_after_seconds, + EXTRACT(EPOCH FROM j.schedule_interval)::bigint AS schedule_interval_seconds, + j.fixed_schedule, + CASE WHEN j.initial_start IS NULL THEN NULL ELSE EXTRACT(HOUR FROM j.initial_start AT TIME ZONE 'UTC')::int END AS phase_hour, + CASE WHEN j.initial_start IS NULL THEN NULL ELSE EXTRACT(MINUTE FROM j.initial_start AT TIME ZONE 'UTC')::int END AS phase_minute, + hypertable_size(format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass) AS materialization_bytes, + ( + SELECT count(*) + FROM timescaledb_information.chunks AS c + WHERE c.hypertable_schema = ca.materialization_hypertable_schema + AND c.hypertable_name = ca.materialization_hypertable_name + AND NOT c.is_compressed + AND c.range_end < now() - INTERVAL '{HourlyAggregateCompressAfter}' + ) AS eligible_under_hourly_rule, + ( + SELECT count(*) + FROM timescaledb_information.chunks AS c + WHERE c.hypertable_schema = ca.materialization_hypertable_schema + AND c.hypertable_name = ca.materialization_hypertable_name + AND NOT c.is_compressed + AND c.range_end < now() - INTERVAL '{DailyAggregateCompressAfter}' + ) AS eligible_under_daily_rule +FROM timescaledb_information.continuous_aggregates AS ca +LEFT JOIN timescaledb_information.jobs AS j + ON (j.proc_name LIKE '%compression%' OR j.proc_name LIKE '%columnstore%') + AND ( + (j.hypertable_schema = ca.view_schema AND j.hypertable_name = ca.view_name) + OR (j.hypertable_schema = ca.materialization_hypertable_schema AND j.hypertable_name = ca.materialization_hypertable_name) + ) +WHERE ca.view_schema = 'collect'"; + + /// + /// Moves one EXISTING aggregate compression policy onto the shipped compress_after, cadence and band + /// instant. $1 the job id (::integer, the #1586 trap), $2 the compress_after + /// text, $3 the hour, $4 the minute. + /// + /// Exists because add_compression_policy(if_not_exists => true) returns -1 and changes + /// nothing against a policy the store already has ( measured + /// it), so a later change to either tier's compress_after, or to the band, would reach fresh stores + /// only — the #1937/#1778 drift this project treats as a defect, pre-empted here rather than filed later. + /// config is updated with jsonb_set against the job's OWN config so its other keys survive; + /// scheduled is not named, so this can neither arm nor pause. The anchor is re-taken at night zero: + /// a policy this converges is one whose hour, minute, cadence or window DIFFERS from the shipped values, and + /// a policy created by this build cannot differ before its first run — so re-anchoring never disturbs a + /// staged first night, and never re-anchors a policy that merely has not run yet. + /// + public static string SetAggregateCompressionPolicySql => + $@"SELECT alter_job( + j.job_id, + config => jsonb_set(j.config, '{{compress_after}}', to_jsonb($2::text)), + schedule_interval => INTERVAL '{AggregateCompressionScheduleInterval}', + fixed_schedule => true, + initial_start => (date_trunc('day', now() AT TIME ZONE 'UTC') + INTERVAL '1 day' + ($3::int * INTERVAL '1 hour') + ($4::int * INTERVAL '1 minute')) AT TIME ZONE 'UTC') +FROM timescaledb_information.jobs AS j +WHERE j.job_id = $1::integer"; + + /// + /// One aggregate as the ensure read it: what the store holds for it right now. + /// + public sealed record AggregateCompressionState( + string View, + bool CompressionEnabled, + int? JobId, + long? CompressAfterSeconds, + long? ScheduleIntervalSeconds, + bool FixedSchedule, + int? PhaseHour, + int? PhaseMinute, + long MaterializationBytes, + long EligibleChunksNow); + + /// + /// THE STAGING (#3581): which night each aggregate that still needs a compression policy gets its FIRST run + /// — one aggregate per night, largest first, and every aggregate with nothing to compress on night zero. + /// Returns (view, nightOffset) in the order the policies should be created. + /// + /// Why a first run is the heavy event and a steady-state run is not. A compression policy's + /// first run compresses EVERY chunk older than compress_after, one chunk per transaction, in one + /// pass. On a store that has been materializing for weeks that is the whole aggregate minus its newest + /// — on the largest production store, four Query Store aggregates + /// of 71.5, 55.6, 54.7 and 33.5 GiB, ~235 GiB across all twenty. Every run after the first finds at most the + /// chunks that aged in since yesterday. So the question the staging answers is only ever about the first + /// run, and it answers it the way #3581 ruled: one aggregate's backlog per night, so each night's rewrite + /// is one bounded relation and the phase grid's runtime watch (#3044/#3166) sees the effect of one before + /// the next begins. + /// + /// Largest first, by measured materialization size. The largest aggregate is the one whose + /// backlog pass costs the most and saves the most disk; taking it first front-loads the saving onto the + /// store that needs it and puts the biggest single night where the operator is watching most closely — + /// the first one. Size is read from the store () rather than + /// assumed from the registry, because which aggregate is largest is a property of the workload: on a store + /// with no writable Query Store primary the Query Store family is empty and the query_stats rollups + /// lead. + /// + /// Nothing to compress means night zero, not a place in the queue. An aggregate with no chunk + /// past its compress_after — every aggregate on a fresh store, and the empty ones on any store — has + /// no backlog pass to stage, so its policy simply starts tomorrow and finds nothing. A fresh store therefore + /// gets all twenty policies at once, exactly as the ruling asks, with no special case: the queue is empty + /// and everything is on night zero. + /// + /// Carried by the schedule, not by state. The nights are written into each policy's + /// initial_start when it is created, so the plan is visible in timescaledb_information.jobs + /// (next_start reads as consecutive calendar days) and survives a restart with nothing to remember — + /// the alternative, adding one policy per daily tick, needs a tick the startup path does not have and a + /// cursor that a restart mid-sequence would have to recover. The one thing this shape does not do is + /// re-plan around a partial failure: a start that created ten of twenty policies and lost its connection + /// stages the remaining ten from night zero on the next start, so up to two aggregates can share a night + /// on that path, each still at its own hour. That is the bounded, visible consequence and it is accepted + /// over a stateful cursor. + /// + /// Pure — no clock, no catalog — so the order pins directly, and ties on size break on registry + /// order so the result is deterministic on a store where two empty aggregates read the same bytes. + /// + public static IReadOnlyList<(string View, int NightOffset)> StageAggregateCompressionNights( + IReadOnlyList needingPolicy) + { + if (needingPolicy is null) + { + throw new ArgumentNullException(nameof(needingPolicy)); + } + + var registryIndex = new Dictionary(StringComparer.Ordinal); + for (var index = 0; index < AggregateCompressionTargets.Count; index++) + { + registryIndex[AggregateCompressionTargets[index].View] = index; + } + + var staged = new List<(string View, int NightOffset)>(); + + var night = 0; + foreach (var state in needingPolicy + .Where(s => s.EligibleChunksNow > 0) + .OrderByDescending(s => s.MaterializationBytes) + .ThenBy(s => registryIndex.TryGetValue(s.View, out var at) ? at : int.MaxValue)) + { + staged.Add((state.View, night++)); + } + + foreach (var state in needingPolicy + .Where(s => s.EligibleChunksNow <= 0) + .OrderBy(s => registryIndex.TryGetValue(s.View, out var at) ? at : int.MaxValue)) + { + staged.Add((state.View, 0)); + } + + return staged; + } + + /* ─────────────── materialization chunk width (#3620) ─────────────── */ + + /// + /// The chunk interval every continuous aggregate's materialization hypertable is held at: ONE raw chunk, + /// , as a span. Derived from the constant rather than written as a day so the + /// raw tables and their rollups cannot disagree — a store whose raw tier moved to a different width would + /// move its materializations with it on the next start. + /// + public static TimeSpan MaterializationChunkIntervalSpan => TimeSpan.FromDays(ChunkIntervalDays); + + /// The interval literal interpolates, rendered + /// through the same whole-days renderer the compression windows use. + public static string MaterializationChunkInterval => WholeDaysInterval(MaterializationChunkIntervalSpan); + + /// + /// Every registered aggregate's CURRENT materialization chunk interval, in seconds, keyed by view name. + /// timescaledb_information.dimensions carries one time_interval per hypertable dimension; a + /// materialization has exactly one, its time dimension, and the join is on the materialization identity + /// because that is the only name the dimensions view knows an aggregate by (the same rule as the chunk + /// counts in ). Seconds rather than the interval itself so the + /// comparison is an integer equality in C# against , and a + /// session setting cannot change how the interval renders. + /// + public static string MaterializationChunkIntervalStateSql => + @" +SELECT + ca.view_name, + EXTRACT(EPOCH FROM d.time_interval)::bigint AS chunk_interval_seconds +FROM timescaledb_information.continuous_aggregates AS ca +JOIN timescaledb_information.dimensions AS d + ON d.hypertable_schema = ca.materialization_hypertable_schema + AND d.hypertable_name = ca.materialization_hypertable_name + AND d.dimension_type = 'Time' +WHERE ca.view_schema = 'collect'"; + + /// + /// Moves ONE aggregate's materialization hypertable onto . The + /// materialization is resolved from the catalog inside the statement — its schema and name are TimescaleDB's + /// (_timescaledb_internal._materialized_hypertable_N), never stable across stores, so the view name + /// from the registry is the only identity this file should carry. set_chunk_time_interval affects + /// chunks created AFTER the call and nothing else — measured on 2.28.1: existing chunks keep their range, + /// the aggregate's refresh, compression and retention jobs read back byte-identical, and re-running with the + /// interval the hypertable already has is a no-op — but the ensure still gates it on the catalog so a + /// settled store issues no statement at all. + /// + public static string SetMaterializationChunkIntervalSql(string view) + { + _ = AggregateCompressionTargetFor(view); + return $@"SELECT set_chunk_time_interval( + format('%I.%I', ca.materialization_hypertable_schema, ca.materialization_hypertable_name)::regclass, + INTERVAL '{MaterializationChunkInterval}') +FROM timescaledb_information.continuous_aggregates AS ca +WHERE ca.view_schema = 'collect' AND ca.view_name = '{view}'"; + } + + /// + /// Holds every registered aggregate's materialization at one raw chunk of width (#3620). Idempotent under + /// the catalog — a settled store reads timescaledb_information.dimensions once and issues nothing — + /// and failure-isolated per aggregate. Returns the number of materializations whose interval this call + /// changed. Called first thing by , so the order the worker + /// sees is: aggregates exist → chunk width set → compression enabled and scheduled. + /// + /// The default this overrides, measured on 2.28.1. TimescaleDB sizes a first-level continuous + /// aggregate's materialization chunks at TEN TIMES the raw hypertable's chunk interval as it stands at + /// creation (1-day raw → 10-day, 2-day → 20-day, 3-day → 30-day), and a hierarchical aggregate's at exactly + /// its PARENT materialization's interval as it stands at creation (a parent set to 5 days yields a 5-day + /// daily; a parent at 1 day yields a 1-day daily). On an untouched store the two rules agree — the parent is + /// already root × 10 — which is why the #3581 rig read the dailies as inheriting from the root. Nothing in + /// this file ever set the materializations' interval, so every one of the twenty aggregates on every store + /// carries 10-day chunks: on the largest production store all twenty read 10 days, the interval-dedup + /// tiers in two chunks spanning twenty days and the rest in four spanning forty. + /// + /// The two consequences, with that store's numbers. First, retention over-holds: + /// drop_chunks removes a chunk only once its WHOLE range is past the horizon, so a 7-day tier on + /// 10-day chunks holds between 7 and 17 days of rows at any moment (the 7-day + /// was holding Sep 4 → Sep 24 in two chunks, ~14 days) and + /// the 10-day tier between 10 and 20 — tiers that are "internal plumbing sized only to outlive what gates on + /// them" (#1958) holding roughly double their design, and a per-day ingest figure derived from their size at + /// the assumed width reading ~2× high. Second, compression reaches less than #3581 projected: a policy + /// compresses a chunk only once its whole range is past compress_after, so with a 10-day head chunk + /// the newest ~10 days of every aggregate stay uncompressed regardless of a 2-day window — ~11% of a 90-day + /// tier, but most of a 7-day one, whose only compressible chunk is the one already past the horizon's edge. + /// Both were reasoned about as if the chunks were a day wide. + /// + /// Why one raw chunk. It is the granularity every raw table already has + /// (), the granularity retention and compression both act at (chunks close at + /// UTC midnight; eligibility flips at UTC midnight), and the unit the whole compression derivation on + /// is stated in. It does not touch the disjointness argument #3581 + /// pinned: a chunk compresses only when its whole range is past compress_after, which holds at any + /// width, and the daily tier's 3-day refresh start_offset reads hourly rows by bucket, not by chunk. + /// The cost is ten times as many materialization chunks — a few hundred rather than a few dozen, inside + /// what the raw tier already carries across ~70 hypertables — and one more job-free catalog read per start. + /// + /// The honest side-effect: convergence is gradual, not immediate. Existing 10-day chunks keep + /// their range and age out on their own schedule; only chunks created after this runs are a day wide. So the + /// interval tiers' VISIBLE hold tightens toward its designed 7 / 10 days over roughly two weeks as the old + /// chunks drop — a one-time ~5–10 GiB reclaim on the largest store, spread over that fortnight — and + /// #3581's compression reaches everything older than compress_after + 1 day instead of + 10 on the + /// same timetable. Modest movement on the first nights after this ships is the expected shape, not evidence + /// against either change. + /// + /// All twenty, every start, under the catalog check — not only the first-level aggregates. + /// The creation sweep () builds every aggregate BEFORE this + /// runs, so on a fresh store the hierarchical dailies inherit 10 days from hourlies that have not been + /// narrowed yet; on an existing store they already carry it. Setting each one directly is correct under + /// either inheritance rule and costs nothing on a settled store. + /// + public static async Task EnsureMaterializationChunkIntervalAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + var currentSeconds = new Dictionary(StringComparer.Ordinal); + try + { + using var probe = new NpgsqlCommand(MaterializationChunkIntervalStateSql, connection) { CommandTimeout = SetupTimeoutSeconds }; + await using var reader = await probe.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var view = reader.GetString(0); + if (!IsAggregateCompressionTarget(view) || reader.IsDBNull(1)) + { + continue; + } + + currentSeconds[view] = Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "TimescaleDB: could not read the continuous aggregates' materialization chunk intervals, so none was changed this start — new materialization chunks keep whatever width the store gives them (ten raw chunks, by default) until the next start reads it: {Message}", + ex.Message); + return 0; + } + + var wantedSeconds = (long)MaterializationChunkIntervalSpan.TotalSeconds; + var atTarget = 0; + var changed = 0; + var absent = new List(); + + foreach (var (_, view, _) in AggregateCompressionTargets) + { + if (!currentSeconds.TryGetValue(view, out var seconds)) + { + /* The creation sweep is failure-isolated per aggregate and has already warned about this one. */ + absent.Add(view); + continue; + } + + if (seconds == wantedSeconds) + { + atTarget++; + continue; + } + + try + { + using var set = new NpgsqlCommand(SetMaterializationChunkIntervalSql(view), connection) { CommandTimeout = SetupTimeoutSeconds }; + await set.ExecuteNonQueryAsync(cancellationToken); + atTarget++; + changed++; + + logger?.LogInformation( + "TimescaleDB: continuous aggregate {View}'s materialization now chunks at {Interval} (was {WasDays:0.#} days) — existing chunks keep their range and age out on their own schedule; only chunks created from now on are one raw chunk wide, so its retention tier converges to its designed hold and its compression policy reaches its newest days over the next couple of weeks, not tonight (#3620).", + view, MaterializationChunkInterval, seconds / 86400d); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "Could not set continuous aggregate {View}'s materialization chunk interval to {Interval} — its new chunks keep the store's default width ({WasDays:0.#} days), so its retention tier keeps over-holding and its compression policy keeps missing its newest chunk, until the next restart retries (often a permission issue: the store login must own the materialization): {Message}", + view, MaterializationChunkInterval, seconds / 86400d, ex.Message); + } + } + + /* The #1958 shape: the count that matters (how many are at the designed width), what this start did about + it, and the interval interpolated rather than restated. */ + logger?.LogInformation( + "TimescaleDB: {AtTarget}/{Total} materializations chunked at {Interval} (one raw chunk, ChunkIntervalDays), {Changed} changed this start, {Absent} aggregate(s) absent: {AbsentViews}", + atTarget, AggregateCompressionTargets.Count, MaterializationChunkInterval, changed, absent.Count, absent.Count == 0 ? "none" : string.Join(", ", absent)); + + return changed; + } + + /* ─────────────── interval-dedup materialization indexes (#3597) ─────────────── */ + + /// + /// The per-GROUP-BY-column indexes TimescaleDB's default create_group_indexes built on + /// 's materialization hypertable, resolved from the catalog + /// by SHAPE rather than by name: a btree on the materialization whose key is exactly one column followed by + /// bucket DESC. The bucket index ((bucket DESC) alone) does not match and stays — the refresh's + /// DELETE, the coverage probe's min(bucket) and the children's bucket-range reads all use it. The + /// materialization's schema and name are TimescaleDB's (_timescaledb_internal._materialized_hypertable_N), + /// never stable across stores, so the statement resolves them from the view name, the same rule as + /// . Hypertable-level indexes only (pg_indexes on the + /// parent): dropping the parent drops every chunk's copy, so the per-chunk names never need to be known — + /// they are read here only to SIZE what a drop releases, and for that the naming convention is the only + /// map 2.28.1 offers (no catalog row and no pg_inherits/pg_depend edge ties a chunk's copy to + /// its parent index): a copy is named <chunk>_<parent index> truncated to 63 characters. + /// A copy TimescaleDB had to suffix to keep unique after truncation is missed by that join, which + /// understates the logged figure and changes nothing else. + /// + public static string IntervalDedupMaterializationGroupIndexesSql => + $@" +SELECT + i.schemaname, + i.indexname, + i.indexdef, + pg_relation_size(format('%I.%I', i.schemaname, i.indexname)::regclass) + + COALESCE((SELECT sum(pg_relation_size(format('%I.%I', ci.schemaname, ci.indexname)::regclass)) + FROM timescaledb_information.chunks AS ch + JOIN pg_indexes AS ci + ON ci.schemaname = ch.chunk_schema + AND ci.tablename = ch.chunk_name + AND ci.indexname = left(ch.chunk_name || '_' || i.indexname, 63) + WHERE ch.hypertable_schema = ca.materialization_hypertable_schema + AND ch.hypertable_name = ca.materialization_hypertable_name), 0) AS bytes +FROM timescaledb_information.continuous_aggregates AS ca +JOIN pg_indexes AS i + ON i.schemaname = ca.materialization_hypertable_schema + AND i.tablename = ca.materialization_hypertable_name +WHERE ca.view_schema = 'collect' +AND ca.view_name = '{QueryStoreStatsIntervalHourlyView}' +AND i.indexdef ~ 'USING btree \([a-z_]+, bucket DESC\)$' +ORDER BY i.indexname"; + + /// + /// The lock wait the index drops below will tolerate before giving the start back, as a PostgreSQL + /// lock_timeout literal. DROP INDEX takes AccessExclusiveLock on the materialization and + /// its chunks, and the hourly refresh that this exists to lighten holds RowExclusiveLock on the same + /// relation for its whole run — up to fifteen minutes on the largest store. A drop that queued behind it + /// would not merely wait: a QUEUED exclusive request blocks every later shared request too (the convoy + /// documents), so the three child aggregates' refreshes and the + /// coverage probe would pile up behind a lock that was only requested. Ten seconds is long enough for the + /// lock to be free whenever no refresh is running and short enough that a running refresh costs this start + /// nothing but a warning; the next start retries. Set with SET LOCAL inside each drop's own + /// transaction, so it never outlives the statement it guards. + /// + public const string IntervalDedupIndexDropLockTimeout = "10s"; + + /// + /// Drops the per-column group indexes an earlier build's CREATE MATERIALIZED VIEW left on + /// 's materialization (#3597), so an existing store reaches + /// the shape 's create_group_indexes = false gives + /// a fresh one. Idempotent under the catalog — a settled store reads pg_indexes once and issues + /// nothing — and failure-isolated per index. Returns the number of indexes dropped this start. + /// + /// What was measured and what was not, stated apart because the lever is licensed by the first + /// and not the second. The refresh's cost per re-materialized bucket was measured with and without + /// these indexes on a rig at one tenth of the largest store's scale (the figures are on + /// ): 4.3x the WAL, 1.19 M extra buffer touches and + /// 108x the dirtied buffers per bucket with them, from twelve index inserts per row where one suffices. + /// That is the write amplification, and it is a property of the statement, not of the rig. What the rig + /// could NOT reproduce is the production I/O regime — its indexes fit in shared buffers, so its wall clock + /// barely moved — and so this file does not claim a refresh-duration figure for the largest store. The + /// issue's own job_history series after this lands is that measurement; the trough value + /// (276–286 s at the quietest hours, when only the newly-closed bucket is dirty) is the per-bucket floor + /// this should lower. + /// + /// Why a drop at startup rather than a recreate. The option that keeps a fresh store from + /// building these speaks only at CREATE, and recreating the aggregate would discard seven days of + /// materialization the corrected tiers are gated on. DROP INDEX on the parent hypertable is + /// transactional, propagates to every chunk, and is measured harmless to what remains: the hourly refresh, + /// the three child aggregates' refreshes, compress_chunk on a materialization chunk and + /// decompress_chunk all ran unchanged on the rig with the bucket index alone. + /// + /// Ordering. After (the aggregate must exist) + /// and before (so the nightly compression pass compresses a + /// relation that is already smaller). Under a so a refresh + /// in flight at startup is yielded to rather than convoyed — that arm logs at Warning and the next start + /// retries, which on an hourly grid is at most one refresh away from succeeding. + /// + public static async Task EnsureIntervalDedupMaterializationIndexesAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + var indexes = new List<(string Schema, string Name, string Definition, long Bytes)>(); + try + { + using var probe = new NpgsqlCommand(IntervalDedupMaterializationGroupIndexesSql, connection) { CommandTimeout = SetupTimeoutSeconds }; + await using var reader = await probe.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + indexes.Add(( + reader.GetString(0), + reader.GetString(1), + reader.GetString(2), + reader.IsDBNull(3) ? 0L : Convert.ToInt64(reader.GetValue(3), CultureInfo.InvariantCulture))); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "TimescaleDB: could not read {View}'s materialization indexes, so none was dropped this start — its hourly refresh keeps paying twelve index inserts per re-materialized row until the next start reads it (#3597): {Message}", + QueryStoreStatsIntervalHourlyView, ex.Message); + return 0; + } + + if (indexes.Count == 0) + { + logger?.LogInformation( + "TimescaleDB: {View}'s materialization carries no per-column group index — the bucket index alone, the shape its refresh is cheapest in (#3597)", + QueryStoreStatsIntervalHourlyView); + return 0; + } + + var dropped = 0; + long freed = 0; + foreach (var (schema, name, definition, bytes) in indexes) + { + /* One transaction per index, each with its own lock timeout: a drop that cannot get its lock leaves + the others untried THIS start rather than half-done, because the convoy argument on the timeout + constant applies to every one of them equally — if the first is blocked by a refresh, so are the + rest, and eleven ten-second waits is a startup stalled for two minutes behind a lock it decided + not to wait for. */ + try + { + await using var transaction = await connection.BeginTransactionAsync(cancellationToken); + using (var timeout = new NpgsqlCommand($"SET LOCAL lock_timeout = '{IntervalDedupIndexDropLockTimeout}'", connection, transaction) { CommandTimeout = SetupTimeoutSeconds }) + { + await timeout.ExecuteNonQueryAsync(cancellationToken); + } + + using (var drop = new NpgsqlCommand($"DROP INDEX IF EXISTS {QuoteIdentifier(schema)}.{QuoteIdentifier(name)}", connection, transaction) { CommandTimeout = SetupTimeoutSeconds }) + { + await drop.ExecuteNonQueryAsync(cancellationToken); + } + + await transaction.CommitAsync(cancellationToken); + dropped++; + freed += bytes; + logger?.LogInformation( + "TimescaleDB: dropped {Index} ({SizeMiB:0.#} MiB across the materialization and its chunks) from {View}'s materialization — a per-column group index nothing read, whose maintenance every hourly refresh paid on every re-materialized row (#3597). Definition was: {Definition}", + name, bytes / 1048576d, QueryStoreStatsIntervalHourlyView, definition); + } + catch (PostgresException ex) when (string.Equals(ex.SqlState, PostgresErrorCodes.LockNotAvailable, StringComparison.Ordinal)) + { + logger?.LogWarning( + "TimescaleDB: {Index} on {View}'s materialization could not be dropped within {Timeout} — its hourly refresh is holding the relation, and waiting would queue every reader behind this drop; the remaining {Remaining} group index(es) are left for the next start rather than each waiting its own turn (#3597).", + name, QueryStoreStatsIntervalHourlyView, IntervalDedupIndexDropLockTimeout, indexes.Count - dropped); + break; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "TimescaleDB: could not drop {Index} from {View}'s materialization — its refresh keeps maintaining it until the next restart retries (often a permission issue: the store login must own the materialization) (#3597): {Message}", + name, QueryStoreStatsIntervalHourlyView, ex.Message); + } + } + + logger?.LogInformation( + "TimescaleDB: {Dropped}/{Found} per-column group index(es) dropped from {View}'s materialization this start, {FreedMiB:0.#} MiB released; {Remaining} remain (#3597)", + dropped, indexes.Count, QueryStoreStatsIntervalHourlyView, freed / 1048576d, indexes.Count - dropped); + + return dropped; + } + + /// Double-quotes one SQL identifier, doubling any embedded quote — the catalog names the + /// drops above interpolate are TimescaleDB's own, but a name is a name and gets quoted. + private static string QuoteIdentifier(string identifier) + => "\"" + identifier.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""; + + /// + /// Puts every continuous aggregate this product owns on the compression ladder (#3581): enables columnar + /// compression on each materialization, attaches a once-a-day compression policy on the daily band, stages + /// the first runs one aggregate per night largest first, and converges any policy an earlier build left on + /// different values. Idempotent under the catalog — a settled store runs no ALTER, adds no policy and + /// alters no job — and failure-isolated per aggregate, the + /// shape. Returns the number of aggregates with a compression policy in place afterwards. + /// + /// The omission this closes, with the production numbers that set its stakes. Every raw + /// hypertable has compressed since the archival tier existed (); no + /// continuous aggregate ever did, and nothing in this file said so — an omission, not a decision. On the + /// largest production store the twenty materializations were 235 GiB of a 415 GiB database with every one + /// reading compression_enabled = false: the four Query Store aggregates alone 71.5 / 55.6 / 54.7 / + /// 33.5 GiB against a raw query_store_stats of 25 GiB compressing at ~9x, because the hourly grain is + /// only a ~4x row reduction and an uncompressed rollup of a 9x-compressed source is LARGER than the source. + /// The 90-day hourly tier was a month into its first fill, so uncompressed the store was on course for a + /// 600–650 GB plateau in mid-November; at the raw tier's measured ratio the same plateau is ~200 GB. The + /// rig reproduces the shape in miniature — 1,056 MB of raw compressing to 32 MB while its five Query Store + /// aggregates held 110 / 110 / 179 / 148 / 5 MB uncompressed — and the aggregates compressed at 10.1x to + /// 18.6x once enabled. + /// + /// A startup ensure, not a schema rung, for the reason the retention and raw-compression + /// policies are: ALTER MATERIALIZED VIEW ... SET (timescaledb.compress ...) is idempotent under a + /// catalog check and add_compression_policy has if_not_exists, so this re-converges on every + /// start and contends for nothing on the migration ladder. + /// + /// Ordering. MUST run AFTER — the aggregates have + /// to exist; it then sets their materializations' chunk width () + /// before anything else, and compression has to be enabled on a materialization BEFORE a policy is attached to it (the + /// ALTER precedes the add_compression_policy inside each aggregate's own try, so a failed ALTER costs + /// that aggregate its policy too rather than attaching a policy to an uncompressible relation). It runs + /// BEFORE only because that is where the raw tier's compression + /// sits relative to its retention; nothing here depends on the retention sweep either way. + /// + /// The materialization chunk width, which this design is correct at either value of and which + /// now holds at one raw chunk (#3620). Left to + /// TimescaleDB, a materialization's chunks are ten times the raw hypertable's interval — measured on 2.28.1: + /// a 1-day raw table yields 10-day materialization chunks, hierarchical aggregates take their parent's width + /// (see that method for the exact rule). #3581's staging note describes the production backlog as "~60 1-day + /// chunks" per aggregate; a store whose raw tables were created with = 1 had + /// 10-day materialization chunks instead. The separation argument on + /// holds at any width, because a chunk compresses only when its + /// whole range is past compress_after. What changes with the width is the SHAPE of the work: at 1-day + /// chunks each aggregate compresses one day's chunk every night and the 7-day interval tier spends five of + /// its seven days compressed; at 10-day chunks each aggregate compresses one 10-day chunk every tenth + /// night, a 10-day chunk of the 7-day tier lives seventeen days and is compressed for the last five of + /// them, and the 90-day tiers hold up to 100 days. This ensure therefore narrows the width FIRST, so every + /// chunk the policies it attaches will ever see created is a day wide; the 10-day chunks a store already + /// holds keep their range and compress on the tenth-night shape until they age out. + /// + /// Concurrency with the rest of the store. A compression run holds AccessShareLock on + /// the materialization and escalates only on the chunk it rewrites + /// (), so it cannot block the aggregate's own refresh; a + /// hierarchical daily's refresh READS the compressed hourly region (measured at 578 ms for three days on + /// the rig, working) and a reader of the chunk being swapped waits for the swap. The daily refreshes and the + /// nightly purge (DarlingWorker's _nextPurgeUtc, anchored to service start) run at drifting + /// or per-process instants that no fixed band can avoid by construction; the purge is chunk drops on a + /// TimescaleDB store and the refreshes are read-only against these chunks, so neither is a lock hazard, and + /// that is stated rather than papered over with a schedule that claims to dodge them. + /// + /// What the raw converge must not do to these jobs is handled on its side: + /// skips every + /// job, because its cadence test would otherwise retune this + /// family to the hourly tick on the first start after it exists. The stuck-job check (#1581/#3575) covers + /// these jobs as it covers every compression job, with a of 48 hours at this + /// cadence, and its samples at :30 of the minute stay off these jobs' :00 starts exactly as + /// they stay off the raw band's. + /// + /// The summary line names every aggregate and its window, the #1958 way: an operator + /// cross-checking it against timescaledb_information.jobs should meet every job it promises and no + /// job it does not. + /// + public static async Task EnsureAggregateCompressionAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + /* Width before policies (#3620): the chunk interval governs only chunks created from now on, so the + earlier it is set the fewer 10-day chunks the compression policies below ever have to wait out. Its own + read, its own per-aggregate isolation, its own summary line; a failure there costs no aggregate its + compression. */ + await EnsureMaterializationChunkIntervalAsync(connection, logger, cancellationToken); + + var states = new Dictionary(StringComparer.Ordinal); + try + { + using var probe = new NpgsqlCommand(AggregateCompressionStateSql, connection) { CommandTimeout = SetupTimeoutSeconds }; + await using var reader = await probe.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + var view = reader.GetString(0); + if (!IsAggregateCompressionTarget(view)) + { + continue; + } + + var hourly = AggregateCompressionTargetFor(view).Hourly; + states[view] = new AggregateCompressionState( + View: view, + CompressionEnabled: !reader.IsDBNull(1) && reader.GetBoolean(1), + JobId: reader.IsDBNull(2) ? null : Convert.ToInt32(reader.GetValue(2), CultureInfo.InvariantCulture), + CompressAfterSeconds: reader.IsDBNull(3) ? null : reader.GetInt64(3), + ScheduleIntervalSeconds: reader.IsDBNull(4) ? null : reader.GetInt64(4), + FixedSchedule: !reader.IsDBNull(5) && reader.GetBoolean(5), + PhaseHour: reader.IsDBNull(6) ? null : reader.GetInt32(6), + PhaseMinute: reader.IsDBNull(7) ? null : reader.GetInt32(7), + MaterializationBytes: reader.IsDBNull(8) ? 0L : Convert.ToInt64(reader.GetValue(8), CultureInfo.InvariantCulture), + EligibleChunksNow: Convert.ToInt64(reader.GetValue(hourly ? 9 : 10), CultureInfo.InvariantCulture)); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + /* A plain-PostgreSQL store, or a catalog too old for one of these columns. The caller already gates + on the extension; with no state there is nothing safe to add, and a warning names the cost. */ + logger?.LogWarning( + "TimescaleDB: could not read the continuous aggregates' compression state, so none was enabled or converged this start — the aggregates keep whatever they have (uncompressed, on a store that never had this build) until the next start reads it: {Message}", + ex.Message); + return 0; + } + + var enabled = 0; + var needingPolicy = new List(); + var absent = new List(); + + foreach (var (_, view, _) in AggregateCompressionTargets) + { + if (!states.TryGetValue(view, out var state)) + { + /* The creation sweep is failure-isolated per aggregate and has already warned about this one; + there is no materialization to compress. Named in the summary rather than re-warned. */ + absent.Add(view); + continue; + } + + try + { + if (!state.CompressionEnabled) + { + using var enable = new NpgsqlCommand(EnableAggregateCompressionSql(view), connection) { CommandTimeout = SetupTimeoutSeconds }; + await enable.ExecuteNonQueryAsync(cancellationToken); + state = state with { CompressionEnabled = true }; + states[view] = state; + } + + enabled++; + + if (state.JobId is null) + { + needingPolicy.Add(state); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "Compression could not be enabled on continuous aggregate {View} — it stays uncompressed, at its full materialized size, until the next restart retries: {Message}", + view, ex.Message); + } + } + + var added = 0; + var stagedLines = new List(); + foreach (var (view, nightOffset) in StageAggregateCompressionNights(needingPolicy)) + { + try + { + using var policy = new NpgsqlCommand(AddAggregateCompressionPolicySql(view, nightOffset), connection) { CommandTimeout = SetupTimeoutSeconds }; + var jobId = Convert.ToInt32(await policy.ExecuteScalarAsync(cancellationToken), CultureInfo.InvariantCulture); + added++; + + var state = states[view]; + var hour = AggregateCompressionBandHourFor(view); + stagedLines.Add($"{view} night {nightOffset} at {hour:00}:{AggregateCompressionBandMinute:00}Z ({FormatGiB(state.MaterializationBytes)}, {state.EligibleChunksNow} chunk(s) eligible now)"); + + logger?.LogInformation( + "TimescaleDB: continuous aggregate {View} ({Size}) gets a once-a-day compression policy (job {JobId}, compress_after {CompressAfter}) at {Hour:00}:{Minute:00}Z, first running {Nights} night(s) after the next UTC midnight — {Eligible} chunk(s) are past the window now and that first run compresses all of them, chunk by chunk, in one pass (#3581).", + view, FormatGiB(state.MaterializationBytes), jobId, AggregateCompressAfterFor(view), hour, AggregateCompressionBandMinute, nightOffset, state.EligibleChunksNow); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "Compression policy for continuous aggregate {View} failed — compression is enabled on it but nothing compresses its chunks until the next restart retries: {Message}", + view, ex.Message); + } + } + + var converged = 0; + foreach (var state in states.Values) + { + if (state.JobId is not int jobId) + { + continue; + } + + var wantedSeconds = (long)AggregateCompressAfterSpanFor(state.View).TotalSeconds; + var wantedCadence = (long)AggregateCompressionScheduleSpan.TotalSeconds; + int wantedHour; + try + { + wantedHour = AggregateCompressionBandHourFor(state.View); + } + catch (InvalidOperationException ex) + { + logger?.LogWarning("Continuous aggregate {View}'s compression policy cannot be placed on the daily band: {Message}", state.View, ex.Message); + continue; + } + + if (state.CompressAfterSeconds == wantedSeconds + && state.ScheduleIntervalSeconds == wantedCadence + && state.FixedSchedule + && state.PhaseHour == wantedHour + && state.PhaseMinute == AggregateCompressionBandMinute) + { + continue; + } + + try + { + using var alter = new NpgsqlCommand(SetAggregateCompressionPolicySql, connection) { CommandTimeout = SetupTimeoutSeconds }; + alter.Parameters.AddWithValue(jobId); + alter.Parameters.AddWithValue(AggregateCompressAfterFor(state.View)); + alter.Parameters.AddWithValue(wantedHour); + alter.Parameters.AddWithValue(AggregateCompressionBandMinute); + await alter.ExecuteNonQueryAsync(cancellationToken); + converged++; + + logger?.LogInformation( + "TimescaleDB: moved {View}'s compression policy (job {JobId}) onto compress_after {CompressAfter}, a {Cadence} cadence and a fixed {Hour:00}:{Minute:00}Z schedule (was {WasSeconds}s of compress_after, {WasCadence}s of cadence, fixed_schedule={WasFixed}, {WasHour}:{WasMinute}) — add_compression_policy does not update a policy that already exists (#3581).", + state.View, jobId, AggregateCompressAfterFor(state.View), AggregateCompressionScheduleInterval, wantedHour, AggregateCompressionBandMinute, + state.CompressAfterSeconds, state.ScheduleIntervalSeconds, state.FixedSchedule, state.PhaseHour, state.PhaseMinute); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "Could not converge {View}'s compression policy (job {JobId}) onto the shipped window and band — it keeps its old values (often a permission issue: the store login must own the job): {Message}", + state.View, jobId, ex.Message); + } + } + + var inPlace = states.Values.Count(s => s.JobId is not null) + added; + var hourlyTier = AggregateCompressionTargets.Where(t => t.Hourly).Select(t => t.View).ToArray(); + var dailyTier = AggregateCompressionTargets.Where(t => !t.Hourly).Select(t => t.View).ToArray(); + + /* EVERY aggregate is named with its window, and the windows are interpolated rather than restated + (#1958's rule for the retention line): an operator cross-checking timescaledb_information.jobs meets + thirteen policies at one compress_after and seven at another, and a line that said "the aggregates + compress after 2 days" would be a universal claim with seven counterexamples in the catalog. */ + logger?.LogInformation( + "TimescaleDB: continuous-aggregate compression on {Enabled}/{Total} aggregates, {InPlace} once-a-day policies in place ({Added} added this start, {Converged} converged, {Absent} aggregate(s) absent: {AbsentViews}) — compress_after {HourlyAfter} for the hourly-refreshed tier ({HourlyViews}) and {DailyAfter} for the daily tier ({DailyViews}); one aggregate per hour, minute :{Minute:00}Z, from hour {FirstHour:00}Z; staged first runs, largest first: {Staged}", + enabled, AggregateCompressionTargets.Count, inPlace, added, converged, absent.Count, absent.Count == 0 ? "none" : string.Join(", ", absent), + HourlyAggregateCompressAfter, string.Join(", ", hourlyTier), DailyAggregateCompressAfter, string.Join(", ", dailyTier), + AggregateCompressionBandMinute, AggregateCompressionBandFirstHour, + stagedLines.Count == 0 ? "none this start" : string.Join("; ", stagedLines)); + + return inPlace; + } + + /// A byte count as GiB with one decimal, for the log lines above — the unit the issue's own figures + /// are stated in. + private static string FormatGiB(long bytes) + => (bytes / 1073741824d).ToString("0.0", CultureInfo.InvariantCulture) + " GiB"; + + /* ─────────────── rollup availability (the plain-PostgreSQL guard, #1664) ─────────────── */ + + /// + /// One catalog round trip answering "which retention rollups exist in THIS store?" — the availability + /// input to . to_regclass + /// needs no table privilege and returns NULL for a missing relation, so this is safe under the viewer's + /// least-privilege role and on any store shape. Column order matches + /// 's constructor. + /// + public static readonly string RollupProbeSql = + "SELECT " + + $"to_regclass('collect.{QueryStatsHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStatsDailyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStatsDbHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStatsDbDailyView}') IS NOT NULL, " + + $"to_regclass('collect.{ProcedureStatsHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{ProcedureStatsDailyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStoreStatsHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStoreStatsDailyView}') IS NOT NULL, " + + /* The corrected Query Store rollups (#1849). A store on an older service has none of them and reads + fall back to the pair above — the same per-tier degrade #1664/#1665 built, which is why these need + no schema migration or version gate: existence IS the probe. */ + $"to_regclass('collect.{QueryStoreStatsIntervalHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStoreStatsCorrectedHourlyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStoreStatsCorrectedDailyView}') IS NOT NULL, " + + /* The day-grain daily and its dedup layer (#1869) — the same existence-is-the-probe degrade, so a + store on a #1849-era service keeps reading the corrected daily and needs no version gate either. */ + $"to_regclass('collect.{QueryStoreStatsIntervalDailyView}') IS NOT NULL, " + + $"to_regclass('collect.{QueryStoreStatsDayGrainDailyView}') IS NOT NULL"; + + /// + /// Detects which continuous-aggregate rollups exist in the store (). On a + /// plain-PostgreSQL store every flag is false — and that is a COMPLETE configuration, not a degraded one: + /// without the extension no retention policy ever drops raw, so the raw tables hold full history and + /// routing everything to raw loses nothing. On a TimescaleDB store the worker's ensure sweep creates the + /// views before any reader can need them; a partially-built store (one aggregate's failure-isolated + /// setup failed) reports exactly what exists, so the router degrades per tier instead of a reader + /// throwing 42P01 at a user (#1664, the gated-live catch on #1661's first cut). + /// + public static async Task DetectRollupsAsync(NpgsqlDataSource dataSource, CancellationToken cancellationToken = default) + { + if (dataSource is null) + { + throw new ArgumentNullException(nameof(dataSource)); + } + + await using var command = dataSource.CreateCommand(RollupProbeSql); + command.CommandTimeout = JobCatalogReadTimeoutSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + await reader.ReadAsync(cancellationToken); + return new RollupAvailability( + reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), + reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), + reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), + reader.GetBoolean(11), reader.GetBoolean(12)); + } + + /* ─────────────── rollup COVERAGE (the un-materialized-history guard, #1759) ─────────────── */ + + /// The hourly rollups' bucket width — named so reads as data. + /// Declared BEFORE and that is not cosmetic: C# runs static field + /// initializers in DECLARATION order, so a list declared above these would capture + /// default(TimeSpan) — zero — for every width, and every backfill bucket count would divide by + /// zero-width buckets. Caught by RollupBackfillTests going red on exactly that. + public static readonly TimeSpan HourlyBucket = TimeSpan.FromHours(1); + + /// The daily rollups' bucket width. See on declaration order. + public static readonly TimeSpan DailyBucket = TimeSpan.FromDays(1); + + /// + /// Every rollup view in probe order, with the two DIFFERENT relations it is measured against. One list, so + /// the coverage probe's column order, 's constructor and + /// cannot drift into disagreeing. + /// + /// RawTable and Source are not the same question, and conflating them was #1798. + /// RawTable is where a READ falls back to when this rollup cannot answer a window — always the raw + /// hypertable, because that is the only relation holding per-sweep rows. Source is what this rollup + /// is BUILT FROM, and therefore the most history it can ever contain: raw for the hourlies, but the HOURLY + /// VIEW for every daily, since all four dailies are hierarchical continuous aggregates + /// (time_bucket('1 day', bucket) FROM collect.<x>_hourly). + /// + /// The distinction decides whether a backfill can ever finish. The #1680 arming gate for an + /// HOURLY-tier retention policy is SOURCE-relative — the daily must cover what the hourly holds — while the + /// backfill verb converged every rollup to RAW's oldest row. On a store whose raw purges are armed, raw is + /// a few days deep and the hourlies legitimately hold weeks, so a daily converged "to raw" stops well short + /// of its hourly and the gate stays correctly held while the verb reports DONE. Worse, a hierarchical daily + /// added AFTER its hourly on such a store enters a hold NOTHING can clear: the pre-raw region exists only + /// in the hourly, and a verb aiming at raw never targets it. + /// + /// SourceTimeColumn follows from that: raw tables are keyed on collection_time, + /// rollup views on bucket. + /// + /// BucketWidth is carried EXPLICITLY, not inferred (#1849). Until the corrected Query + /// Store rollups existed, "hierarchical" and "daily" were the same fact, so the backfill derived a rollup's + /// bucket width from whether its source time column was bucket. + /// breaks that: it is hierarchical (sourced from L1) but its buckets are HOURS. Inferring would have given + /// its backfill a 24x-too-wide bucket, so every bucket count, slice count and disk estimate for it would + /// have been silently wrong — an under-estimate, which is the one direction the preflight exists to + /// prevent. Ordering still keys on the source column (raw-sourced rollups must be backfilled before the + /// rollups that read them); only the width became its own column. + /// + public static readonly (string View, string RawTable, string Source, string SourceTimeColumn, TimeSpan BucketWidth)[] RollupViews = + { + (QueryStatsHourlyView, "query_stats", "query_stats", "collection_time", HourlyBucket), + (QueryStatsDailyView, "query_stats", QueryStatsHourlyView, "bucket", DailyBucket), + (QueryStatsDbHourlyView, "query_stats", "query_stats", "collection_time", HourlyBucket), + (QueryStatsDbDailyView, "query_stats", QueryStatsDbHourlyView, "bucket", DailyBucket), + (ProcedureStatsHourlyView, "procedure_stats", "procedure_stats", "collection_time", HourlyBucket), + (ProcedureStatsDailyView, "procedure_stats", ProcedureStatsHourlyView, "bucket", DailyBucket), + (QueryStoreStatsHourlyView, "query_store_stats", "query_store_stats", "collection_time", HourlyBucket), + (QueryStoreStatsDailyView, "query_store_stats", QueryStoreStatsHourlyView, "bucket", DailyBucket), + + /* The corrected Query Store rollups (#1849). L1 is raw-sourced; BOTH corrected views read L1 — the + daily is L1's second child, not the corrected hourly's, so it converges to L1 like its sibling. */ + (QueryStoreStatsIntervalHourlyView, "query_store_stats", "query_store_stats", "collection_time", HourlyBucket), + (QueryStoreStatsCorrectedHourlyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", HourlyBucket), + (QueryStoreStatsCorrectedDailyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", DailyBucket), + + /* The day-grain daily and its dedup layer (#1869) — L1's THIRD child, and the first rollup in this + list whose own source is itself hierarchical. Both are DAY-bucketed, which is why the explicit + BucketWidth above is what keeps the backfill honest here as well. */ + (QueryStoreStatsIntervalDailyView, "query_store_stats", QueryStoreStatsIntervalHourlyView, "bucket", DailyBucket), + (QueryStoreStatsDayGrainDailyView, "query_store_stats", QueryStoreStatsIntervalDailyView, "bucket", DailyBucket), + }; + + /// The three raw tables the rollups roll up, in coverage-probe order (deduplicated + /// ). + public static readonly string[] RolledRawTables = { "query_stats", "procedure_stats", "query_store_stats" }; + + /// + /// How far back each rollup has actually MATERIALIZED, and how far back each raw table still reaches — + /// the input needs to stop routing a window at a rollup that cannot + /// answer it (#1759). + /// + /// The mechanism this exists for: a continuous aggregate created WITH NO DATA over + /// pre-existing history serves ONLY what was materialized. Real-time aggregation cannot rescue it — + /// the watermark is a hard partition (materialized below UNION ALL raw at-or-above), so raw + /// older than the watermark is excluded by construction, not merely un-accelerated. Every rollup's + /// refresh policy only reaches its own start offset back, so on a store that existed before its rollups + /// the materialized span begins at roughly creation minus that offset and NEVER reaches further back on + /// its own. + /// + /// to_regclass-safe by construction, not by guard. A relation named in a statement + /// is resolved at PARSE time, so no in-statement to_regclass test can keep min(bucket) + /// off a view that does not exist. Instead the SQL is BUILT from + /// — a view the round trip just proved + /// absent contributes a literal NULL and is never named. Column count is fixed either way, so + /// the reader's indexing does not depend on the store's shape. + /// + /// min(bucket) is deliberately the SAME expression + /// gates arming on. Routing and arming must agree about what a rollup covers, or the router would + /// serve a window the arming gate considers uncovered (or worse, the reverse). + /// + public static string RollupCoverageProbeSql(RollupAvailability availability) + { + var columns = RollupViews + .Select(r => availability.Has(r.View) + ? $"(SELECT min(bucket) FROM collect.{r.View})" + : "NULL::timestamp") + /* The raw tables are migration-created and always exist, so they need no availability gate. */ + .Concat(RolledRawTables.Select(t => $"(SELECT min(collection_time) FROM collect.{t})")); + + return "SELECT " + string.Join(", ", columns); + } + + /// + /// Reads every rollup's materialized floor and every rolled raw table's oldest row + /// (). comes from + /// in the same probe cycle and decides which relations are named at + /// all. + /// + /// NOTE that a DAILY rollup's floor is the day-FLOOR of its oldest hourly bucket, so it can read + /// up to a day earlier than the hourly it is sourced from. That over-claims coverage by at most one + /// bucket, and it is exactly the semantics the arming gate already runs on — matching it is the point. + /// + public static async Task DetectRollupCoverageAsync( + NpgsqlDataSource dataSource, RollupAvailability availability, CancellationToken cancellationToken = default) + { + if (dataSource is null) + { + throw new ArgumentNullException(nameof(dataSource)); + } + + await using var command = dataSource.CreateCommand(RollupCoverageProbeSql(availability)); + command.CommandTimeout = JobCatalogReadTimeoutSeconds; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + if (!await reader.ReadAsync(cancellationToken)) + { + return RollupCoverage.Unknown; + } + + var floors = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < RollupViews.Length; i++) + { + if (!await reader.IsDBNullAsync(i, cancellationToken)) + { + floors[RollupViews[i].View] = reader.GetDateTime(i); + } + } + + var rawOldest = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < RolledRawTables.Length; i++) + { + var ordinal = RollupViews.Length + i; + if (!await reader.IsDBNullAsync(ordinal, cancellationToken)) + { + rawOldest[RolledRawTables[i]] = reader.GetDateTime(ordinal); + } + } + + return new RollupCoverage(floors, rawOldest); + } + + /// + /// Converts every collector table to a hypertable ( scope; + /// per table). Failure-isolated per table: one failed + /// conversion warns and the sweep continues — that table stays a plain PG table, keeps + /// working (COPY and DELETE-based retention are hypertable-agnostic), and is retried on the + /// next service start. Returns the number of tables that converted (or no-op'd) cleanly. + /// + public static async Task ConvertToHypertablesAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + var converted = 0; + foreach (var schema in HypertableTables) + { + try + { + using var command = new NpgsqlCommand(CreateHypertableSql(schema), connection) { CommandTimeout = SetupTimeoutSeconds }; + await command.ExecuteNonQueryAsync(cancellationToken); + converted++; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning("Hypertable conversion failed for {Table} — it stays a plain table: {Message}", + schema.TargetTable, ex.Message); + } + } + + logger?.LogInformation("TimescaleDB: {Converted}/{Total} collector table(s) are hypertables", + converted, HypertableTables.Count); + return converted; + } + + /// + /// Enables compression and adds the -day background policy on + /// every collector table (both statements per table, failure-isolated per table — a table + /// that failed hypertable conversion warns here too and stays uncompressed). Compressed + /// chunks remain fully queryable: this is Darling's archival tier (see + /// ). Returns the number of tables with a policy in place. + /// + public static async Task ApplyCompressionPolicyAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) { throw new ArgumentNullException(nameof(connection)); } @@ -5421,6 +6719,19 @@ named like one of ours must not inherit its minute. */ && !reader.IsDBNull(6) && string.Equals(reader.GetString(6), PgSchemaGenerator.CollectSchema, StringComparison.Ordinal); + /* The continuous aggregates' compression policies are NOT this converge's to retune (#3581). + Their job reports the aggregate's user view as its hypertable (collect / ), so it + lands in this unscoped read, and its once-a-day cadence would read as stale against the + hourly tick — the first start after that family exists would put all twenty on the hourly + grid, which is the placement #3581 ruled out. They have their own converge + (EnsureAggregateCompressionAsync). Excluded by name on the narrow read, which carries no + schema, and by name-in-collect on the wide one, so a foreign hypertable in another schema + that happens to share an aggregate's name keeps #1778's cadence converge exactly as before. */ + if (IsAggregateCompressionTarget(hypertable) && (!withPhase || ours)) + { + continue; + } + int? phase = ours && hypertable is not null && TryCompressionPhaseMinutesFor(hypertable, out var slot) ? slot : null; @@ -5583,10 +6894,119 @@ public static async Task EnsureCollectionLogHypertableAsync(NpgsqlConnecti /* ---------------- compression-job self-heal (#1581) ---------------- */ + /// + /// The TimescaleDB release from which a persisted next_start = -infinity stopped being a PERMANENT + /// state (#3591): upstream #9360, "Sanitize DT_NOBEGIN next_start to recover jobs stuck after + /// primary failover", shipped in the 2.26.4 patch release (2026-04-28 — the CHANGELOG lists it under + /// 2.26.4, not 2.27.0 as the issue first said) and is therefore in every 2.27+ release as well. + /// is the predicate over it. + /// + /// What the fix changed, from the 2.28.1 source (src/bgw/job_stat.c, + /// ts_bgw_job_stat_next_start). The scheduler computes each job's in-memory next start from + /// its stat row in three arms, in this order: a row with consecutive_crashes > 0 gets the CRASH + /// BACKOFF (below); otherwise, since #9360, a row whose persisted next_start is -infinity is + /// sanitized to "now" and runs at once; otherwise the persisted value stands. Before the fix the second arm + /// did not exist: the sentinel was returned as-is, the scheduler's due-time subtraction on INT64_MIN + /// wrapped, and the job was never due again — the permanent dead state #1581's arm was built against and + /// the field incident that filled a disk. A row reaches that state with consecutive_crashes = 0 through + /// on_failure_to_start_job's next_start != DT_NOBEGIN restore guard, or by inheriting a + /// mid-run row across a primary failover; both are named in the upstream fix's own comment. + /// + /// What a PERSISTENT -infinity is on a fixed store. Only the first arm's row: a worker + /// killed between mark_start and mark_end (a SIGKILL, a crash-restart, a failover — a SIGTERM + /// is caught and marked as a FAILURE with a finite next start) leaves next_start = -infinity, + /// last_finish = -infinity (the view's last_run_status IS NULL) and consecutive_crashes = 1, + /// and the scheduler holds it there, un-persisted, for + /// max(MIN_WAIT_AFTER_CRASH_MS, retry_period × crashes) capped at five schedule intervals and + /// jittered ±13 %: at least FIVE MINUTES, and for a compression policy — whose retry_period defaults + /// to one hour, measured on 2.28.1 — about an hour. Then it re-runs the job itself. (#3575's rig read + /// exactly five minutes because its 10-second probe policy capped the retry term at 50 s; the five-minute + /// floor is the whole backoff only when the retry term is smaller than it.) So on 2.26.4+ the row this + /// product's dead-job arm fires on is a self-recovering condition, not a permanent one, and the alert's + /// sentence has to say which — + /// does, by version. + /// + /// The version is read from pg_extension.extversion by + /// — the first place in this file to read it. Nothing else here declares a TimescaleDB floor (the stated + /// target is "2.x"), and this does not either: a version that cannot be read or parsed is treated as OLD, + /// because "the scheduler will never run it again" is the sentence that costs nothing when wrong on a new + /// store and everything when wrong on an old one. + /// + public static readonly Version TimescaleNextStartSanitizedFrom = new(2, 26, 4); + + /// + /// Whether has upstream #9360 (#3591): true from + /// up, false below it AND for null — an unknown + /// version is the old behaviour, deliberately (see the constant). Pure, so the version arms pin. + /// + public static bool SchedulerRecoversNegativeInfinity(Version? timescaleVersion) + => timescaleVersion is not null && timescaleVersion >= TimescaleNextStartSanitizedFrom; + + /// + /// pg_extension.extversion for timescaledb as a , or null when it cannot + /// be read (#3591). Failure-isolated for the same reason the job-stat read is: this decides a SENTENCE, + /// not whether to page, and a store hiccup on it must fall back to the conservative text rather than + /// fail the check. Debug on failure; the caller already gates on the extension being present. + /// + public static async Task ReadTimescaleVersionAsync( + NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + try + { + using var command = new NpgsqlCommand( + "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'", connection) { CommandTimeout = JobCatalogReadTimeoutSeconds }; + var raw = await command.ExecuteScalarAsync(cancellationToken) as string; + var parsed = ParseTimescaleVersion(raw); + if (parsed is null) + { + logger?.LogDebug("TimescaleDB extversion '{Raw}' did not parse as a version — treating the store as pre-2.26.4 for the compression dead-job text (#3591)", raw ?? "(absent)"); + } + + return parsed; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogDebug("Could not read the TimescaleDB extension version — treating the store as pre-2.26.4 for the compression dead-job text (#3591): {Message}", ex.Message); + return null; + } + } + + /// + /// The numeric prefix of an extversion string as a (#3591): 2.28.1 + /// parses whole; a development suffix (2.29.0-dev) is dropped and the number kept, because the + /// scheduler code in a dev build past 2.26.4 has the fix; anything with fewer than two dotted numbers + /// (null, empty, a bare 2) is null, which the caller reads as OLD. Pure so it pins. + /// + public static Version? ParseTimescaleVersion(string? extversion) + { + if (string.IsNullOrWhiteSpace(extversion)) + { + return null; + } + + var span = extversion.AsSpan().Trim(); + int end = 0; + while (end < span.Length && (char.IsAsciiDigit(span[end]) || span[end] == '.')) + { + end++; + } + + var numeric = span[..end].TrimEnd('.'); + return numeric.Contains('.') && Version.TryParse(numeric, out var version) ? version : null; + } + /// /// The parameterized re-arm statement (#1581): reschedule a background job to run immediately, which /// un-sticks a job whose next_start has become -infinity (the scheduler will never re-fire - /// it otherwise — the field-incident root cause). The job_id is ALWAYS bound as $1, never + /// it otherwise — the field-incident root cause, on TimescaleDB below 2.26.4; see + /// for what the same row is above it, and + /// for why this statement must NOT be run against it + /// there — measured, it resets the scheduler's crash backoff rather than shortening it, #3591). The job_id is ALWAYS bound as $1, never /// interpolated (the discipline is uniform with DarlingRetention's parameterized paths); now() is /// SQL, not a value. It is cast $1::integer because TimescaleDB's alter_job takes /// job_id integer, but is a long that Npgsql sends as @@ -5629,17 +7049,140 @@ public static TimeSpan StuckRunningBound(TimeSpan? scheduleInterval) return s_stuckRunningFloor; } + /// + /// How long + /// waits before it RE-READS a job whose -infinity arm tripped, and requires the trip to persist + /// (#3575). Only taken when that arm trips; a pass with nothing to confirm costs nothing. + /// + /// Why a confirm-read exists at all. The -infinity arm already carried a running + /// guard (nextStartIsNegativeInfinity && !isRunning) and a false page came through it + /// anyway, on a production store, with the alert's stamp 53 ms inside a 63 ms scheduled run that + /// succeeded. The guard's two inputs are read from INDEPENDENT sources inside TimescaleDB's own view, + /// and the 2.28.1 definition (pg_get_viewdef('timescaledb_information.job_stats'), read live) + /// says so exactly: job_status is CASE WHEN pgs.state = 'active' THEN 'Running' WHEN + /// j.scheduled = false THEN 'Paused' ELSE 'Scheduled' END over a LEFT JOIN pg_stat_activity pgs + /// ON pgs.application_name = j.application_name, while next_start is + /// _timescaledb_internal.bgw_job_stat.next_start. The scheduler's mark_start writes + /// next_start = -infinity (and last_finish = -infinity) in its OWN transaction and commits + /// it BEFORE the worker process is even registered; the worker then has to start, initialise its + /// connection, run one catalog transaction, report its application_name and finally call + /// pgstat_report_activity(STATE_RUNNING) before the join can say Running. Every read that + /// lands in that START EDGE sees -infinity AND Scheduled, which is this predicate's + /// dead-job arm. There is an END EDGE too, with a different cause: the catalog row is read under the + /// statement's MVCC snapshot while pg_stat_activity is read live from shared memory, so one SELECT + /// can pair a pre-mark_end row (-infinity) with a post-exit activity view (no backend, so + /// Scheduled). Both edges were captured on a PG18 + TimescaleDB 2.28.1 rig by polling + /// in a tight loop across a 10-second-cadence policy: every run + /// showed ~3 ms of -infinity + Scheduled before the first Running sample and one more such + /// sample after the last, in a run ~7.5 ms long end to end. On a Windows store — where the production + /// page came from — backend process creation is far slower than a Linux fork, so the start edge is a + /// larger share of a run that is itself only tens of milliseconds when there is nothing to compress. + /// + /// Why a confirm-read and not a same-source running signal. The stat row DOES carry its own + /// mid-run marker — mark_start sets last_finish = -infinity, which the view surfaces as + /// last_run_status IS NULL and last_run_duration IS NULL (the duration is + /// CASE WHEN js.last_finish > js.last_start in every sql/views.sql from 2.14 through + /// 2.28.1, so it reads NULL mid-run and never negative; the belief that it "goes negative" is not borne + /// out by any version checked) — and deriving "running" from it would make both inputs one row. It was + /// rejected because that marker is IDENTICAL for a run whose worker died + /// before mark_end ever ran, which is precisely the state this arm exists to catch: reproduced on + /// the rig by SIGKILLing a job's worker, after which the row read -infinity + Scheduled + + /// last_run_status NULL for the whole five-minute crash backoff. A same-source guard would have read + /// that as "running" and stayed silent on the one state it is for. Re-reading after a delay makes no such + /// assumption: an edge is over in milliseconds, a dead row is still dead seconds later. + /// + /// Why five seconds. The transient it has to outlast is bounded by the run's own edges: the + /// start edge is one process start plus one catalog transaction (~3 ms measured on Linux; tens of + /// milliseconds is the realistic Windows figure, and a pathological second is still covered five times + /// over), and a short run bounds the whole exposure at its own duration (40–100 ms is what a production + /// store's hourly no-op compressions measure). Against what it costs, five seconds is 1/720 of the hourly + /// cadence and 1/4,320 of the six-hour floor, so a genuinely dead job is + /// detected on the same hourly pass it always was, five seconds later. And it is far below the shortest + /// PERSISTENT -infinity state there is: TimescaleDB's crash backoff holds the row at + /// -infinity for at least MIN_WAIT_AFTER_CRASH_MS (five minutes) before the scheduler + /// re-runs a crashed job, so a real crash cannot slip between the two reads. The check is AWAITED on the + /// worker's serial sweep loop (#2327's concern), so the delay is bounded, cancellable, and paid only on + /// the rare pass where the arm tripped at all. + /// + public static readonly TimeSpan StuckCompressionConfirmDelay = TimeSpan.FromSeconds(5); + + /// + /// The compression-job health check's wall-clock phase (#3575): how many seconds past a minute boundary + /// the hourly check is pinned to. applies it. + /// + /// Why the check needs a phase at all, stated against how the two schedules are really + /// anchored — which is not how the postmortem first described them. Every compression policy this + /// product owns runs on a FIXED schedule whose initial_start is date_trunc('hour', now()) + + /// 1 hour + <phase minutes> (, #3035), so its runs + /// begin at :MM:00.000 of the wall clock for the 24 minutes of + /// — three jobs to a minute, every hour, on every store. The continuous aggregates' compression policies + /// (#3581) are anchored the same way at :MM:00.000 of their own minute + /// (), one job an hour once a day, so the half-step below is + /// as far from their starts as it is from the raw band's. The health check, by contrast, was anchored to + /// nothing in the wall clock: its first sample was the first sweep pass after service start and each + /// later one was scheduled as UtcNow + 1 hour at the moment of the previous fire, which lands on + /// the first 15-second sweep pass at or after that instant. So its second-of-the-hour SLIPPED forward by + /// the loop's latency every hour — a few seconds to fifteen — and swept across every minute boundary in + /// the band in turn, roughly one boundary every twelve hours. The service behind the production page + /// started at :46:13; seven hourly slips later its sample sat 53 ms past :47:00, the minute + /// file_io_stats compresses on. The schedules were therefore NOT aligned by construction — a + /// service-start anchor drifts, a wall-clock anchor does not — and the collision was the ~1-in-a-hundred + /// draw a drifting sample takes each time it crosses a boundary: the crossing is certain, the landing + /// inside a ~60 ms window is chance. That is still a false page every month or two per store, forever, + /// which is what "structural" correctly meant. + /// + /// Why thirty seconds and not an offset from the service start. Any offset from a DRIFTING + /// anchor drifts with it, so "+N minutes from start" would cross the same boundaries N minutes later. + /// The only offset that holds is one measured from the jobs' own grid, and the grid step is one minute + /// with every job at :00 of its minute — so half a step, :30, is the point furthest from + /// every job's start instant in both directions: 30 s from the previous boundary and 30 s from the next, + /// against edges measured in milliseconds and runs measured in tens of milliseconds when there is nothing + /// to compress. (A run that IS compressing a chunk lasts minutes and straddles :30, but a job that + /// long reports Running, which the -infinity arm already yields to.) Pinning to the wall + /// clock rather than the previous fire is what stops the drift: + /// floors the hourly due time to its minute and adds this phase, so a fire at :47:3x schedules + /// the next at exactly :47:30. A fire that lands LATE in its minute (a slow sweep pass) snaps the + /// next due back to :30 of that same minute, a few seconds short of a full hour; one that lands in + /// the NEXT minute moves the check one whole minute later. Either way the sample is at :30, and + /// nothing the loop does can walk it toward :00. + /// + /// The first check after a restart is deliberately NOT phased — it runs on the first sweep pass, + /// because a restart is when an operator is reading the log and wants the store's job health now — so + /// that single sample keeps the pre-#3575 odds (24 minutes × ~0.1 s of edge in 3,600 s, under 0.1 %), + /// and covers it the same way it covers every other sample. + /// The phase is the hardening; the confirm-read is the fix. + /// + public const int CompressionCheckPhaseSeconds = 30; + + /// + /// When the compression-job health check should next run (#3575): one after + /// , snapped to past that minute so no + /// steady-state sample is ever taken on the :MM:00 instant the compression policies fire on. + /// Pure, so it pins. The snap moves the due time by at most 30 s either way, so the cadence stays + /// hourly to within the loop's own latency; what it never does is land on :00. + /// + public static DateTime NextCompressionCheckUtc(DateTime nowUtc, TimeSpan interval) + { + var due = nowUtc + interval; + var minute = new DateTime(due.Ticks - (due.Ticks % TimeSpan.TicksPerMinute), DateTimeKind.Utc); + return minute.AddSeconds(CompressionCheckPhaseSeconds); + } + /// /// The pure stuck-compression-job decision (#1581). A compression policy job is STUCK when either: /// - /// its next_start is -infinity while the job is NOT currently running — the scheduler - /// abandoned it and will NEVER re-fire it (the dead-job bug that let uncompressed data grow without bound - /// until the disk filled), or + /// its next_start is -infinity while the job is NOT currently running — on TimescaleDB + /// below 2.26.4 the scheduler abandoned it and will NEVER re-fire it (the dead-job bug that let uncompressed + /// data grow without bound until the disk filled); from 2.26.4 on (upstream #9360, + /// ) it is a crashed run in the scheduler's crash backoff, + /// which the scheduler clears by itself (#3591) — the same arm, a different sentence, and no re-arm; or /// it has been in the Running state since a last_run_started_at older than /// — a run that began long ago and never finished (a hung run). /// /// A job with neither condition is healthy and is NOT flagged. No I/O, so it pins directly with a /// controllable clock. Scoping to compression jobs happens in the query — this decides only "stuck". + /// is the same decision naming WHICH arm fired, for the caller that + /// has to treat the two arms differently. /// /// -infinity is ALSO the engine's mid-run marker, measured live on TimescaleDB /// 2.x (pg17): from the moment the scheduler picks up a due job until its run completes, @@ -5651,11 +7194,47 @@ public static TimeSpan StuckRunningBound(TimeSpan? scheduleInterval) /// left to the second arm, whose elapsed bound is what actually distinguishes a hung run from a /// healthy one. /// + /// And the running guard is itself a non-atomic read (#3575). job_status comes from + /// pg_stat_activity and next_start from bgw_job_stat; the scheduler commits + /// -infinity before the worker exists, and the worker is gone before its mark_end is + /// visible to a snapshot taken a moment earlier, so at both edges of every run the view reads + /// -infinity AND Scheduled — this arm, on a healthy job, for a few milliseconds an hour. + /// A production store paged on exactly that: the alert stamp sat 53 ms inside a 63 ms run that + /// succeeded. This predicate stays pure and single-shot on purpose; the caller closes the race by + /// re-reading after and requiring the -infinity arm + /// to persist (), + /// and the worker keeps its samples off the jobs' run instant (). + /// /// A of counts as NEVER RAN, /// not as "started in year 1" (#1760). already NULLIFs TimescaleDB's /// -infinity never-ran sentinel, so this is the second line of defence: the sentinel maps to /// MinValue through Npgsql, and any future caller reading the column un-guarded would otherwise compute a /// ~739,000-day elapsed that clears every bound and flag a healthy job on its very first run. + /// + /// "Will never run it again" is true of TimescaleDB below 2.26.4 and false above it (#3591). + /// Upstream #9360 () made the scheduler sanitize a persisted + /// -infinity, so on a fixed store the only PERSISTENT -infinity is a crashed run sitting out + /// its crash backoff, which the scheduler clears by itself. The VERDICT is the same on every version — the + /// row is still reported, still re-armed once, still paged, because the arm is also the backstop for the + /// older 2.x stores the compatibility target admits and for whatever the next upstream regression is — but + /// the SENTENCE differs, and this overload without a version says the old one. Production goes through + /// + /// with the version read; the version-less form is the pre-#3591 + /// pins' entry point and the "unknown version" case, which are the same text. + /// + public static bool IsCompressionJobStuck( + bool nextStartIsNegativeInfinity, + string? jobStatus, + DateTime? lastRunStartedAtUtc, + TimeSpan? scheduleInterval, + DateTime nowUtc, + out string reason) + => ClassifyCompressionJob(nextStartIsNegativeInfinity, jobStatus, lastRunStartedAtUtc, scheduleInterval, nowUtc, timescaleVersion: null, out reason) + != StuckCompressionJobArm.None; + + /// + /// with the store's TimescaleDB version, so the -infinity arm's + /// reason tells the truth for that version (#3591). null is "unknown, assume old". /// public static bool IsCompressionJobStuck( bool nextStartIsNegativeInfinity, @@ -5663,14 +7242,71 @@ public static bool IsCompressionJobStuck( DateTime? lastRunStartedAtUtc, TimeSpan? scheduleInterval, DateTime nowUtc, + Version? timescaleVersion, + out string reason) + => ClassifyCompressionJob(nextStartIsNegativeInfinity, jobStatus, lastRunStartedAtUtc, scheduleInterval, nowUtc, timescaleVersion, out reason) + != StuckCompressionJobArm.None; + + /// + /// with the arm named (#3575): the confirm-read applies ONLY to + /// , because that is the arm whose inputs + /// race; is judged on six hours of elapsed time and + /// a second read five seconds later could not change it. Same decision, same reason text — this is the + /// implementation and the boolean is its projection, so the two cannot drift. Version-less: the + /// -infinity reason is the pre-2.26.4 text (#3591); see the overload below. + /// + public static StuckCompressionJobArm ClassifyCompressionJob( + bool nextStartIsNegativeInfinity, + string? jobStatus, + DateTime? lastRunStartedAtUtc, + TimeSpan? scheduleInterval, + DateTime nowUtc, + out string reason) + => ClassifyCompressionJob(nextStartIsNegativeInfinity, jobStatus, lastRunStartedAtUtc, scheduleInterval, nowUtc, timescaleVersion: null, out reason); + + /// + /// The -infinity arm's reason on a TimescaleDB below , + /// or of unknown version: the #1581 sentence, byte for byte, because on those stores it is true — the + /// scheduler returns the sentinel as the due time and the job is never due again. + /// + public const string NextStartNegativeInfinityPermanentReason = + "next_start is -infinity — the scheduler will never run it again"; + + /// + /// The -infinity arm's reason on a TimescaleDB with upstream #9360 (#3591): the row is a crashed + /// run in the scheduler's crash backoff, and the scheduler clears it without help. Names the fix and the + /// floor so an operator reading the page knows which condition they are looking at and where the claim + /// comes from; how long the backoff is, and what the re-arm does to it, is the evaluator's detail text. + /// + public const string NextStartNegativeInfinityCrashBackoffReason = + "next_start is -infinity — a crashed run left the job in the scheduler's crash backoff, which the scheduler clears on its own (TimescaleDB 2.26.4+, upstream #9360)"; + + /// + /// with the + /// store's TimescaleDB version (#3591). The -infinity arm fires on exactly the same inputs whatever + /// the version — the confirm-read, the re-arm, the alert key and the severity all see one arm — and only + /// its changes: when + /// says the scheduler has the fix, + /// below it and for null. The stuck-Running + /// arm does not read the version; nothing about a hung run changed upstream. + /// + public static StuckCompressionJobArm ClassifyCompressionJob( + bool nextStartIsNegativeInfinity, + string? jobStatus, + DateTime? lastRunStartedAtUtc, + TimeSpan? scheduleInterval, + DateTime nowUtc, + Version? timescaleVersion, out string reason) { var isRunning = string.Equals(jobStatus, "Running", StringComparison.OrdinalIgnoreCase); if (nextStartIsNegativeInfinity && !isRunning) { - reason = "next_start is -infinity — the scheduler will never run it again"; - return true; + reason = SchedulerRecoversNegativeInfinity(timescaleVersion) + ? NextStartNegativeInfinityCrashBackoffReason + : NextStartNegativeInfinityPermanentReason; + return StuckCompressionJobArm.NextStartNegativeInfinity; } if (isRunning @@ -5685,12 +7321,12 @@ public static bool IsCompressionJobStuck( CultureInfo.InvariantCulture, "stuck in the Running state for {0:F0} minutes (over the {1:F0}-minute bound) — the run hung and never finished", elapsed.TotalMinutes, bound.TotalMinutes); - return true; + return StuckCompressionJobArm.RunningPastBound; } } reason = ""; - return false; + return StuckCompressionJobArm.None; } /// @@ -5711,6 +7347,27 @@ public static bool IsCompressionJobStuck( /// while last_run_started_at is bgw_job_stat.last_start. A job's FIRST run therefore reads /// Running while its start time is still the sentinel, and that window flagged a perfectly healthy /// job as stuck — which the self-heal then "fixed" by re-arming a job that was running fine. + /// + /// The same two sources are why one execution of this statement cannot be trusted alone on the + /// -infinity arm (#3575). next_start_neg_infinity is the stat row under the statement's + /// snapshot; job_status is pg_stat_activity read live. At the start of every run the row + /// already says -infinity while no backend yet says Running, and at the end the backend can + /// be gone while the snapshot still holds the pre-mark_end row. No rewrite of this SELECT closes + /// that — the skew is between a catalog snapshot and live shared memory inside TimescaleDB's own view — + /// which is why + /// executes it TWICE, apart, when that arm trips. The text is + /// unchanged from #1760; what changed is how many times it is asked. + /// + /// What a confirmed next_start_neg_infinity row IS depends on the store's TimescaleDB + /// (#3591), and this statement does not carry that. Below 2.26.4 it is the permanent dead state #1581 + /// was built against. From 2.26.4 (upstream #9360, ) the + /// scheduler sanitizes a persisted -infinity to "now", so the only row that stays -infinity + /// across the confirm delay is a crashed run in the scheduler's crash backoff, which the view shows as + /// -infinity + Scheduled + last_run_status IS NULL — and that last_run_status IS NULL is NOT a + /// discriminator between the two, because both are a mark_start whose mark_end never came. + /// The discriminator is pg_extension.extversion, read separately by + /// on the pass where the arm trips, and applied to the verdict's + /// text and re-arm by . /// public const string StuckCompressionJobsSql = @" SELECT @@ -5734,8 +7391,34 @@ WHERE j.proc_name LIKE '%compression%' /// every other job type are untouched. Failure-isolated: a store hiccup, or the views being absent (a /// plain-PostgreSQL store — the caller also gates on the extension), yields an empty list and a Debug line, /// never a throw. - /// - public static async Task> ReadStuckCompressionJobsAsync( + /// + /// The -infinity arm is CONFIRMED before it is reported (#3575). When, and only when, + /// the first read flags a job on that arm, this waits , runs + /// once more, and reports the job only if the same arm trips + /// again. A run-instant edge — the view pairing the scheduler's already-committed -infinity with a + /// worker that is not yet, or no longer, visible as Running — is over in milliseconds and clears; + /// a row the scheduler has genuinely abandoned (TimescaleDB below 2.26.4), or a crashed run sitting out its + /// crash backoff (the only persistent -infinity from 2.26.4 on, #3591 — reported with + /// set so the evaluator neither re-arms nor pages it on + /// first sight), reads the same on both passes and is reported with the latency it always had plus five seconds. One + /// confirm per pass, not per job: the delay is taken once however many jobs tripped. The + /// arm is reported from the first read as before — + /// a six-hour elapsed bound has nothing to gain from a second look five seconds later. + /// + /// A confirm read that FAILS confirms nothing. Its -infinity trips are dropped for + /// this pass, with a Warning naming the jobs and the consequence, rather than reported on the strength of + /// the one read this issue proved insufficient: compression is a slow archival tier where a dead job + /// takes hours to matter, so deferring a real detection to the next hourly pass costs little, while a + /// false page on the family that reports the store's own health is the very thing being fixed. The + /// stuck-Running results from the first read are still returned. Like the first read's own catch, this + /// swallow is logged but not counted by the #3013 read-failure surface — that census covers the worker's + /// catch blocks, and both reads sit one level below it. + /// + /// The gated live test polls this method until a job it just re-armed reads healthy, and the + /// confirm only makes that settle sooner: the mid-run marker it used to have to wait out is now judged + /// twice and cleared inside one call instead of surfacing as a flagged poll. + /// + public static Task> ReadStuckCompressionJobsAsync( NpgsqlConnection connection, DateTime nowUtc, ILogger? logger, CancellationToken cancellationToken = default) { if (connection is null) @@ -5743,39 +7426,240 @@ public static async Task> ReadStuckCompressio throw new ArgumentNullException(nameof(connection)); } - var stuck = new List(); + return ReadStuckCompressionJobsAsync( + ct => ReadCompressionJobStatRowsAsync(connection, ct), + Task.Delay, + nowUtc, + logger, + cancellationToken, + ct => ReadTimescaleVersionAsync(connection, logger, ct)); + } + + /// + /// The seam + /// is built on, with the two things a test needs to control injected: the read (so a transient edge and + /// a persistent dead row can each be scripted as a pair of result sets, and a failing confirm as a throw) + /// and the delay (so the pin can assert it is taken exactly when the -infinity arm tripped and + /// never otherwise, without sleeping). Internal rather than private for that reason alone; production + /// reaches it only through the connection overload. + /// + /// (#3591) is the store's TimescaleDB version, read ONLY on a pass + /// where the -infinity arm tripped — it decides that arm's sentence and nothing else, so the common + /// hourly pass still costs one read. Omitted (the pre-#3591 pins) or failing, the version is unknown and + /// the sentence is the conservative pre-2.26.4 one. Read before the confirm delay, on the same connection + /// the first read used, so the two reads that decide the page are not pushed further apart. + /// + internal static async Task> ReadStuckCompressionJobsAsync( + Func>> readRows, + Func delay, + DateTime nowUtc, + ILogger? logger, + CancellationToken cancellationToken, + Func>? readVersion = null) + { + if (readRows is null) + { + throw new ArgumentNullException(nameof(readRows)); + } + + if (delay is null) + { + throw new ArgumentNullException(nameof(delay)); + } + try { - using var command = new NpgsqlCommand(StuckCompressionJobsSql, connection) { CommandTimeout = JobCatalogReadTimeoutSeconds }; + var first = ClassifyStuckCompressionJobs(await readRows(cancellationToken), nowUtc); + if (!first.Any(f => f.Arm == StuckCompressionJobArm.NextStartNegativeInfinity)) + { + /* Nothing on the racing arm: no delay, no second read, no version read. The common hourly + pass costs exactly what it did before #3575. */ + return first.Select(f => f.ToJob()).ToList(); + } - await using var reader = await command.ExecuteReaderAsync(cancellationToken); - while (await reader.ReadAsync(cancellationToken)) + /* #3591: the sentence the confirmed row will carry depends on whether this store's scheduler + recovers a -infinity row by itself. ReadTimescaleVersionAsync is failure-isolated to null, and + a scripted reader that throws is treated the same way here — the version decides words, never + the verdict, so it must not be able to fail the pass. */ + Version? timescaleVersion = null; + if (readVersion is not null) { - long jobId = Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture); - bool negInfinity = !reader.IsDBNull(1) && reader.GetBoolean(1); - string? jobStatus = reader.IsDBNull(2) ? null : reader.GetString(2); - DateTime? lastRunStartedAt = reader.IsDBNull(3) - ? null - : DateTime.SpecifyKind(reader.GetDateTime(3), DateTimeKind.Utc); - TimeSpan? scheduleInterval = reader.IsDBNull(4) - ? null - : TimeSpan.FromSeconds(Convert.ToDouble(reader.GetValue(4), CultureInfo.InvariantCulture)); - string? hypertable = reader.IsDBNull(5) ? null : reader.GetString(5); - - if (IsCompressionJobStuck(negInfinity, jobStatus, lastRunStartedAt, scheduleInterval, nowUtc, out var reason)) + try + { + timescaleVersion = await readVersion(cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) { - stuck.Add(new StuckCompressionJob(jobId, hypertable, reason)); + logger?.LogDebug("Could not read the TimescaleDB extension version — treating the store as pre-2.26.4 for the compression dead-job text (#3591): {Message}", ex.Message); } } + + await delay(StuckCompressionConfirmDelay, cancellationToken); + + IReadOnlyList? confirm; + try + { + confirm = await readRows(cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + /* The views were readable seconds ago, so this is a store hiccup on the one read that decides + whether to page — say so at Warning, name what is deferred, and judge it next hour. */ + confirm = null; + logger?.LogWarning( + "Compression-job health check: {Count} job(s) read next_start = -infinity while not Running, but the confirm read {Delay:F0} s later failed — not judged this pass, re-checked next hour (#3575): {Message}", + first.Count(f => f.Arm == StuckCompressionJobArm.NextStartNegativeInfinity), + StuckCompressionConfirmDelay.TotalSeconds, + ex.Message); + } + + return ConfirmStuckCompressionJobs(first, confirm, nowUtc, logger, timescaleVersion); } catch (Exception ex) when (ex is not OperationCanceledException) { /* The views are absent (a plain-PG store or the extension was removed) or the store hiccuped — no signal this check. The caller already gates on the extension; this is belt-and-suspenders. */ logger?.LogDebug("Compression-job health check: could not read job stats: {Message}", ex.Message); + return Array.Empty(); + } + } + + /// + /// The pure merge of a first pass with its confirm pass (#3575), separated so the decision table pins + /// without a clock or a store: + /// + /// from the first pass — reported, untouched by + /// the confirm. + /// from the first pass, and the SAME arm + /// on the confirm pass — reported, carrying the confirm pass's reason (the two are identical today; the + /// later read is the one that stood). + /// That arm on the first pass but not on the confirm — the run-instant edge; cleared, and logged at + /// Information because a person reading the log after this alert family fires deserves to find the near + /// miss, and it is rare enough (a few a year per store) never to be noise. + /// null (the confirm read failed) — every -infinity trip is + /// dropped; the caller has already logged why. + /// + /// A job that appears on the confirm pass but not the first is not reported either: the confirm exists to + /// ratify the first pass, not to widen it, and a job that only just went -infinity gets its own + /// two reads next hour. + /// + /// (#3591) phrases the confirmed rows' reason for the store's + /// TimescaleDB; it is applied to the CONFIRM pass's classification because that is the row carried. The + /// first pass is classified without it on purpose — the arm is version-independent, and the first pass + /// only decides whether there is anything to confirm. + /// + internal static IReadOnlyList ConfirmStuckCompressionJobs( + IReadOnlyList first, + IReadOnlyList? confirm, + DateTime nowUtc, + ILogger? logger, + Version? timescaleVersion = null) + { + if (first is null) + { + throw new ArgumentNullException(nameof(first)); } - return stuck; + var result = new List(first.Count); + var confirmed = confirm is null + ? null + : ClassifyStuckCompressionJobs(confirm, nowUtc, timescaleVersion) + .Where(c => c.Arm == StuckCompressionJobArm.NextStartNegativeInfinity) + .ToDictionary(c => c.Row.JobId); + + foreach (var flagged in first) + { + switch (flagged.Arm) + { + case StuckCompressionJobArm.RunningPastBound: + result.Add(flagged.ToJob()); + break; + + case StuckCompressionJobArm.NextStartNegativeInfinity: + if (confirmed is null) + { + break; + } + + if (confirmed.TryGetValue(flagged.Row.JobId, out var still)) + { + result.Add(still.ToJob()); + } + else + { + logger?.LogInformation( + "Compression-job health check: job {JobId}{Hypertable} read next_start = -infinity while not Running, and {Delay:F0} s later it was scheduled normally — the run-instant edge of TimescaleDB's job_stats view, not a stuck job; nothing re-armed, nothing alerted (#3575)", + flagged.Row.JobId, + string.IsNullOrEmpty(flagged.Row.HypertableName) ? "" : " on " + flagged.Row.HypertableName, + StuckCompressionConfirmDelay.TotalSeconds); + } + + break; + } + } + + return result; + } + + /// + /// One pass of the pure predicate over a result set: every row it flags, with the arm that fired. Rows + /// the predicate clears are not returned. (#3591) phrases the + /// -infinity arm's reason; null is the pre-2.26.4 text. + /// + internal static List ClassifyStuckCompressionJobs( + IReadOnlyList rows, DateTime nowUtc, Version? timescaleVersion = null) + { + if (rows is null) + { + throw new ArgumentNullException(nameof(rows)); + } + + var flagged = new List(); + foreach (var row in rows) + { + var arm = ClassifyCompressionJob( + row.NextStartIsNegativeInfinity, row.JobStatus, row.LastRunStartedAtUtc, row.ScheduleInterval, nowUtc, timescaleVersion, out var reason); + if (arm != StuckCompressionJobArm.None) + { + flagged.Add(new ClassifiedCompressionJob( + row, arm, reason, + SchedulerRetries: arm == StuckCompressionJobArm.NextStartNegativeInfinity && SchedulerRecoversNegativeInfinity(timescaleVersion))); + } + } + + return flagged; + } + + /// + /// One execution of , mapped row for row and NOT failure-isolated: + /// the isolation belongs to the caller, which has to tell a failed FIRST read (no signal, Debug) from a + /// failed CONFIRM read (a deferred judgement, Warning). Both -infinity tests already ran in SQL; + /// the #1760 sentinel arrives here as a NULL. + /// + private static async Task> ReadCompressionJobStatRowsAsync( + NpgsqlConnection connection, CancellationToken cancellationToken) + { + var rows = new List(); + using var command = new NpgsqlCommand(StuckCompressionJobsSql, connection) { CommandTimeout = JobCatalogReadTimeoutSeconds }; + + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + long jobId = Convert.ToInt64(reader.GetValue(0), CultureInfo.InvariantCulture); + bool negInfinity = !reader.IsDBNull(1) && reader.GetBoolean(1); + string? jobStatus = reader.IsDBNull(2) ? null : reader.GetString(2); + DateTime? lastRunStartedAt = reader.IsDBNull(3) + ? null + : DateTime.SpecifyKind(reader.GetDateTime(3), DateTimeKind.Utc); + TimeSpan? scheduleInterval = reader.IsDBNull(4) + ? null + : TimeSpan.FromSeconds(Convert.ToDouble(reader.GetValue(4), CultureInfo.InvariantCulture)); + string? hypertable = reader.IsDBNull(5) ? null : reader.GetString(5); + + rows.Add(new CompressionJobStatRow(jobId, negInfinity, jobStatus, lastRunStartedAt, scheduleInterval, hypertable)); + } + + return rows; } /// @@ -6084,7 +7968,7 @@ public static void LogHeaviestRefreshSlotHeadroom(HeaviestRefreshSlotReading? re case RefreshSlotHeadroom.ApproachingSlot: logger.LogWarning( - "TimescaleDB: {View}'s hourly refresh last ran {Seconds:F0}s against the {Slot}s refresh slot it has to fit inside ({Percent:F1}% of it, {Clear:F0}s clear) — past the {Warn}s watch line. These runtimes scale with raw data volume, and at the slot width the compression phase grid has to be re-derived rather than renumbered (#3035). Its per-run history is timescaledb_information.job_history, one row per run, but only where timescaledb.enable_job_execution_logging is on — it is off by default, and a store provisioned before that GUC gained its own conf marker reports nothing there until it heals, so an empty result is that gap and not a quiet hour (#3175/#3177). The hourly collect.store_metrics series (object_kind = 'background_job') samples one reading an hour and serves a daily point that is the day's LAST, so neither answers a maximum question on its own (#3044, #3119).", + "TimescaleDB: {View}'s hourly refresh last ran {Seconds:F0}s against the {Slot}s refresh slot it has to fit inside ({Percent:F1}% of it, {Clear:F0}s clear) — past the {Warn}s watch line. These runtimes scale with raw data volume, and at the slot width the compression phase grid has to be re-derived rather than renumbered (#3035). Its per-run history is timescaledb_information.job_history, one row per run, but only where timescaledb.enable_job_execution_logging is on — it is off by default, and with it off only FAILED runs are written (a failure is logged regardless of the GUC; a success needs it) — so a store provisioned before that GUC gained its own conf marker shows this job's successes there only once it heals, and a result holding no successful runs is that gap and not a quiet hour (#3175/#3177). The hourly collect.store_metrics series (object_kind = 'background_job') samples one reading an hour and serves a daily point that is the day's LAST, so neither answers a maximum question on its own (#3044, #3119).", reading.View, reading.LastRunSeconds, RefreshPhaseSlotSeconds, reading.PercentOfSlot, reading.ClearOfSlotSeconds, RefreshSlotWarningSeconds); break; @@ -6297,6 +8181,21 @@ public static async Task> ReadRetentionHoldR /// reads Running while its start is still the sentinel. Un-guarded, this observability path would /// report that healthy first run as having been going for ~739,000 days. The two queries were written in /// parallel branches and each was green on its own; this is the seam between them, not either side. + /// + /// The backlog is counted on the relation the chunks actually belong to, at the job's OWN delay + /// (#3581). A continuous aggregate's compression job reports the aggregate's user view as its + /// hypertable, while timescaledb_information.chunks knows the aggregate only by its materialization + /// hypertable — so a count keyed on the job's own name read ZERO for every aggregate job, forever, with + /// nothing to say it was looking at the wrong relation. The LEFT JOIN to + /// continuous_aggregates resolves the materialization when the job is an aggregate's and falls + /// through to the job's own identity for a raw hypertable. The eligibility delay comes from the job's + /// config for the same reason: the raw tier compresses after , the + /// aggregates after their tier's , and a count taken at the + /// wrong delay reports a chunk as waiting that the policy is not yet allowed to take. The constant + /// remains as the fallback for a policy whose config carries no compress_after at all + /// (compress_created_before policies), which this product never creates. The delay is also emitted + /// as a column so the log line can name the delay it counted against rather than restating the raw + /// constant for every job. /// public static string CompressionActivitySql => $@" @@ -6311,13 +8210,17 @@ THEN EXTRACT(EPOCH FROM (js.last_successful_finish - NULLIF(js.last_run_started_ ( SELECT count(*) FROM timescaledb_information.chunks AS c - WHERE c.hypertable_schema = j.hypertable_schema - AND c.hypertable_name = j.hypertable_name + WHERE c.hypertable_schema = COALESCE(ca.materialization_hypertable_schema, j.hypertable_schema) + AND c.hypertable_name = COALESCE(ca.materialization_hypertable_name, j.hypertable_name) AND NOT c.is_compressed - AND c.range_end < now() - INTERVAL '{CompressAfterDays} days' - ) AS eligible_uncompressed + AND c.range_end < now() - COALESCE((j.config->>'compress_after')::interval, INTERVAL '{CompressAfterDays} days') + ) AS eligible_uncompressed, + EXTRACT(EPOCH FROM (j.config->>'compress_after')::interval)::bigint AS compress_after_seconds FROM timescaledb_information.jobs AS j JOIN timescaledb_information.job_stats AS js USING (job_id) +LEFT JOIN timescaledb_information.continuous_aggregates AS ca + ON ca.view_schema = j.hypertable_schema + AND ca.view_name = j.hypertable_name WHERE j.proc_name LIKE '%compression%' OR j.proc_name LIKE '%columnstore%'"; @@ -6348,7 +8251,10 @@ public static async Task> ReadCompressionActi reader.IsDBNull(3) ? null : TimeSpan.FromSeconds(Convert.ToDouble(reader.GetValue(3), CultureInfo.InvariantCulture)), - Convert.ToInt64(reader.GetValue(4), CultureInfo.InvariantCulture))); + Convert.ToInt64(reader.GetValue(4), CultureInfo.InvariantCulture), + reader.IsDBNull(5) + ? null + : TimeSpan.FromSeconds(Convert.ToDouble(reader.GetValue(5), CultureInfo.InvariantCulture)))); } } catch (Exception ex) when (ex is not OperationCanceledException) @@ -6427,9 +8333,15 @@ public static void LogCompressionActivity( } else if (item.EligibleUncompressedChunks > 0) { + /* The delay and the cadence are the JOB's, not the raw constants: a continuous aggregate's policy + (#3581) waits two or four days and wakes once a day, and a line that said "1d" and "1 hour" + about it would be the #1958 drift in a new place. The raw constants are what a raw job's + config reads back as, so a raw hypertable's line is byte-identical to what it was. */ logger.LogInformation( "TimescaleDB: {Hypertable} has {Waiting} chunk(s) past the {Days}d compression delay and still uncompressed; its policy wakes every {Interval} (last completed run took {Seconds:F0}s).", - item.HypertableName, item.EligibleUncompressedChunks, CompressAfterDays, CompressScheduleInterval, + item.HypertableName, item.EligibleUncompressedChunks, + item.CompressAfter?.TotalDays ?? CompressAfterDays, + IsAggregateCompressionTarget(item.HypertableName) ? AggregateCompressionScheduleInterval : CompressScheduleInterval, item.LastRunDuration?.TotalSeconds ?? 0d); } @@ -6438,9 +8350,10 @@ public static void LogCompressionActivity( if (running == 0 && backlog == 0) { + var aggregatePolicies = activity.Count(item => IsAggregateCompressionTarget(item.HypertableName)); logger.LogDebug( - "TimescaleDB: {Count} compression policies on a {Interval} tick, nothing running, no eligible chunk uncompressed.", - activity.Count, CompressScheduleInterval); + "TimescaleDB: {Count} compression policies on a {Interval} tick and {AggregateCount} continuous-aggregate policies on a {AggregateInterval} cadence, nothing running, no eligible chunk uncompressed.", + activity.Count - aggregatePolicies, CompressScheduleInterval, aggregatePolicies, AggregateCompressionScheduleInterval); } LogCompressionClearance(activity, logger); @@ -6639,8 +8552,68 @@ public static async Task TryRearmJobAsync( /// A COMPRESSION-policy background job that flagged /// as stuck (#1581): its immutable job_id, the hypertable it compresses (for a friendlier alert label — /// may be null on an odd catalog), and the human-readable reason the pure predicate produced. +/// +/// (#3591) is true for a -infinity row on a TimescaleDB at or +/// past : the row is a crashed run in the +/// scheduler's crash backoff, and the scheduler will re-run it by itself. The evaluator MUST NOT re-arm such a +/// row, measured on a 2.28.1 rig: alter_job(next_start => now()) against a job in crash backoff does +/// not shorten the wait — the scheduler's crash arm ignores the persisted next_start while +/// consecutive_crashes > 0 — it RESETS it. The re-arm refreshes the scheduler's job list, every crash +/// row's backoff is recomputed from that instant, and the retry moved from crash + 5:00 to re-arm + 5:04, for +/// the re-armed job AND for an un-re-armed sibling crash row in the same database. It also overwrites the +/// -infinity with a finite value, so the next hourly read would report the job healthy and post +/// "Recovered" before it had run. On the fleet's hourly policies (default retry_period one hour) that is a +/// retry pushed out by up to an hour and two untrue messages. false — the pre-2.26.4 row, an unknown +/// version, or the stuck-Running arm — keeps #1581's re-arm-once semantics exactly. +/// +public sealed record StuckCompressionJob(long JobId, string? HypertableName, string Reason, bool SchedulerRetries = false); + +/// +/// WHICH arm of fired (#3575), from +/// . Exists because the two arms need different +/// treatment downstream: is judged on two inputs that TimescaleDB's +/// view reads from independent sources and is therefore CONFIRMED by a second read before it is reported; +/// is judged on hours of elapsed time and is reported from the first read. +/// +public enum StuckCompressionJobArm +{ + /// Healthy — neither arm fired. + None, + + /// next_start = -infinity while the job is not reporting Running: the dead-job arm, + /// and the one that races the run instant. + NextStartNegativeInfinity, + + /// Running since before : the hung-run + /// arm. + RunningPastBound, +} + +/// +/// One row of as the predicate consumes it (#3575): +/// the two -infinity tests already applied in SQL, the #1760 sentinel already NULLIFed. Internal because +/// it is the seam the confirm-read pins through, not a product surface; the product's result type is +/// . +/// +internal sealed record CompressionJobStatRow( + long JobId, + bool NextStartIsNegativeInfinity, + string? JobStatus, + DateTime? LastRunStartedAtUtc, + TimeSpan? ScheduleInterval, + string? HypertableName); + +/// +/// A the predicate flagged, with the arm that fired and its reason — +/// the unit merges two passes of (#3575). +/// is , decided where the arm +/// was (#3591): the -infinity arm on a store whose scheduler has upstream #9360. /// -public sealed record StuckCompressionJob(long JobId, string? HypertableName, string Reason); +internal sealed record ClassifiedCompressionJob(CompressionJobStatRow Row, StuckCompressionJobArm Arm, string Reason, bool SchedulerRetries = false) +{ + /// The product-facing shape of this flag. + public StuckCompressionJob ToJob() => new(Row.JobId, Row.HypertableName, Reason, SchedulerRetries); +} /// /// One background job's cadence reading (#2136): the last SUCCESSFUL run's duration against the job's own @@ -6822,13 +8795,20 @@ SpanSeconds is > 0 && HorizonSeconds is > 0 /// /// One hypertable's compression-policy activity (#1778): whether a run is in progress, when it started, how /// long the last COMPLETED run took, and how many chunks are past the eligibility delay but still uncompressed. +/// +/// is the delay the count was taken against — the job's own, read from its +/// config (#3581): one day for a raw hypertable, two or four for a continuous aggregate by tier. Null only for +/// a policy whose config carries no compress_after, which this product never creates; the log line then +/// falls back to . Optional and last so the five-argument +/// shape every existing caller and test constructs is unchanged. /// public sealed record CompressionActivity( string? HypertableName, string? JobStatus, DateTime? LastRunStartedAtUtc, TimeSpan? LastRunDuration, - long EligibleUncompressedChunks) + long EligibleUncompressedChunks, + TimeSpan? CompressAfter = null) { /// Is a compression run in progress right now? public bool IsRunning => string.Equals(JobStatus, "Running", StringComparison.OrdinalIgnoreCase); @@ -6862,6 +8842,15 @@ public sealed record CompressionActivity( /// hypertable — a bring-your-own store's own table, or a fixture table. Null is what keeps every /// clearance figure below silent for a FOREIGN hypertable: this code chose no minute for it, so it has no /// standing to say whether its run overran anything. + /// + /// Null for the continuous aggregates' compression jobs too, deliberately (#3581). Those jobs + /// are on once a day rather than on this + /// hourly grid, and the clearance findings below reason from the raw tier's geometry — the #3112 overrun + /// text explains a run by the daily chunk close of a 1-day raw chunk, which is not what an aggregate's + /// run is. Reporting an aggregate job through that text would name the wrong mechanism; a watch over the + /// daily band's own clearance (25 minutes to the next hour's first refresh) is a separate instrument, and + /// until it exists these jobs are silent here the way a foreign hypertable is, rather than misdescribed. + /// The #1778 backlog count above DOES cover them, at their own delay. /// public int? AssignedPhaseMinute => HypertableName is not null diff --git a/Darling/PerformanceMonitor.Darling.Viewer/AlertToastCoordinator.cs b/Darling/PerformanceMonitor.Darling.Viewer/AlertToastCoordinator.cs index 8a668c998..1a9216de1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/AlertToastCoordinator.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/AlertToastCoordinator.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Linq; +using PerformanceMonitor.Notifications; namespace PerformanceMonitor.Darling.Viewer; @@ -34,6 +35,22 @@ namespace PerformanceMonitor.Darling.Viewer; /// /// Muted rows never toast (Lite parity: the service still logs a muted row, flagged, with channels /// skipped). Bookkeeping is pruned each call so neither map grows without bound over a long-lived viewer. +/// +/// The tray is the VIEWER's channel, and the viewer honors mute rules for it itself (#3570). Before +/// this, a row toasted unless the SERVICE had stamped it muted — a flag that records the service's +/// decision about the service's channels (email/webhook), made against the service's in-memory rule cache, +/// which only a control-plane reload refreshes. So a Snooze from a toast (a config_mute_rules row) +/// suppressed the next toast only after a four-link cross-process chain completed: the reload beacon +/// observed on the service's next 15 s tick, a monolithic store re-read that succeeds in full, the mute +/// cache refreshed, and the alert re-firing THROUGH that cache. Nothing on the viewer side observed any of +/// it; when a link was slow or broken the rule sat in the table — visible in Manage Mute Rules — while the +/// toasts kept coming, which is exactly the report. Lite never had the gap: its Snooze lands in the same +/// in-process its deliverer consults before showing the balloon. This is the +/// headless equivalent: takes the viewer's own read of the active rules and skips +/// any row one of them covers, judged with the SAME the service uses, over +/// the context builds — so a rule the snooze just wrote stops +/// the toasts on the very next poll, whatever the service has or has not done with it yet. The service's +/// muted flag is still honored too; the two are ORed. /// public sealed class AlertToastCoordinator { @@ -89,8 +106,9 @@ public void Prime(IEnumerable existingRows) /// Filters a freshly-polled batch to the rows that should toast now, updating the seen-set and cooldown /// state. Rows are considered oldest-first so, within a burst that shares a condition, the EARLIEST row /// wins the cooldown slot (the rest are suppressed until the window elapses). A row is emitted only when - /// it is new (not seen/primed), not muted, and its condition is outside the cooldown window; every - /// processed row is marked seen regardless of outcome so it is considered exactly once. + /// it is new (not seen/primed), not muted — neither by the service's flag nor by a rule in + /// — and its condition is outside the cooldown window; every processed row + /// is marked seen regardless of outcome so it is considered exactly once. /// /// The latest read of recent, non-dismissed alert rows (any order). /// The current time (injected for testability). @@ -98,8 +116,19 @@ public void Prime(IEnumerable existingRows) /// The per-condition cooldown window (the "Tray notification cooldown" setting). /// or negative disables the cooldown so every new, unmuted row toasts. /// + /// + /// The mute rules as THIS viewer currently knows them (#3570): its latest read of config_mute_rules + /// plus any rule it has just written itself (a tray Snooze, a server Silence) and not yet re-read. A row + /// covered by any rule that is enabled and unexpired at is skipped exactly as a + /// service-muted row is: marked seen, never toasted — so a rule that later expires does not replay the + /// rows it covered. Null or empty means "no viewer-side rules", which leaves the pre-#3570 behavior (the + /// service's flag alone). The judgement is over + /// — the shared matcher and the shared context, so the tray + /// agrees with the service's channels about what a rule covers rather than approximating it. + /// public IReadOnlyList SelectToasts( - IEnumerable polledRows, DateTime nowUtc, TimeSpan cooldown) + IEnumerable polledRows, DateTime nowUtc, TimeSpan cooldown, + IReadOnlyList? muteRules = null) { ArgumentNullException.ThrowIfNull(polledRows); @@ -120,6 +149,11 @@ public IReadOnlyList SelectToasts( continue; /* Lite parity: a muted row is logged but never toasted */ } + if (IsMutedByViewerRules(row, muteRules, nowUtc)) + { + continue; /* #3570: a rule this viewer holds covers the row — the tray honors it without waiting on the service */ + } + var conditionKey = ConditionKey(row); if (cooldown > TimeSpan.Zero && _lastToast.TryGetValue(conditionKey, out var last) @@ -136,6 +170,32 @@ public IReadOnlyList SelectToasts( return toasts; } + /// + /// True when any rule in covers at + /// (#3570). The context is built ONCE per row and only when there are rules to test, so a fleet with no + /// mute rules pays nothing for the detail-text parse. Pure: same matcher () + /// the service's applies, judged on the coordinator's injected + /// clock rather than the ambient one so a test can place a rule's expiry on either side of "now". + /// + internal static bool IsMutedByViewerRules(ViewerAlertRow row, IReadOnlyList? muteRules, DateTime nowUtc) + { + if (muteRules is null || muteRules.Count == 0) + { + return false; + } + + var context = row.ToMuteContext(); + foreach (var rule in muteRules) + { + if (rule is not null && rule.MatchesAt(context, nowUtc)) + { + return true; + } + } + + return false; + } + /// /// Drops bookkeeping that can no longer affect a decision: a seen row older than the retention window /// (the poll can never re-read it) and a cooldown entry older than the effective cooldown window. diff --git a/Darling/PerformanceMonitor.Darling.Viewer/AlertsHistoryTab.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/AlertsHistoryTab.xaml.cs index 37e638d28..e603f8473 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/AlertsHistoryTab.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/AlertsHistoryTab.xaml.cs @@ -455,17 +455,10 @@ private async void MuteThisAlert_Click(object sender, RoutedEventArgs e) return; } - var context = new AlertMuteContext - { - ServerName = item.ServerName, - MetricName = item.MetricName - }; - /* #3309: pass the metric name so a custom alert ("Custom:") skips detail_text pre-fill parsing - - a custom rule has no Database/Wait Type/Job/Query dimension to pre-fill, and its user-authored name - must not be able to forge a mute-context label line. */ - context.PopulateFromDetailText(item.DetailText, item.MetricName); - - await CreateMuteRuleAsync(context); + /* The row's own mute context (server + metric + the dimensions parsed from detail_text; #3309's + custom-alert skip lives inside it). Shared with the tray-toast filter (#3570) so a rule authored + from this row is judged against the same context the toast filter will later judge it with. */ + await CreateMuteRuleAsync(item.ToMuteContext()); } private async void MuteSimilarAlerts_Click(object sender, RoutedEventArgs e) diff --git a/Darling/PerformanceMonitor.Darling.Viewer/App.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/App.xaml.cs index ddca9c139..5b4046ecf 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/App.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/App.xaml.cs @@ -7,6 +7,7 @@ */ using System; +using System.IO; using System.Reflection; using System.Threading.Tasks; using System.Windows; @@ -67,6 +68,17 @@ diagnostic the one guaranteed to vanish. Nothing between here and its old positi failure. */ ViewerLogger.Initialize(); + /* #3577: the operator's per-theme color overrides live beside viewer-settings.json, in the viewer's + own per-user directory - LOCAL to this machine, never in the store or the control plane, because + a color is a preference of the person at this screen and not a fact about the fleet. The path + and the log hooks go in BEFORE the first Apply so the very first paint carries them (the same + no-flash reasoning as the theme itself); the watcher that re-applies an outside edit starts + right after. */ + ThemeManager.OverridesFilePath = Path.Combine( + Path.GetDirectoryName(ViewerAppSettingsStore.DefaultFilePath())!, ThemeColorOverrides.FileName); + ThemeManager.LogWarning = message => ViewerLogger.Warn("Theme", message); + ThemeManager.LogInfo = message => ViewerLogger.Info("Theme", message); + /* Apply the saved color theme through ThemeManager (App.xaml merges Dark as the design-time default) so ThemeManager owns the app-level merged dictionary at runtime, before StartupUri creates MainWindow. Reads the viewer-local settings directly (cheap JSON read) so the very @@ -81,6 +93,8 @@ on any error. Light / CoolBreeze are honored via the Settings window's theme sel ThemeManager.Apply("Dark"); } + ThemeManager.WatchOverridesFile(); + /* #1050 (companion to the ported tray's WindowResumeGuard): WPF's GPU render thread can zombie its surface across sleep/wake or RDP, leaving a live-but-blank window — now reachable in the viewer because minimize-to-tray can hide it. Software rendering removes the GPU dependency entirely; charts diff --git a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml index 216d685a2..811b73337 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml @@ -91,7 +91,9 @@ - + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml.cs b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml.cs index 7b87db9d2..6bcc5ede6 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleEditorWindow.xaml.cs @@ -11,6 +11,7 @@ using System.Linq; using System.Windows; using System.Windows.Controls; +using PerformanceMonitor.Collectors; namespace PerformanceMonitor.Darling.Viewer; @@ -62,6 +63,15 @@ private async void OnLoaded(object sender, RoutedEventArgs e) PopulateScopeCombos(); + /* The delta cadence cap (#3532), from the shared constants so the shown numbers and collector + list can never drift from what ValidateSchedule enforces. */ + var deltaNames = CollectorSchedulePresets.BuildDefaultSchedule() + .Where(s => CollectorDeltaCalculator.IsDeltaFamily(s.Name)) + .Select(s => s.Name); + FooterHintText.Text += + $" Delta collectors ({string.Join(", ", deltaNames)}) accept at most {CollectorDeltaCalculator.MaxDeltaFrequencyMinutes} minutes: " + + $"past the {CollectorDeltaCalculator.DefaultMaxGapSeconds / 60}-minute delta gap policy every reading would be discarded as stale and recorded as zero."; + try { _allOverrides = await _dataService.GetCollectorSchedulesAsync(); @@ -290,7 +300,7 @@ private async void SaveButton_Click(object sender, RoutedEventArgs e) var usesDefault = _scopeServerId is not null && UseDefaultCheckBox.IsChecked == true; - if (!usesDefault && !ValidateSchedule(out var error)) + if (!usesDefault && !CollectorScheduleOverlay.ValidateSchedule(_editing, out var error)) { MessageBox.Show(error, "Collector Schedules", MessageBoxButton.OK, MessageBoxImage.Warning); return; @@ -332,29 +342,6 @@ private async void SaveButton_Click(object sender, RoutedEventArgs e) } } - /// Enforces the V17 CHECK constraints before the write (frequency >= 0, retention >= 1) so a - /// bad value surfaces as a friendly message rather than a raw Postgres error. - private bool ValidateSchedule(out string error) - { - foreach (var item in _editing) - { - if (item.FrequencyMinutes < 0) - { - error = $"'{item.Name}': frequency (minutes) can't be negative. Use 0 to collect once on server load."; - return false; - } - - if (item.RetentionDays < 1) - { - error = $"'{item.Name}': retention (days) must be at least 1."; - return false; - } - } - - error = ""; - return true; - } - /// /// "Apply Default to All Servers" — the fleet-scale bulk reset: removes EVERY server's per-server schedule /// override in one write () so they all fall back diff --git a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleOverlay.cs b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleOverlay.cs index 033ad4630..94e49c1dd 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleOverlay.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/CollectorScheduleOverlay.cs @@ -90,6 +90,44 @@ public static List ParseDatabases(string? text) => .Where(s => s.Length > 0) .ToList(); + /// + /// Enforces what the store and the collection pipeline can honor before the write, so a bad value + /// surfaces as a friendly message rather than a raw Postgres error or silent bad data: the V17 CHECK + /// constraints (frequency >= 0, retention >= 1) plus the delta gap-policy cadence cap (#3532) — + /// a delta-family collector past would + /// exceed every cycle and record permanent + /// zeros (the service's StoreConfigProvider.ResolveSchedule refuses such a row too, by falling + /// through to the default). Pure, so Darling.Tests exercise it without a Window. + /// + public static bool ValidateSchedule(IReadOnlyList edited, out string error) + { + ArgumentNullException.ThrowIfNull(edited); + + foreach (var item in edited) + { + if (item.FrequencyMinutes < 0) + { + error = $"'{item.Name}': frequency (minutes) can't be negative. Use 0 to collect once on server load."; + return false; + } + + if (CollectorDeltaCalculator.DeltaFrequencyError(item.Name, item.FrequencyMinutes) is string frequencyError) + { + error = frequencyError; + return false; + } + + if (item.RetentionDays < 1) + { + error = $"'{item.Name}': retention (days) must be at least 1."; + return false; + } + } + + error = ""; + return true; + } + /// True when the store holds any override row for this server (the editor's "custom vs. use /// default" initial state). public static bool ServerHasOverride(IReadOnlyList allOverrides, int serverId) diff --git a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml index fecfcdda4..00e8ac495 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/FinOpsTab.xaml @@ -610,7 +610,9 @@ public string InsufficientDataMessage { get; } + /// + /// The message shown in the state — the engine's own + /// persisted message (or when it supplied none), always + /// suffixed with the Collection Health pointer. Empty in every other state. + /// + public string WindowEmptyMessage { get; } + /// Total card count across all sections. public int TotalCount => Sections.Sum(s => s.Count); private RecommendationsViewModel( - IReadOnlyList sections, RecommendationsState state, string insufficientDataMessage) + IReadOnlyList sections, RecommendationsState state, string insufficientDataMessage, + string windowEmptyMessage = "") { Sections = sections; State = state; InsufficientDataMessage = insufficientDataMessage; + WindowEmptyMessage = windowEmptyMessage; } /// The default insufficient-data prose when the engine supplied no message (mirrors Lite's). @@ -410,29 +427,58 @@ public static RecommendationsViewModel InsufficientData(string? message) => RecommendationsState.InsufficientData, string.IsNullOrWhiteSpace(message) ? DefaultInsufficientDataMessage : message!); + /// The default window-empty prose when the marker carried no message (mirrors Lite's). + public const string DefaultWindowEmptyMessage = + "Nothing was collected in this analysis window, so nothing was measured — this is not an all-clear."; + /// - /// Builds a loaded/empty/insufficient-data view-model from the persisted finding rows and the - /// per-server analysis-state marker. Maps each row to an advise-only item, appends the co-fired + /// Appended to every window-empty message so the operator lands on the surface that diagnoses a + /// dead collector — the viewer's rendering of the same pointer the MCP analyze_server tool + /// appends (get_collection_health there, the in-app tab here). Mirrors Lite's. + /// + public const string WindowEmptyCollectionHealthPointer = + "Check the Collection Health tab to see when collectors last succeeded."; + + /// + /// Builds the window-empty-state view-model (#3524/#3551) from the persisted marker's message (or + /// the default when it is null/blank), suffixed with the Collection Health pointer — the viewer's + /// mirror of Lite's LiteRecommendationsViewModel.WindowEmpty, sourced from the V19 marker's + /// window-empty shape () rather than a live engine call. + /// + public static RecommendationsViewModel WindowEmpty(string? message) => + new( + Array.Empty(), + RecommendationsState.WindowEmpty, + string.Empty, + (string.IsNullOrWhiteSpace(message) ? DefaultWindowEmptyMessage : message!) + + " " + WindowEmptyCollectionHealthPointer); + + /// + /// Builds a loaded/empty/insufficient-data/window-empty view-model from the persisted finding rows + /// and the per-server analysis-state marker. Maps each row to an advise-only item, appends the co-fired /// cross-reference, and groups by incident. State selection: /// /// one or more findings -> (findings always win); /// zero findings AND (the persisted marker says the engine /// has not cleared its 24h data-span gate) -> /// ("still collecting"); - /// zero findings and no insufficient-data marker -> - /// (the genuine all-clear — enough data, nothing to report). + /// zero findings AND (the marker records a window-empty pass, + /// #3524/#3551) -> ("collection appears broken"); + /// zero findings and neither marker -> + /// (the genuine all-clear — enough data, facts measured, nothing to report). /// /// is carried onto each card for the Ask-AI prompt's window. The /// rows arrive pre-sorted (severity band desc, raw desc, database, title) from the read, and grouping - /// preserves that order. defaults false so the callers that carry - /// no marker keep the prior loaded/empty behavior. + /// preserves that order. and default + /// false so the callers that carry no marker keep the prior loaded/empty behavior. /// public static RecommendationsViewModel FromFindings( IReadOnlyList rows, string serverName, int utcOffsetMinutes = 0, - bool insufficientData = false, string? insufficientDataMessage = null) + bool insufficientData = false, string? insufficientDataMessage = null, + bool windowEmpty = false, string? windowEmptyMessage = null) { if (rows is null || rows.Count == 0) - return ZeroFindingState(insufficientData, insufficientDataMessage); + return ZeroFindingState(insufficientData, insufficientDataMessage, windowEmpty, windowEmptyMessage); var items = new List(rows.Count); foreach (var row in rows) @@ -443,7 +489,7 @@ public static RecommendationsViewModel FromFindings( } if (items.Count == 0) - return ZeroFindingState(insufficientData, insufficientDataMessage); + return ZeroFindingState(insufficientData, insufficientDataMessage, windowEmpty, windowEmptyMessage); AppendCoFired(items); return new(GroupByIncident(items, utcOffsetMinutes), RecommendationsState.Loaded, string.Empty); @@ -452,13 +498,20 @@ public static RecommendationsViewModel FromFindings( /// /// Picks the state for a zero-finding read: when /// the persisted marker says the analysis pass has not cleared the 24h data-span gate (so the tab - /// shows "still collecting" rather than a false all-clear), else - /// (a genuine all-clear). + /// shows "still collecting" rather than a false all-clear), + /// when it records a window-empty pass instead (#3524/#3551 — "collection appears broken", also never + /// the all-clear; the two marker shapes are mutually exclusive at the writer, and insufficient-data is + /// checked first defensively), else (a genuine all-clear). /// - private static RecommendationsViewModel ZeroFindingState(bool insufficientData, string? message) => - insufficientData - ? InsufficientData(message) - : new(Array.Empty(), RecommendationsState.Empty, string.Empty); + private static RecommendationsViewModel ZeroFindingState( + bool insufficientData, string? insufficientMessage, bool windowEmpty, string? windowEmptyMessage) + { + if (insufficientData) + return InsufficientData(insufficientMessage); + if (windowEmpty) + return WindowEmpty(windowEmptyMessage); + return new(Array.Empty(), RecommendationsState.Empty, string.Empty); + } /// /// Maps one persisted finding row to an advise-only . Reuses the diff --git a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml index 20adee4c9..ddb579f0f 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/SettingsWindow.xaml @@ -1,6 +1,7 @@ + + + + @@ -354,11 +362,16 @@ Foreground="{DynamicResource ForegroundMutedBrush}"/> - + - + + Foreground="{DynamicResource ForegroundMutedBrush}"/> - + + - - = 0 and <= 100) row.SelfDiskFreeWarnPercent = selfDiskPct; + /* #3528: validated to the same bound DarlingAlertSettings clamps (Math.Max(0, ...)) and the MCP + writer accepts ([0, int.MaxValue]) — 0 is IN range because it removes the floor. */ + if (int.TryParse(AlertSelfDiskWarnGbBox.Text, out var selfDiskGb) && selfDiskGb >= 0) + row.SelfDiskFreeWarnGb = selfDiskGb; if (int.TryParse(AlertCollectionStaleMinutesBox.Text, out var staleMin) && staleMin is >= 5 and <= 1440) row.CollectionStaleMinutes = staleMin; if (int.TryParse(AlertCollectionFailureThresholdBox.Text, out var failThresh) && failThresh is >= 1 and <= 1000) @@ -1013,6 +1018,10 @@ leave this button writing a threshold nobody chose. */ AlertDiskCriticalPercentBox.Text = "3"; AlertDiskCriticalGbBox.Text = "2"; AlertSelfDiskWarnPercentBox.Text = "10"; + /* #3528: the shipped GB floor (DarlingSelfAlertEvaluator.DiskFreeWarnFloorGb) as a literal — the + constant lives on the Service assembly the viewer does not reference, and the V126 rung test + pins this literal equal to it. */ + AlertSelfDiskWarnGbBox.Text = "50"; AlertCollectionStaleMinutesBox.Text = "30"; AlertCollectionFailureThresholdBox.Text = "10"; /* #3060: derived, unlike its neighbours, because this one is not merely a mirrored default — it is @@ -1094,7 +1103,9 @@ duplicate rather than as information. */ && !string.Equals(AlertPgBlockingThresholdBox.Text, AlertBlockingThresholdBox.Text, StringComparison.Ordinal)) parts.Add($"pg blocking >= {AlertPgBlockingThresholdBox.Text}"); if (AlertPoisonWaitCheckBox.IsChecked == true) - parts.Add($"poison waits >= {AlertPoisonWaitThresholdBox.Text}ms avg"); + /* #3539 A4: the live bar, from the shared constants — not the retired ms box, which nothing reads. */ + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"poison waits >= {PoisonWaitEvaluator.WarningAvgWaiters * PoisonWaitEvaluator.WindowMinutes * 60:N0}s accumulated in {PoisonWaitEvaluator.WindowMinutes}min")); if (AlertLongRunningQueryCheckBox.IsChecked == true) parts.Add($"queries > {AlertLongRunningQueryThresholdBox.Text}min"); if (AlertTempDbSpaceCheckBox.IsChecked == true) @@ -1104,7 +1115,10 @@ duplicate rather than as information. */ if (AlertPvsCheckBox.IsChecked == true) parts.Add($"PVS >= {AlertPvsThresholdPercentBox.Text}% of database"); if (AlertFileGrowthCheckBox.IsChecked == true) - parts.Add($"file growth > {AlertFileGrowthRiseMbBox.Text}MB/{AlertFileGrowthLookbackMinutesBox.Text}m or volume > {AlertFileGrowthVolumePercentBox.Text}%"); + /* #3539 A8c: the rise is a RATE (MB per hour) averaged over the lookback, in the same unit phrase the + row's label, the alert's threshold line and the MCP payload description use. It used to read + "10240MB/60m", which was the per-window delta the engine then compared literally. */ + parts.Add($"file growth > {AlertFileGrowthRiseMbBox.Text} {AlertContextBuilders.FileGrowthRiseUnit} over {AlertFileGrowthLookbackMinutesBox.Text}m or volume > {AlertFileGrowthVolumePercentBox.Text}%"); if (AlertLongRunningJobCheckBox.IsChecked == true) parts.Add($"jobs > {AlertLongRunningJobMultiplierBox.Text}x avg"); if (AlertFailedJobCheckBox.IsChecked == true) @@ -1136,7 +1150,10 @@ private void UpdateAlertControlStates() AlertPgDeadlockThresholdBox.IsEnabled = enabled; AlertPgBlockingThresholdBox.IsEnabled = enabled; AlertPoisonWaitCheckBox.IsEnabled = enabled; - AlertPoisonWaitThresholdBox.IsEnabled = enabled; + /* #3539 A4: the poison-wait ms box is retired (nothing reads it) and stays disabled regardless of the + master switch — the XAML sets IsEnabled="False", and this loop must not re-enable it on load or + on toggle, or the operator is back to tuning a number the engine ignores. */ + AlertPoisonWaitThresholdBox.IsEnabled = false; AlertLongRunningQueryCheckBox.IsEnabled = enabled; AlertLongRunningQueryThresholdBox.IsEnabled = enabled; /* V20 long-running-query read-shape controls follow the master switch like the rest of the engine. */ @@ -1158,6 +1175,7 @@ private void UpdateAlertControlStates() AlertDiskCriticalPercentBox.IsEnabled = enabled; AlertDiskCriticalGbBox.IsEnabled = enabled; AlertSelfDiskWarnPercentBox.IsEnabled = enabled; + AlertSelfDiskWarnGbBox.IsEnabled = enabled; AlertCollectionStaleMinutesBox.IsEnabled = enabled; AlertCollectionFailureThresholdBox.IsEnabled = enabled; AlertStoreJobCadenceWarnPercentBox.IsEnabled = enabled; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/Themes/CoolBreezeTheme.xaml b/Darling/PerformanceMonitor.Darling.Viewer/Themes/CoolBreezeTheme.xaml index 4c42dc07f..62becce46 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/Themes/CoolBreezeTheme.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/Themes/CoolBreezeTheme.xaml @@ -10,8 +10,17 @@ #1E6FA8 - #2B87C8 + + #267BB8 #155A8A + + #FFFFFF #CFDDE9 @@ -34,7 +43,13 @@ #2E7D32 - #F57F17 + + #9E4A0B #C62828 #1E6FA8 @@ -54,6 +69,7 @@ + @@ -85,6 +101,21 @@ + + + + + + + @@ -162,6 +193,8 @@ - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1099,7 +1172,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1098,7 +1165,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1099,7 +1174,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1099,7 +1172,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1098,7 +1165,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + - - + + + use the theme accent and the matching readable text color (the on-accent ink, #3577). --> - + - + + @@ -1099,7 +1174,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + @@ -1146,7 +1202,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + @@ -1145,7 +1196,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + + @@ -1146,7 +1200,12 @@ + VerticalAlignment="{TemplateBinding VerticalContentAlignment}"> + + +