diff --git a/Lite.Tests/FinOpsCpuSampleWindowTests.cs b/Lite.Tests/FinOpsCpuSampleWindowTests.cs new file mode 100644 index 000000000..f333af18d --- /dev/null +++ b/Lite.Tests/FinOpsCpuSampleWindowTests.cs @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Darling.Tests; +using DuckDB.NET.Data; +using Lite.Tests; +using PerformanceMonitorLite.Analysis; +using PerformanceMonitorLite.Database; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// Every FinOps scenario's CPU samples must sit inside the window the FinOps utilization read uses +/// (GetUtilizationEfficiencyAsync: the last 24 hours from now), at any time of day the suite runs. +/// +/// Seeded from , which is anchored to 04:00 UTC (#4385), the +/// samples sat 24 to 28 hours back between 03:45 and 04:00 UTC. The read found none, so HasCpuSample was false +/// and the CPU and VM right-sizing rules gave no advice: six FinOpsTests failed every day in that quarter hour. +/// The seeder takes a clock, so this class places the samples as a run at 03:50 would, without waiting for the +/// real clock to get there. The source pin below keeps the window this class checks equal to the read's. +/// +public sealed class FinOpsCpuSampleWindowTests : IClassFixture +{ + /// The hours GetUtilizationEfficiencyAsync reads back from now. Pinned to its source below. + private const int UtilizationWindowHours = 24; + + /// Every FinOps scenario that seeds cpu_utilization_stats. A new one goes here too. + private static readonly Dictionary> Scenarios = new() + { + ["OverProvisionedEnterprise"] = s => s.SeedOverProvisionedEnterpriseAsync(), + ["RightSizing"] = s => s.SeedRightSizingScenarioAsync(engineEdition: 3, withCpuSamples: true), + ["VmRightSizingTarget"] = s => s.SeedVmRightSizingTargetAsync(), + ["AzureSqlDbVcore"] = s => s.SeedAzureSqlDbVcoreAsync(), + ["CleanFinOpsServer"] = s => s.SeedCleanFinOpsServerAsync(), + ["StableCpuForReservedCapacity"] = s => s.SeedStableCpuForReservedCapacityAsync(), + ["BurstyCpu"] = s => s.SeedBurstyCpuAsync() + }; + + private readonly DuckDbInitializer _duckDb; + + public FinOpsCpuSampleWindowTests(SharedDuckDbFixture fixture) + { + fixture.ResetData(); + _duckDb = fixture.DuckDb; + } + + /// + /// Each scenario at 03:50 UTC (inside the quarter hour before the 04:00 anchor rolls forward a day, when the + /// anchored samples were all more than 24 hours old) and at 12:00 UTC (an ordinary hour). + /// + /// Both are on a day 30 days before the run. A sample placed from , + /// which the real clock anchors, then falls outside the window at either time, so a scenario that goes back to + /// the anchored seed fails here on any day, not only in the quarter hour it breaks in. + /// + public static TheoryData ScenariosAtTimesOfDay() + { + var data = new TheoryData(); + foreach (var scenario in Scenarios.Keys) + { + data.Add(scenario, 3, 50); + data.Add(scenario, 12, 0); + } + return data; + } + + [Theory] + [MemberData(nameof(ScenariosAtTimesOfDay))] + public async Task EveryCpuSampleIsInsideTheUtilizationWindow(string scenario, int hour, int minute) + { + var nowUtc = DateTime.UtcNow.Date.AddDays(-30).AddHours(hour).AddMinutes(minute); + + using (var seeder = new TestDataSeeder(_duckDb, () => nowUtc)) + { + await Scenarios[scenario](seeder); + } + + using var readLock = _duckDb.AcquireReadLock(); + using var connection = _duckDb.CreateConnection(); + await connection.OpenAsync(); + using var command = connection.CreateCommand(); + command.CommandText = @" +SELECT + COUNT(*) AS seeded, + COUNT(*) FILTER (WHERE collection_time >= $2 AND collection_time <= $3) AS in_window +FROM v_cpu_utilization_stats +WHERE server_id = $1"; + command.Parameters.Add(new DuckDBParameter { Value = TestDataSeeder.TestServerId }); + command.Parameters.Add(new DuckDBParameter { Value = nowUtc.AddHours(-UtilizationWindowHours) }); + command.Parameters.Add(new DuckDBParameter { Value = nowUtc }); + + using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + var seeded = Convert.ToInt64(reader.GetValue(0)); + var inWindow = Convert.ToInt64(reader.GetValue(1)); + + Assert.True(seeded > 0, $"{scenario} seeded no CPU samples."); + if (inWindow != seeded) + { + Assert.Fail( + $"{scenario} at {nowUtc:HH:mm} UTC: {inWindow} of its {seeded} CPU samples are inside the " + + $"{UtilizationWindowHours}-hour utilization read " + + $"({nowUtc.AddHours(-UtilizationWindowHours):yyyy-MM-dd HH:mm} to {nowUtc:yyyy-MM-dd HH:mm} UTC)."); + } + } + + /// + /// The read's CPU window is the last hours from now: the cutoff, the + /// parameter that carries it, and the cpu_stats filter that uses it. + /// + [Fact] + public void UtilizationReadCpuWindowIsTheLast24Hours() + { + const string Signature = "Task GetUtilizationEfficiencyAsync("; + + var source = ParitySource.ReadFile("Lite/Services/LocalDataService.FinOps.Utilization.cs"); + var stripped = CSharpSourceWalker.StripCommentsAndStrings(source); + + var at = stripped.IndexOf(Signature, StringComparison.Ordinal); + Assert.True(at >= 0, "GetUtilizationEfficiencyAsync was not found."); + Assert.Equal(-1, stripped.IndexOf(Signature, at + 1, StringComparison.Ordinal)); + + var open = stripped.IndexOf('{', at); + var body = CSharpSourceWalker.BraceBalanced(stripped, open); + var code = Regex.Replace(body, @"\s+", " "); + + Assert.Contains($"var cutoff = DateTime.UtcNow.AddHours(-{UtilizationWindowHours});", code); + Assert.Contains( + "command.Parameters.Add(new DuckDBParameter { Value = serverId }); " + + "command.Parameters.Add(new DuckDBParameter { Value = cutoff });", + code); + + /* The SQL is a string literal, blanked in the stripped copy: read it from the same span of the source. */ + var rawBody = source.Substring(open, body.Length); + var cte = rawBody.IndexOf("cpu_stats AS (", StringComparison.Ordinal); + Assert.True(cte >= 0, "The cpu_stats CTE was not found."); + + var depth = 0; + var end = rawBody.IndexOf('(', cte); + for (; end < rawBody.Length; end++) + { + if (rawBody[end] == '(') depth++; + else if (rawBody[end] == ')' && --depth == 0) break; + } + + var cpuStats = Regex.Replace(rawBody[cte..end], @"\s+", " "); + Assert.Contains("FROM v_cpu_utilization_stats", cpuStats); + Assert.Contains("collection_time >= $2", cpuStats); + } +} diff --git a/Lite.Tests/TestDataSeeder.cs b/Lite.Tests/TestDataSeeder.cs index 9065e5690..a8ff2025d 100644 --- a/Lite.Tests/TestDataSeeder.cs +++ b/Lite.Tests/TestDataSeeder.cs @@ -65,9 +65,16 @@ private static DateTime AnchorPeriodEndToUtcMidnight(DateTime nowUtc) private long _nextId = -1_000_000; - public TestDataSeeder(DuckDbInitializer duckDb) + /// The clock the FinOps CPU samples are placed by (). + private readonly Func _utcNow; + + /// The database to seed. + /// The clock the FinOps scenarios place their CPU samples by. Omitted, it is the real UTC + /// clock. A test passes a fixed instant to place the samples as a run at that time of day would. + public TestDataSeeder(DuckDbInitializer duckDb, Func? utcNow = null) { _duckDb = duckDb; + _utcNow = utcNow ?? (() => DateTime.UtcNow); } public void Dispose() => _seedConn?.Dispose(); @@ -1338,7 +1345,11 @@ INSERT INTO server_config } /// - /// Seeds cpu_utilization_stats across 16 collection points. + /// Seeds cpu_utilization_stats across 16 collection points, 15 minutes apart from : + /// the 04:00 UTC anchor the analysis scenarios need. A FinOps scenario uses + /// instead, because the FinOps utilization read keeps the last 24 + /// hours from now and the anchored samples are older than that between 03:45 and 04:00 UTC. A new FinOps + /// scenario also goes in FinOpsCpuSampleWindowTests' scenario list. /// internal async Task SeedCpuUtilizationAsync(int avgSqlCpu, int avgOtherCpu) { @@ -2130,7 +2141,7 @@ public async Task SeedOverProvisionedEnterpriseAsync() await SeedTestServerAsync(); // 32 cores, 256GB RAM, but avg CPU 8%, buffer pool only 40GB of 256GB - await SeedCpuUtilizationAsync(8, 2); + await SeedFinOpsCpuUtilizationAsync(8, 2); await SeedMemoryStatsAsync(totalPhysicalMb: 262_144, bufferPoolMb: 40_960, targetMb: 245_760); await SeedServerPropertiesAsync(cpuCount: 32, htRatio: 2, physicalMemMb: 262_144, edition: "Enterprise Edition"); @@ -2156,7 +2167,7 @@ public async Task SeedRightSizingScenarioAsync(int engineEdition, bool withCpuSa if (withCpuSamples) { - await SeedCpuUtilizationAsync(8, 2); + await SeedFinOpsCpuUtilizationAsync(8, 2); } var azureSqlDatabase = engineEdition == 5; @@ -2244,30 +2255,14 @@ public async Task SeedCleanFinOpsServerAsync() // Healthy: 50% CPU, 75% buffer pool ratio, no idle databases. // - // GetUtilizationEfficiencyAsync (the CPU right-sizing check's data source) reads - // collection_time >= DateTime.UtcNow.AddHours(-24) -- a fixed 24h window measured from - // "now", not from TestPeriodEnd. TestPeriodStart/End is anchored to the most recent - // UTC-midnight-plus-4h boundary (#4385), which can land up to ~28h before "now" (worst - // case: 03:59 UTC, one minute before the anchor rolls forward a day). At that worst - // hour, SeedCpuUtilizationAsync's 16 points (TestPeriodStart .. TestPeriodStart+3h45m) - // fall entirely outside a naive 24h-from-now lookback, so the CPU check would read zero - // samples and could compute a false P95 -- so seed a second, always-in-window copy of - // the same healthy CPU signal anchored to "now" instead of TestPeriodStart, the same - // fix shape as #4558 but on the seed side (no hoursBack parameter exists on this read - // path to move to the test side instead). - // variance: 0 -- SeedCpuUtilizationAsync's own 16 points are a flat 50 (no jitter), so - // matching that here keeps the combined 32-point series' stddev at exactly 0 across - // both time windows the FinOps engine reads (24h-from-now for the CPU right-sizing - // check, 7-day for reserved-capacity). Rule 14 (reserved capacity, ~line 806 in - // LocalDataService.FinOps.Recommendations.cs) only fires when avgCpu > 20 AND - // stddevCpu > 0 AND CV (stddev/avg) < 0.3; a nonzero variance here (previously 5, - // giving avg ~49.3%, CV ~0.04) made it fire at every hour. stddev == 0 keeps that - // condition false regardless of the clock. P95 stays at 50%, well clear of rule 2's - // "CPU over-provisioned" P95 < 30% threshold, at any hour too. - await SeedCpuUtilizationAsync(50, 5); - await SeedCpuUtilizationInRangeAsync( - DateTime.UtcNow.AddHours(-3), DateTime.UtcNow.AddMinutes(-5), - avgCpu: 50, variance: 0, samples: 16); + // The CPU is 32 samples of a flat 50. Rule 14 (reserved capacity) reads 7 days and returns no + // row under 24 samples, so 32 is what puts this server in front of its guard: it fires only + // when avgCpu > 20 AND stddevCpu > 0 AND CV < 0.3. A flat 50 has a standard deviation of + // exactly 0, and that is what keeps it quiet; a jittered series (variance 5, CV ~0.04) made + // it fire. With 16 samples the rule would never reach the guard, and this scenario would stop + // holding it. The P95 of 50 is well clear of rule 2's "CPU over-provisioned" P95 < 30%. + // SeedFinOpsCpuUtilizationAsync puts the samples inside the 24-hour read at any time of day. + await SeedFinOpsCpuUtilizationAsync(50, 5, samples: 32); await SeedMemoryStatsAsync(totalPhysicalMb: 65_536, bufferPoolMb: 49_152, targetMb: 57_344); await SeedServerPropertiesAsync(cpuCount: 8, htRatio: 2, physicalMemMb: 65_536, edition: "Developer Edition"); @@ -2301,6 +2296,57 @@ await SeedServerPropertiesAsync(cpuCount: 8, htRatio: 2, physicalMemMb: 65_536, // FinOps Seed Helpers // ============================================ + /// + /// The times of a FinOps scenario's CPU samples for a run at : 15 minutes apart, oldest + /// first, the newest 5 minutes before . + /// + /// The FinOps utilization read (GetUtilizationEfficiencyAsync, behind CPU and VM right-sizing) keeps the + /// last 24 hours from now, so these samples are placed from now. Placed from like + /// the analysis scenarios' samples, they were 24 to 28 hours old between 03:45 and 04:00 UTC and the read found + /// none. The analysis scenarios keep the 04:00 anchor (see ). + /// + internal static DateTime[] FinOpsCpuSampleTimes(DateTime nowUtc, int samples) + { + var newest = nowUtc.AddMinutes(-5); + var times = new DateTime[samples]; + for (var i = 0; i < samples; i++) + { + times[i] = newest.AddMinutes(-15 * (samples - 1 - i)); + } + return times; + } + + /// + /// Seeds a FinOps scenario's cpu_utilization_stats: samples at + /// , each with the given SQL Server and other-process CPU. + /// + internal async Task SeedFinOpsCpuUtilizationAsync(int avgSqlCpu, int avgOtherCpu, int samples = 16) + { + using var readLock = _duckDb.AcquireReadLock(); + var connection = await SeedConnectionAsync(); + using var batch = new SeedBatch(connection); + + foreach (var t in FinOpsCpuSampleTimes(_utcNow(), samples)) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" +INSERT INTO cpu_utilization_stats + (collection_id, collection_time, server_id, server_name, + sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization) +VALUES ($1, $2, $3, $4, $5, $6, $7)"; + + cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- }); + cmd.Parameters.Add(new DuckDBParameter { Value = t }); + cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId }); + cmd.Parameters.Add(new DuckDBParameter { Value = TestServerName }); + cmd.Parameters.Add(new DuckDBParameter { Value = t }); + cmd.Parameters.Add(new DuckDBParameter { Value = avgSqlCpu }); + cmd.Parameters.Add(new DuckDBParameter { Value = avgOtherCpu }); + + await cmd.ExecuteNonQueryAsync(); + } + } + /// /// Seeds database_size_stats with 3 databases for idle-database testing. /// "ActiveDB" will have query_stats activity (seeded separately). @@ -2554,7 +2600,7 @@ public async Task SeedVmRightSizingTargetAsync() // 32 cores, 256GB RAM, but P95 CPU only 12%, buffer pool 50GB of 256GB (19%) // Should recommend: 8 cores (P95 < 15%), 64GB RAM (ratio < 25%) - await SeedCpuUtilizationAsync(12, 2); + await SeedFinOpsCpuUtilizationAsync(12, 2); await SeedMemoryStatsAsync(totalPhysicalMb: 262_144, bufferPoolMb: 51_200, targetMb: 245_760); await SeedServerPropertiesAsync(cpuCount: 32, htRatio: 2, physicalMemMb: 262_144); await SeedFileSizeAsync(totalDataSizeMb: 51_200); @@ -2623,7 +2669,7 @@ public async Task SeedAzureSqlDbVcoreAsync() // Azure SQL DB: node has 20 cores, but this DB has HS_Gen5_14 (14 vCores) // CPU at 8% avg — overprovisioned relative to 14 vCores - await SeedCpuUtilizationAsync(8, 2); + await SeedFinOpsCpuUtilizationAsync(8, 2); await SeedMemoryStatsAsync(totalPhysicalMb: 65_536, bufferPoolMb: 40_960, targetMb: 57_344); await SeedServerPropertiesAsync(cpuCount: 20, htRatio: 1, physicalMemMb: 65_536, edition: "SQL Azure", engineEdition: 5, @@ -2648,6 +2694,7 @@ internal async Task SeedCpuUtilizationWithVarianceAsync(int mean, int variance) // Pattern: mean-variance, mean, mean+variance, mean — repeating var offsets = new[] { -variance, 0, variance, 0 }; + var times = FinOpsCpuSampleTimes(_utcNow(), 32); for (var i = 0; i < 32; i++) { @@ -2660,7 +2707,7 @@ INSERT INTO cpu_utilization_stats sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization) VALUES ($1, $2, $3, $4, $5, $6, $7)"; - var t = TestPeriodStart.AddMinutes(i * 15); + var t = times[i]; cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- }); cmd.Parameters.Add(new DuckDBParameter { Value = t }); cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId }); @@ -2683,6 +2730,7 @@ internal async Task SeedCpuUtilizationAlternatingAsync(int low, int high) using var readLock = _duckDb.AcquireReadLock(); var connection = await SeedConnectionAsync(); using var batch = new SeedBatch(connection); + var times = FinOpsCpuSampleTimes(_utcNow(), 32); for (var i = 0; i < 32; i++) { @@ -2695,7 +2743,7 @@ INSERT INTO cpu_utilization_stats sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization) VALUES ($1, $2, $3, $4, $5, $6, $7)"; - var t = TestPeriodStart.AddMinutes(i * 15); + var t = times[i]; cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- }); cmd.Parameters.Add(new DuckDBParameter { Value = t }); cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId });