Skip to content

Lite FinOps tests: CPU samples sit inside the 24-hour utilization read at any time of day #4891

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erikdarlingdata merged 3 commits into from
Oct 1, 2026
Merged

Lite FinOps tests: CPU samples sit inside the 24-hour utilization read at any time of day #4891

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c909db1
Lite FinOps tests: CPU samples sit inside the 24-hour utilization rea…
erikdarlingdata Oct 1, 2026
d253010
FinOpsCpuSampleWindowTests: seed on a day 30 days back, so a scenario…
erikdarlingdata Oct 1, 2026
6d57ae9
Lite FinOps tests: the clean server keeps 32 CPU samples, so rule 14 …
erikdarlingdata Oct 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 159 additions & 0 deletions Lite.Tests/FinOpsCpuSampleWindowTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
/*
* Copyright (c) 2026 Erik Darling, Darling Data LLC
*
* This file is part of the SQL Server Performance Monitor Lite.
*
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/

using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Darling.Tests;
using DuckDB.NET.Data;
using Lite.Tests;
using PerformanceMonitorLite.Analysis;
using PerformanceMonitorLite.Database;
using Xunit;

namespace PerformanceMonitorLite.Tests;

/// <summary>
/// Every FinOps scenario's CPU samples must sit inside the window the FinOps utilization read uses
/// (<c>GetUtilizationEfficiencyAsync</c>: the last 24 hours from now), at any time of day the suite runs.
///
/// <para>Seeded from <see cref="TestDataSeeder.TestPeriodStart"/>, which is anchored to 04:00 UTC (#4385), the
/// samples sat 24 to 28 hours back between 03:45 and 04:00 UTC. The read found none, so HasCpuSample was false
/// and the CPU and VM right-sizing rules gave no advice: six FinOpsTests failed every day in that quarter hour.
/// The seeder takes a clock, so this class places the samples as a run at 03:50 would, without waiting for the
/// real clock to get there. The source pin below keeps the window this class checks equal to the read's.</para>
/// </summary>
public sealed class FinOpsCpuSampleWindowTests : IClassFixture<SharedDuckDbFixture>
{
/// <summary>The hours <c>GetUtilizationEfficiencyAsync</c> reads back from now. Pinned to its source below.</summary>
private const int UtilizationWindowHours = 24;

/// <summary>Every FinOps scenario that seeds cpu_utilization_stats. A new one goes here too.</summary>
private static readonly Dictionary<string, Func<TestDataSeeder, Task>> Scenarios = new()
{
["OverProvisionedEnterprise"] = s => s.SeedOverProvisionedEnterpriseAsync(),
["RightSizing"] = s => s.SeedRightSizingScenarioAsync(engineEdition: 3, withCpuSamples: true),
["VmRightSizingTarget"] = s => s.SeedVmRightSizingTargetAsync(),
["AzureSqlDbVcore"] = s => s.SeedAzureSqlDbVcoreAsync(),
["CleanFinOpsServer"] = s => s.SeedCleanFinOpsServerAsync(),
["StableCpuForReservedCapacity"] = s => s.SeedStableCpuForReservedCapacityAsync(),
["BurstyCpu"] = s => s.SeedBurstyCpuAsync()
};

private readonly DuckDbInitializer _duckDb;

public FinOpsCpuSampleWindowTests(SharedDuckDbFixture fixture)
{
fixture.ResetData();
_duckDb = fixture.DuckDb;
}

/// <summary>
/// Each scenario at 03:50 UTC (inside the quarter hour before the 04:00 anchor rolls forward a day, when the
/// anchored samples were all more than 24 hours old) and at 12:00 UTC (an ordinary hour).
///
/// <para>Both are on a day 30 days before the run. A sample placed from <see cref="TestDataSeeder.TestPeriodStart"/>,
/// which the real clock anchors, then falls outside the window at either time, so a scenario that goes back to
/// the anchored seed fails here on any day, not only in the quarter hour it breaks in.</para>
/// </summary>
public static TheoryData<string, int, int> ScenariosAtTimesOfDay()
{
var data = new TheoryData<string, int, int>();
foreach (var scenario in Scenarios.Keys)
{
data.Add(scenario, 3, 50);
data.Add(scenario, 12, 0);
}
return data;
}

[Theory]
[MemberData(nameof(ScenariosAtTimesOfDay))]
public async Task EveryCpuSampleIsInsideTheUtilizationWindow(string scenario, int hour, int minute)
{
var nowUtc = DateTime.UtcNow.Date.AddDays(-30).AddHours(hour).AddMinutes(minute);

using (var seeder = new TestDataSeeder(_duckDb, () => nowUtc))
{
await Scenarios[scenario](seeder);
}

using var readLock = _duckDb.AcquireReadLock();
using var connection = _duckDb.CreateConnection();
await connection.OpenAsync();
using var command = connection.CreateCommand();
command.CommandText = @"
SELECT
COUNT(*) AS seeded,
COUNT(*) FILTER (WHERE collection_time >= $2 AND collection_time <= $3) AS in_window
FROM v_cpu_utilization_stats
WHERE server_id = $1";
command.Parameters.Add(new DuckDBParameter { Value = TestDataSeeder.TestServerId });
command.Parameters.Add(new DuckDBParameter { Value = nowUtc.AddHours(-UtilizationWindowHours) });
command.Parameters.Add(new DuckDBParameter { Value = nowUtc });

using var reader = await command.ExecuteReaderAsync();
Assert.True(await reader.ReadAsync());
var seeded = Convert.ToInt64(reader.GetValue(0));
var inWindow = Convert.ToInt64(reader.GetValue(1));

Assert.True(seeded > 0, $"{scenario} seeded no CPU samples.");
if (inWindow != seeded)
{
Assert.Fail(
$"{scenario} at {nowUtc:HH:mm} UTC: {inWindow} of its {seeded} CPU samples are inside the " +
$"{UtilizationWindowHours}-hour utilization read " +
$"({nowUtc.AddHours(-UtilizationWindowHours):yyyy-MM-dd HH:mm} to {nowUtc:yyyy-MM-dd HH:mm} UTC).");
}
}

/// <summary>
/// The read's CPU window is the last <see cref="UtilizationWindowHours"/> hours from now: the cutoff, the
/// parameter that carries it, and the cpu_stats filter that uses it.
/// </summary>
[Fact]
public void UtilizationReadCpuWindowIsTheLast24Hours()
{
const string Signature = "Task<UtilizationEfficiencyRow?> GetUtilizationEfficiencyAsync(";

var source = ParitySource.ReadFile("Lite/Services/LocalDataService.FinOps.Utilization.cs");
var stripped = CSharpSourceWalker.StripCommentsAndStrings(source);

var at = stripped.IndexOf(Signature, StringComparison.Ordinal);
Assert.True(at >= 0, "GetUtilizationEfficiencyAsync was not found.");
Assert.Equal(-1, stripped.IndexOf(Signature, at + 1, StringComparison.Ordinal));

var open = stripped.IndexOf('{', at);
var body = CSharpSourceWalker.BraceBalanced(stripped, open);
var code = Regex.Replace(body, @"\s+", " ");

Assert.Contains($"var cutoff = DateTime.UtcNow.AddHours(-{UtilizationWindowHours});", code);
Assert.Contains(
"command.Parameters.Add(new DuckDBParameter { Value = serverId }); " +
"command.Parameters.Add(new DuckDBParameter { Value = cutoff });",
code);

/* The SQL is a string literal, blanked in the stripped copy: read it from the same span of the source. */
var rawBody = source.Substring(open, body.Length);
var cte = rawBody.IndexOf("cpu_stats AS (", StringComparison.Ordinal);
Assert.True(cte >= 0, "The cpu_stats CTE was not found.");

var depth = 0;
var end = rawBody.IndexOf('(', cte);
for (; end < rawBody.Length; end++)
{
if (rawBody[end] == '(') depth++;
else if (rawBody[end] == ')' && --depth == 0) break;
}

var cpuStats = Regex.Replace(rawBody[cte..end], @"\s+", " ");
Assert.Contains("FROM v_cpu_utilization_stats", cpuStats);
Assert.Contains("collection_time >= $2", cpuStats);
}
}
112 changes: 80 additions & 32 deletions Lite.Tests/TestDataSeeder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,16 @@ private static DateTime AnchorPeriodEndToUtcMidnight(DateTime nowUtc)

private long _nextId = -1_000_000;

public TestDataSeeder(DuckDbInitializer duckDb)
/// <summary>The clock the FinOps CPU samples are placed by (<see cref="FinOpsCpuSampleTimes"/>).</summary>
private readonly Func<DateTime> _utcNow;

/// <param name="duckDb">The database to seed.</param>
/// <param name="utcNow">The clock the FinOps scenarios place their CPU samples by. Omitted, it is the real UTC
/// clock. A test passes a fixed instant to place the samples as a run at that time of day would.</param>
public TestDataSeeder(DuckDbInitializer duckDb, Func<DateTime>? utcNow = null)
{
_duckDb = duckDb;
_utcNow = utcNow ?? (() => DateTime.UtcNow);
}

public void Dispose() => _seedConn?.Dispose();
Expand Down Expand Up @@ -1338,7 +1345,11 @@ INSERT INTO server_config
}

/// <summary>
/// Seeds cpu_utilization_stats across 16 collection points.
/// Seeds cpu_utilization_stats across 16 collection points, 15 minutes apart from <see cref="TestPeriodStart"/>:
/// the 04:00 UTC anchor the analysis scenarios need. A FinOps scenario uses
/// <see cref="SeedFinOpsCpuUtilizationAsync"/> instead, because the FinOps utilization read keeps the last 24
/// hours from now and the anchored samples are older than that between 03:45 and 04:00 UTC. A new FinOps
/// scenario also goes in FinOpsCpuSampleWindowTests' scenario list.
/// </summary>
internal async Task SeedCpuUtilizationAsync(int avgSqlCpu, int avgOtherCpu)
{
Expand Down Expand Up @@ -2130,7 +2141,7 @@ public async Task SeedOverProvisionedEnterpriseAsync()
await SeedTestServerAsync();

// 32 cores, 256GB RAM, but avg CPU 8%, buffer pool only 40GB of 256GB
await SeedCpuUtilizationAsync(8, 2);
await SeedFinOpsCpuUtilizationAsync(8, 2);
await SeedMemoryStatsAsync(totalPhysicalMb: 262_144, bufferPoolMb: 40_960, targetMb: 245_760);
await SeedServerPropertiesAsync(cpuCount: 32, htRatio: 2, physicalMemMb: 262_144,
edition: "Enterprise Edition");
Expand All @@ -2156,7 +2167,7 @@ public async Task SeedRightSizingScenarioAsync(int engineEdition, bool withCpuSa

if (withCpuSamples)
{
await SeedCpuUtilizationAsync(8, 2);
await SeedFinOpsCpuUtilizationAsync(8, 2);
}

var azureSqlDatabase = engineEdition == 5;
Expand Down Expand Up @@ -2244,30 +2255,14 @@ public async Task SeedCleanFinOpsServerAsync()

// Healthy: 50% CPU, 75% buffer pool ratio, no idle databases.
//
// GetUtilizationEfficiencyAsync (the CPU right-sizing check's data source) reads
// collection_time >= DateTime.UtcNow.AddHours(-24) -- a fixed 24h window measured from
// "now", not from TestPeriodEnd. TestPeriodStart/End is anchored to the most recent
// UTC-midnight-plus-4h boundary (#4385), which can land up to ~28h before "now" (worst
// case: 03:59 UTC, one minute before the anchor rolls forward a day). At that worst
// hour, SeedCpuUtilizationAsync's 16 points (TestPeriodStart .. TestPeriodStart+3h45m)
// fall entirely outside a naive 24h-from-now lookback, so the CPU check would read zero
// samples and could compute a false P95 -- so seed a second, always-in-window copy of
// the same healthy CPU signal anchored to "now" instead of TestPeriodStart, the same
// fix shape as #4558 but on the seed side (no hoursBack parameter exists on this read
// path to move to the test side instead).
// variance: 0 -- SeedCpuUtilizationAsync's own 16 points are a flat 50 (no jitter), so
// matching that here keeps the combined 32-point series' stddev at exactly 0 across
// both time windows the FinOps engine reads (24h-from-now for the CPU right-sizing
// check, 7-day for reserved-capacity). Rule 14 (reserved capacity, ~line 806 in
// LocalDataService.FinOps.Recommendations.cs) only fires when avgCpu > 20 AND
// stddevCpu > 0 AND CV (stddev/avg) < 0.3; a nonzero variance here (previously 5,
// giving avg ~49.3%, CV ~0.04) made it fire at every hour. stddev == 0 keeps that
// condition false regardless of the clock. P95 stays at 50%, well clear of rule 2's
// "CPU over-provisioned" P95 < 30% threshold, at any hour too.
await SeedCpuUtilizationAsync(50, 5);
await SeedCpuUtilizationInRangeAsync(
DateTime.UtcNow.AddHours(-3), DateTime.UtcNow.AddMinutes(-5),
avgCpu: 50, variance: 0, samples: 16);
// The CPU is 32 samples of a flat 50. Rule 14 (reserved capacity) reads 7 days and returns no
// row under 24 samples, so 32 is what puts this server in front of its guard: it fires only
// when avgCpu > 20 AND stddevCpu > 0 AND CV < 0.3. A flat 50 has a standard deviation of
// exactly 0, and that is what keeps it quiet; a jittered series (variance 5, CV ~0.04) made
// it fire. With 16 samples the rule would never reach the guard, and this scenario would stop
// holding it. The P95 of 50 is well clear of rule 2's "CPU over-provisioned" P95 < 30%.
// SeedFinOpsCpuUtilizationAsync puts the samples inside the 24-hour read at any time of day.
await SeedFinOpsCpuUtilizationAsync(50, 5, samples: 32);
await SeedMemoryStatsAsync(totalPhysicalMb: 65_536, bufferPoolMb: 49_152, targetMb: 57_344);
await SeedServerPropertiesAsync(cpuCount: 8, htRatio: 2, physicalMemMb: 65_536,
edition: "Developer Edition");
Expand Down Expand Up @@ -2301,6 +2296,57 @@ await SeedServerPropertiesAsync(cpuCount: 8, htRatio: 2, physicalMemMb: 65_536,
// FinOps Seed Helpers
// ============================================

/// <summary>
/// The times of a FinOps scenario's CPU samples for a run at <paramref name="nowUtc"/>: 15 minutes apart, oldest
/// first, the newest 5 minutes before <paramref name="nowUtc"/>.
///
/// <para>The FinOps utilization read (GetUtilizationEfficiencyAsync, behind CPU and VM right-sizing) keeps the
/// last 24 hours from now, so these samples are placed from now. Placed from <see cref="TestPeriodStart"/> like
/// the analysis scenarios' samples, they were 24 to 28 hours old between 03:45 and 04:00 UTC and the read found
/// none. The analysis scenarios keep the 04:00 anchor (see <see cref="_periodEnd"/>).</para>
/// </summary>
internal static DateTime[] FinOpsCpuSampleTimes(DateTime nowUtc, int samples)
{
var newest = nowUtc.AddMinutes(-5);
var times = new DateTime[samples];
for (var i = 0; i < samples; i++)
{
times[i] = newest.AddMinutes(-15 * (samples - 1 - i));
}
return times;
}

/// <summary>
/// Seeds a FinOps scenario's cpu_utilization_stats: <paramref name="samples"/> samples at
/// <see cref="FinOpsCpuSampleTimes"/>, each with the given SQL Server and other-process CPU.
/// </summary>
internal async Task SeedFinOpsCpuUtilizationAsync(int avgSqlCpu, int avgOtherCpu, int samples = 16)
{
using var readLock = _duckDb.AcquireReadLock();
var connection = await SeedConnectionAsync();
using var batch = new SeedBatch(connection);

foreach (var t in FinOpsCpuSampleTimes(_utcNow(), samples))
{
using var cmd = connection.CreateCommand();
cmd.CommandText = @"
INSERT INTO cpu_utilization_stats
(collection_id, collection_time, server_id, server_name,
sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization)
VALUES ($1, $2, $3, $4, $5, $6, $7)";

cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- });
cmd.Parameters.Add(new DuckDBParameter { Value = t });
cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId });
cmd.Parameters.Add(new DuckDBParameter { Value = TestServerName });
cmd.Parameters.Add(new DuckDBParameter { Value = t });
cmd.Parameters.Add(new DuckDBParameter { Value = avgSqlCpu });
cmd.Parameters.Add(new DuckDBParameter { Value = avgOtherCpu });

await cmd.ExecuteNonQueryAsync();
}
}

/// <summary>
/// Seeds database_size_stats with 3 databases for idle-database testing.
/// "ActiveDB" will have query_stats activity (seeded separately).
Expand Down Expand Up @@ -2554,7 +2600,7 @@ public async Task SeedVmRightSizingTargetAsync()

// 32 cores, 256GB RAM, but P95 CPU only 12%, buffer pool 50GB of 256GB (19%)
// Should recommend: 8 cores (P95 < 15%), 64GB RAM (ratio < 25%)
await SeedCpuUtilizationAsync(12, 2);
await SeedFinOpsCpuUtilizationAsync(12, 2);
await SeedMemoryStatsAsync(totalPhysicalMb: 262_144, bufferPoolMb: 51_200, targetMb: 245_760);
await SeedServerPropertiesAsync(cpuCount: 32, htRatio: 2, physicalMemMb: 262_144);
await SeedFileSizeAsync(totalDataSizeMb: 51_200);
Expand Down Expand Up @@ -2623,7 +2669,7 @@ public async Task SeedAzureSqlDbVcoreAsync()

// Azure SQL DB: node has 20 cores, but this DB has HS_Gen5_14 (14 vCores)
// CPU at 8% avg — overprovisioned relative to 14 vCores
await SeedCpuUtilizationAsync(8, 2);
await SeedFinOpsCpuUtilizationAsync(8, 2);
await SeedMemoryStatsAsync(totalPhysicalMb: 65_536, bufferPoolMb: 40_960, targetMb: 57_344);
await SeedServerPropertiesAsync(cpuCount: 20, htRatio: 1, physicalMemMb: 65_536,
edition: "SQL Azure", engineEdition: 5,
Expand All @@ -2648,6 +2694,7 @@ internal async Task SeedCpuUtilizationWithVarianceAsync(int mean, int variance)

// Pattern: mean-variance, mean, mean+variance, mean — repeating
var offsets = new[] { -variance, 0, variance, 0 };
var times = FinOpsCpuSampleTimes(_utcNow(), 32);

for (var i = 0; i < 32; i++)
{
Expand All @@ -2660,7 +2707,7 @@ INSERT INTO cpu_utilization_stats
sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization)
VALUES ($1, $2, $3, $4, $5, $6, $7)";

var t = TestPeriodStart.AddMinutes(i * 15);
var t = times[i];
cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- });
cmd.Parameters.Add(new DuckDBParameter { Value = t });
cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId });
Expand All @@ -2683,6 +2730,7 @@ internal async Task SeedCpuUtilizationAlternatingAsync(int low, int high)
using var readLock = _duckDb.AcquireReadLock();
var connection = await SeedConnectionAsync();
using var batch = new SeedBatch(connection);
var times = FinOpsCpuSampleTimes(_utcNow(), 32);

for (var i = 0; i < 32; i++)
{
Expand All @@ -2695,7 +2743,7 @@ INSERT INTO cpu_utilization_stats
sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization)
VALUES ($1, $2, $3, $4, $5, $6, $7)";

var t = TestPeriodStart.AddMinutes(i * 15);
var t = times[i];
cmd.Parameters.Add(new DuckDBParameter { Value = _nextId-- });
cmd.Parameters.Add(new DuckDBParameter { Value = t });
cmd.Parameters.Add(new DuckDBParameter { Value = TestServerId });
Expand Down
Loading