From fce05edbeae1fbd43d515f9d8dcf939362db970d Mon Sep 17 00:00:00 2001 From: Dejmenek Date: Tue, 29 Sep 2026 21:35:40 +0200 Subject: [PATCH 1/5] test: add load phases and budgets for spike, soak and breakpoint tests LoadPhases turns a list of named phases into Inject simulations and names each step after the phase that is running when it starts. LoadThreshold takes an optional abort count, so a runaway breakpoint run can stop early through AbortOnErrors. LoadTestBudgets gets Spike, Soak and Breakpoint groups. The spike budgets come from the worst of three runs with the usual headroom. Co-Authored-By: Claude Sonnet 5.5 --- TournamentAPI.LoadTests/LoadPhases.cs | 32 ++++++++++++++ TournamentAPI.LoadTests/LoadTestBudgets.cs | 51 ++++++++++++++++++++++ TournamentAPI.LoadTests/LoadThresholds.cs | 29 +++++++++--- 3 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 TournamentAPI.LoadTests/LoadPhases.cs diff --git a/TournamentAPI.LoadTests/LoadPhases.cs b/TournamentAPI.LoadTests/LoadPhases.cs new file mode 100644 index 0000000..3b63e7e --- /dev/null +++ b/TournamentAPI.LoadTests/LoadPhases.cs @@ -0,0 +1,32 @@ +using NBomber.Contracts; +using NBomber.CSharp; + +namespace TournamentAPI.LoadTests; + +internal sealed record LoadPhase(string Name, int RatePerSecond, TimeSpan Duration); + +internal static class LoadPhases +{ + public static LoadSimulation[] ToSimulations(IEnumerable phases) + { + return phases + .Select(phase => Simulation.Inject(phase.RatePerSecond, TimeSpan.FromSeconds(1), phase.Duration)) + .ToArray(); + } + + public static string NameAt(IReadOnlyList phases, TimeSpan elapsed) + { + var phaseEnd = TimeSpan.Zero; + + foreach (var phase in phases) + { + phaseEnd += phase.Duration; + if (elapsed < phaseEnd) + { + return phase.Name; + } + } + + return phases[^1].Name; + } +} diff --git a/TournamentAPI.LoadTests/LoadTestBudgets.cs b/TournamentAPI.LoadTests/LoadTestBudgets.cs index 0c45749..951ebec 100644 --- a/TournamentAPI.LoadTests/LoadTestBudgets.cs +++ b/TournamentAPI.LoadTests/LoadTestBudgets.cs @@ -166,4 +166,55 @@ public static class Ramp public const double MaxTopToBottomP95Ratio = 6; public const double MaxRecoveryP95Ratio = 2; } + + public static class Spike + { + public const int BaselineRatePerSecond = 25; + public const int BurstRatePerSecond = 150; + + public const int WarmUpSeconds = 15; + public const int BaselineSeconds = 15; + public const int BurstSeconds = 10; + public const int DrainSeconds = 15; + + public const double BaselineP95Ms = 280; + public const double BaselineP99Ms = 340; + public const double BaselineMaxMs = 1_000; + + public const double BurstP95Ms = 5_030; + public const double BurstP99Ms = 6_860; + public const double BurstMaxMs = 13_920; + + public const double MinMeanPayloadBytes = 3_250; + public const double MaxMeanPayloadBytes = 8_680; + + public const double MaxRecoveryP95Ratio = 2; + public const double RecoveryP95FloorMs = 100; + } + + public static class Soak + { + public const int WarmUpSeconds = 30; + public const int SegmentSeconds = 120; + public const int SegmentCount = 6; + + public const double MaxDriftP95Ratio = 2; + public const double DriftP95FloorMs = 50; + + public const long MaxManagedMemoryGrowthBytes = 150L * 1024 * 1024; + } + + public static class Breakpoint + { + public static readonly int[] RatesPerSecond = [25, 50, 75, 100, 125, 150]; + + public const int WarmUpSeconds = 15; + public const int PhaseSeconds = 15; + + public const double CeilingP95Ms = 800; + public const int MinBreakpointRatePerSecond = 50; + + public const double SafetyMaxFailPercent = 50; + public const int SafetyAbortWhenErrorCount = 3; + } } diff --git a/TournamentAPI.LoadTests/LoadThresholds.cs b/TournamentAPI.LoadTests/LoadThresholds.cs index 4e9e280..67faa20 100644 --- a/TournamentAPI.LoadTests/LoadThresholds.cs +++ b/TournamentAPI.LoadTests/LoadThresholds.cs @@ -13,25 +13,31 @@ internal sealed class LoadThreshold private readonly Expression>? _stepCheck; private readonly Func? _compiledScenarioCheck; private readonly Func? _compiledStepCheck; + private readonly int? _abortWhenErrorCount; private LoadThreshold( string name, string? stepName, Expression>? scenarioCheck, - Expression>? stepCheck) + Expression>? stepCheck, + int? abortWhenErrorCount = null) { Name = name; _stepName = stepName; _scenarioCheck = scenarioCheck; _stepCheck = stepCheck; + _abortWhenErrorCount = abortWhenErrorCount; _compiledScenarioCheck = scenarioCheck?.Compile(); _compiledStepCheck = stepCheck?.Compile(); } public string Name { get; } - public static LoadThreshold ForScenario(string name, Expression> check) - => new(name, null, check, null); + public static LoadThreshold ForScenario( + string name, + Expression> check, + int? abortWhenErrorCount = null) + => new(name, null, check, null, abortWhenErrorCount); public static LoadThreshold ForStep(string name, string stepName, Expression> check) => new(name, stepName, null, check); @@ -45,9 +51,14 @@ public bool IsSatisfiedBy(ScenarioStats stats) public Threshold ToNBomberThreshold() { - return _stepName is null - ? Threshold.Create(_scenarioCheck!) - : Threshold.Create(_stepName, _stepCheck!); + if (_stepName is not null) + { + return Threshold.Create(_stepName, _stepCheck!); + } + + return _abortWhenErrorCount is { } abortWhenErrorCount + ? Threshold.Create(_scenarioCheck!, abortWhenErrorCount: abortWhenErrorCount) + : Threshold.Create(_scenarioCheck!); } } @@ -58,6 +69,12 @@ public static LoadThreshold ErrorBudget(double maxFailPercent) $"error budget: fail rate <= {maxFailPercent}%", s => s.Fail.Request.Percent <= maxFailPercent); + public static LoadThreshold AbortOnErrors(double maxFailPercent, int abortWhenErrorCount) + => LoadThreshold.ForScenario( + $"safety abort: fail rate <= {maxFailPercent}%", + s => s.Fail.Request.Percent <= maxFailPercent, + abortWhenErrorCount); + public static LoadThreshold TailLatency(string stepName, double p95Ms, double p99Ms, double maxMs) => LoadThreshold.ForStep( $"tail latency [{stepName}]: p95 <= {p95Ms}ms, p99 <= {p99Ms}ms, max <= {maxMs}ms", From 18721617ec79c6237b6e5fbf0f978115ef8d336c Mon Sep 17 00:00:00 2001 From: Dejmenek Date: Tue, 29 Sep 2026 21:35:40 +0200 Subject: [PATCH 2/5] test: add spike load test A heavy list query runs at 25 requests per second, jumps to 150 for 10 seconds, drains, and returns to 25. The test checks the error budget, tail latency and payload size per phase. It also requires the p95 after the spike to stay within twice the p95 before it, with a 100 ms floor. A cheap list query showed a p95 of 22 ms even at 200 requests per second, so it could not stress the API. Co-Authored-By: Claude Sonnet 5.5 --- TournamentAPI.LoadTests/SpikeLoadTests.cs | 93 +++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 TournamentAPI.LoadTests/SpikeLoadTests.cs diff --git a/TournamentAPI.LoadTests/SpikeLoadTests.cs b/TournamentAPI.LoadTests/SpikeLoadTests.cs new file mode 100644 index 0000000..2d33815 --- /dev/null +++ b/TournamentAPI.LoadTests/SpikeLoadTests.cs @@ -0,0 +1,93 @@ +using NBomber.Contracts; +using NBomber.Contracts.Stats; +using NBomber.CSharp; +using TournamentQueries = TournamentAPI.Shared.QueryExamples.Queries.Tournaments; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Load")] +public class SpikeLoadTests : BaseLoadTest, IClassFixture +{ + private const string WarmUp = "spike_warm_up"; + private const string PreSpike = "pre_spike"; + private const string Spike = "spike"; + private const string Drain = "spike_drain"; + private const string PostSpike = "post_spike"; + + private static readonly LoadPhase[] Phases = + [ + new(WarmUp, LoadTestBudgets.Spike.BaselineRatePerSecond, TimeSpan.FromSeconds(LoadTestBudgets.Spike.WarmUpSeconds)), + new(PreSpike, LoadTestBudgets.Spike.BaselineRatePerSecond, TimeSpan.FromSeconds(LoadTestBudgets.Spike.BaselineSeconds)), + new(Spike, LoadTestBudgets.Spike.BurstRatePerSecond, TimeSpan.FromSeconds(LoadTestBudgets.Spike.BurstSeconds)), + new(Drain, LoadTestBudgets.Spike.BaselineRatePerSecond, TimeSpan.FromSeconds(LoadTestBudgets.Spike.DrainSeconds)), + new(PostSpike, LoadTestBudgets.Spike.BaselineRatePerSecond, TimeSpan.FromSeconds(LoadTestBudgets.Spike.BaselineSeconds)) + ]; + + public SpikeLoadTests(MediumDatasetWebAppFactory factory) : base(factory) + { + } + + [Fact] + public void Api_ShouldAbsorbASpike_AndRecoverToBaselineLatency() + { + // Arrange + using var client = CreateClient(); + var scenario = GraphQLLoadScenario.Create("browse_tournaments_spike", async context => + await GraphQLLoadStep.RunAsync( + LoadPhases.NameAt(Phases, context.GetScenarioTimerTime()), + context, + client.HttpClient, + TournamentQueries.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) + .WithoutWarmUp() + .WithLoadSimulations(LoadPhases.ToSimulations(Phases)); + + var thresholds = new List + { + LoadThresholds.ErrorBudget(0), + LoadThresholds.StatusMix("200", 100) + }; + + foreach (var phase in new[] { PreSpike, PostSpike }) + { + thresholds.Add(LoadThresholds.StepHealth(phase, 0)); + thresholds.Add(LoadThresholds.TailLatency( + phase, + LoadTestBudgets.Spike.BaselineP95Ms, + LoadTestBudgets.Spike.BaselineP99Ms, + LoadTestBudgets.Spike.BaselineMaxMs)); + } + + thresholds.Add(LoadThresholds.StepHealth(Spike, 0)); + thresholds.Add(LoadThresholds.TailLatency( + Spike, + LoadTestBudgets.Spike.BurstP95Ms, + LoadTestBudgets.Spike.BurstP99Ms, + LoadTestBudgets.Spike.BurstMaxMs)); + + foreach (var phase in new[] { PreSpike, Spike, PostSpike }) + { + thresholds.Add(LoadThresholds.PayloadGuard( + phase, + LoadTestBudgets.Spike.MinMeanPayloadBytes, + LoadTestBudgets.Spike.MaxMeanPayloadBytes)); + } + + // Act + var stats = LoadTestRun.Execute(scenario, thresholds.ToArray()); + + // Assert + var preSpikeP95 = P95(stats, PreSpike); + var postSpikeP95 = P95(stats, PostSpike); + var allowedP95 = Math.Max( + preSpikeP95 * LoadTestBudgets.Spike.MaxRecoveryP95Ratio, + LoadTestBudgets.Spike.RecoveryP95FloorMs); + + Assert.True( + postSpikeP95 <= allowedP95, + $"p95 after the spike ({postSpikeP95}ms) exceeds {LoadTestBudgets.Spike.MaxRecoveryP95Ratio}x the p95 before it ({preSpikeP95}ms) and the {LoadTestBudgets.Spike.RecoveryP95FloorMs}ms floor"); + } + + private static double P95(ScenarioStats stats, string stepName) + => stats.StepStats.Get(stepName).Ok.Latency.Percent95; +} From 9b51c75c9c84840d956caab0beb2fd86e94f8854 Mon Sep 17 00:00:00 2001 From: Dejmenek Date: Tue, 29 Sep 2026 21:35:40 +0200 Subject: [PATCH 3/5] test: add soak load test The mixed browsing load runs for about 12 minutes as a warm up and six named two minute segments. Every step is checked against its Medium baseline per segment, and the last segment must not drift past twice the first. Managed memory is sampled after forced collections at the end of the first segment and at the end of the run. The test carries Category=Soak only, so the normal load run skips it. The 150 MB growth limit has not been calibrated against a measured run. Co-Authored-By: Claude Sonnet 5.5 --- TournamentAPI.LoadTests/SoakLoadTests.cs | 181 +++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 TournamentAPI.LoadTests/SoakLoadTests.cs diff --git a/TournamentAPI.LoadTests/SoakLoadTests.cs b/TournamentAPI.LoadTests/SoakLoadTests.cs new file mode 100644 index 0000000..87d48a5 --- /dev/null +++ b/TournamentAPI.LoadTests/SoakLoadTests.cs @@ -0,0 +1,181 @@ +using NBomber.Contracts; +using NBomber.Contracts.Stats; +using NBomber.CSharp; +using Xunit.Abstractions; +using TournamentQueries = TournamentAPI.Shared.QueryExamples.Queries.Tournaments; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Soak")] +public class SoakLoadTests : BaseLoadTest, IClassFixture +{ + private const string ListScenario = "soak_browse_tournament_list"; + private const string ViewScenario = "soak_view_tournament_details"; + private const string DiscoverScenario = "soak_discover_tournaments"; + + private static readonly TimeSpan WarmUp = TimeSpan.FromSeconds(LoadTestBudgets.Soak.WarmUpSeconds); + private static readonly TimeSpan Segment = TimeSpan.FromSeconds(LoadTestBudgets.Soak.SegmentSeconds); + private static readonly TimeSpan TotalDuration = WarmUp + Segment * LoadTestBudgets.Soak.SegmentCount; + + private readonly MediumDatasetWebAppFactory _factory; + private readonly ITestOutputHelper _output; + + public SoakLoadTests(MediumDatasetWebAppFactory factory, ITestOutputHelper output) : base(factory) + { + _factory = factory; + _output = output; + } + + [Fact] + public void Api_ShouldHoldItsLatencyAndMemory_UnderSustainedBrowsingTraffic() + { + // Arrange + using var client = CreateClient(); + var http = client.HttpClient; + var tournamentIds = _factory.TournamentIds; + var searchTerm = _factory.TournamentSearchTerm; + + var browseList = GraphQLLoadScenario.Create(ListScenario, async context => + await GraphQLLoadStep.RunAsync( + StepAt(StepNames.ListTournaments, context.GetScenarioTimerTime()), + context, + http, + TournamentQueries.GetAllWithTotalCount, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) + .WithoutWarmUp() + .WithLoadSimulations(Steady(LoadTestBudgets.Rates.ListTournamentsPerSecond)); + + var viewDetails = GraphQLLoadScenario.Create(ViewScenario, async context => + await GraphQLLoadStep.RunAsync( + StepAt(StepNames.ViewTournament, context.GetScenarioTimerTime()), + context, + http, + TournamentQueries.GetByIdWithParticipants, + new { id = tournamentIds[context.Random.Next(tournamentIds.Count)] }, + GraphQLLoadStep.NonNullField("tournamentById"))) + .WithoutWarmUp() + .WithLoadSimulations(Steady(LoadTestBudgets.Rates.ViewTournamentPerSecond)); + + var discover = GraphQLLoadScenario.Create(DiscoverScenario, async context => + { + var search = await GraphQLLoadStep.RunAsync( + StepAt(StepNames.SearchTournamentsByName, context.GetScenarioTimerTime()), + context, + http, + TournamentQueries.GetAllWithNameFilter, + new { nameFilter = searchTerm }, + GraphQLLoadStep.NonEmptyConnection("tournaments")); + + if (search.IsError) + { + return search; + } + + return await GraphQLLoadStep.RunAsync( + StepAt(StepNames.SortTournaments, context.GetScenarioTimerTime()), + context, + http, + TournamentQueries.GetAllWithSorting, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments")); + }) + .WithoutWarmUp() + .WithLoadSimulations(Steady(LoadTestBudgets.Rates.DiscoverTournamentsPerSecond)); + + long? memoryAfterFirstSegment = null; + using var memorySampler = new Timer( + _ => memoryAfterFirstSegment = ManagedMemoryAfterCollection(), + null, + WarmUp + Segment, + Timeout.InfiniteTimeSpan); + + // Act + IReadOnlyDictionary stats; + long memoryAtEnd; + + try + { + stats = LoadTestRun.ExecuteAll( + new ScenarioRun(browseList, ThresholdsFor(StepNames.ListTournaments)), + new ScenarioRun(viewDetails, ThresholdsFor(StepNames.ViewTournament)), + new ScenarioRun(discover, ThresholdsFor(StepNames.SearchTournamentsByName, StepNames.SortTournaments))); + } + finally + { + memoryAtEnd = ManagedMemoryAfterCollection(); + + _output.WriteLine($"SOAK managed memory after collection: {(memoryAfterFirstSegment ?? 0) / 1024 / 1024} MB after the first segment, {memoryAtEnd / 1024 / 1024} MB at the end"); + } + + // Assert + AssertNoLatencyDrift(stats[ListScenario], StepNames.ListTournaments); + AssertNoLatencyDrift(stats[ViewScenario], StepNames.ViewTournament); + AssertNoLatencyDrift(stats[DiscoverScenario], StepNames.SearchTournamentsByName); + AssertNoLatencyDrift(stats[DiscoverScenario], StepNames.SortTournaments); + + Assert.NotNull(memoryAfterFirstSegment); + + var growth = memoryAtEnd - memoryAfterFirstSegment.Value; + + Assert.True( + growth <= LoadTestBudgets.Soak.MaxManagedMemoryGrowthBytes, + $"Managed memory after a full collection grew by {growth / 1024 / 1024} MB between the end of the first segment and the end of the run, above the {LoadTestBudgets.Soak.MaxManagedMemoryGrowthBytes / 1024 / 1024} MB limit"); + } + + private static string StepAt(string step, TimeSpan elapsed) + { + if (elapsed < WarmUp) + { + return $"{step}_warm_up"; + } + + var index = (int)((elapsed - WarmUp) / Segment) + 1; + + return SegmentStep(step, Math.Min(index, LoadTestBudgets.Soak.SegmentCount)); + } + + private static string SegmentStep(string step, int segment) => $"{step}_segment_{segment}"; + + private static LoadSimulation Steady(int ratePerSecond) + => Simulation.Inject(ratePerSecond, TimeSpan.FromSeconds(1), TotalDuration); + + private static LoadThreshold[] ThresholdsFor(params string[] steps) + { + var thresholds = new List + { + LoadThresholds.ErrorBudget(0), + LoadThresholds.StatusMix("200", 100) + }; + + foreach (var step in steps) + { + var budget = LoadTestBudgets.Steps.For(LoadTestDataSize.Medium, step); + + for (var segment = 1; segment <= LoadTestBudgets.Soak.SegmentCount; segment++) + { + thresholds.AddRange(LoadThresholds.ForBudget(budget with { Step = SegmentStep(step, segment) })); + } + } + + return thresholds.ToArray(); + } + + private static void AssertNoLatencyDrift(ScenarioStats stats, string step) + { + var firstP95 = stats.StepStats.Get(SegmentStep(step, 1)).Ok.Latency.Percent95; + var lastP95 = stats.StepStats.Get(SegmentStep(step, LoadTestBudgets.Soak.SegmentCount)).Ok.Latency.Percent95; + var allowedP95 = Math.Max(firstP95 * LoadTestBudgets.Soak.MaxDriftP95Ratio, LoadTestBudgets.Soak.DriftP95FloorMs); + + Assert.True( + lastP95 <= allowedP95, + $"p95 of {step} in the last segment ({lastP95}ms) exceeds {LoadTestBudgets.Soak.MaxDriftP95Ratio}x the first segment ({firstP95}ms) and the {LoadTestBudgets.Soak.DriftP95FloorMs}ms floor"); + } + + private static long ManagedMemoryAfterCollection() + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + return GC.GetTotalMemory(forceFullCollection: true); + } +} From e7db5bbf6065dad0ae138c54bc2a11f5f24603a2 Mon Sep 17 00:00:00 2001 From: Dejmenek Date: Tue, 29 Sep 2026 21:35:41 +0200 Subject: [PATCH 4/5] test: add breakpoint load test A heavy query steps through 25, 50, 75, 100, 125 and 150 requests per second after a warm up phase. The breakpoint is the highest rate whose p95 stays under 800 ms with no failures, and the test asserts it is at least 50 requests per second. It measured 125 in two runs. A safety abort on a very high error rate stops a runaway run without failing the test. Co-Authored-By: Claude Sonnet 5.5 --- .../BreakpointLoadTests.cs | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 TournamentAPI.LoadTests/BreakpointLoadTests.cs diff --git a/TournamentAPI.LoadTests/BreakpointLoadTests.cs b/TournamentAPI.LoadTests/BreakpointLoadTests.cs new file mode 100644 index 0000000..b7187f4 --- /dev/null +++ b/TournamentAPI.LoadTests/BreakpointLoadTests.cs @@ -0,0 +1,96 @@ +using NBomber.Contracts; +using NBomber.Contracts.Stats; +using NBomber.CSharp; +using TournamentQueries = TournamentAPI.Shared.QueryExamples.Queries.Tournaments; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Soak")] +public class BreakpointLoadTests : BaseLoadTest, IClassFixture +{ + private static readonly LoadPhase WarmUp = new( + "breakpoint_warm_up", + LoadTestBudgets.Breakpoint.RatesPerSecond[0], + TimeSpan.FromSeconds(LoadTestBudgets.Breakpoint.WarmUpSeconds)); + + private static readonly LoadPhase[] RatePhases = LoadTestBudgets.Breakpoint.RatesPerSecond + .Select(rate => new LoadPhase( + PhaseName(rate), + rate, + TimeSpan.FromSeconds(LoadTestBudgets.Breakpoint.PhaseSeconds))) + .ToArray(); + + private static readonly LoadPhase[] Phases = [WarmUp, .. RatePhases]; + + public BreakpointLoadTests(MediumDatasetWebAppFactory factory) : base(factory) + { + } + + [Fact] + public void Api_ShouldSustainAtLeastTheDocumentedRate_BeforeLatencyBreaks() + { + // Arrange + using var client = CreateClient(); + var scenario = GraphQLLoadScenario.Create("browse_tournaments_breakpoint", async context => + await GraphQLLoadStep.RunAsync( + LoadPhases.NameAt(Phases, context.GetScenarioTimerTime()), + context, + client.HttpClient, + TournamentQueries.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) + .WithoutWarmUp() + .WithLoadSimulations(LoadPhases.ToSimulations(Phases)) + .WithThresholds( + LoadThresholds.AbortOnErrors( + LoadTestBudgets.Breakpoint.SafetyMaxFailPercent, + LoadTestBudgets.Breakpoint.SafetyAbortWhenErrorCount).ToNBomberThreshold()); + + // Act + var result = NBomberRunner + .RegisterScenarios(scenario) + .Run(); + + // Assert + var stats = result.ScenarioStats.Get(scenario.ScenarioName); + var breakpoint = FindBreakpoint(stats); + + Assert.True( + breakpoint >= LoadTestBudgets.Breakpoint.MinBreakpointRatePerSecond, + $"The breakpoint is {breakpoint} req/s, below the documented floor of {LoadTestBudgets.Breakpoint.MinBreakpointRatePerSecond} req/s. p95 per rate: {DescribeRates(stats)}"); + } + + private static string PhaseName(int rate) => $"breakpoint_rate_{rate}"; + + private static int FindBreakpoint(ScenarioStats stats) + { + var breakpoint = 0; + + foreach (var phase in RatePhases) + { + if (!stats.StepStats.Exists(phase.Name)) + { + break; + } + + var step = stats.StepStats.Get(phase.Name); + var isHealthy = step.Fail.Request.Count == 0 + && step.Ok.Latency.Percent95 <= LoadTestBudgets.Breakpoint.CeilingP95Ms; + + if (!isHealthy) + { + break; + } + + breakpoint = phase.RatePerSecond; + } + + return breakpoint; + } + + private static string DescribeRates(ScenarioStats stats) + { + return string.Join(", ", RatePhases.Select(phase => stats.StepStats.Exists(phase.Name) + ? $"{phase.RatePerSecond} req/s = {stats.StepStats.Get(phase.Name).Ok.Latency.Percent95}ms" + : $"{phase.RatePerSecond} req/s = not reached")); + } +} From 63a6c721c0316f873918842dfe941ffb2717f6ea Mon Sep 17 00:00:00 2001 From: Dejmenek Date: Tue, 29 Sep 2026 21:35:41 +0200 Subject: [PATCH 5/5] docs: add ADR 0009 for soak and breakpoint tests ADR 0009 records how the breakpoint is found and how the soak is run, with the measured breakpoint and spike figures. ADR 0008 gets the headroom rules the budgets use and a link to the new ADR. CLAUDE.md gets the command for the soak and breakpoint tests. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 3 ++ docs/adr/0008_load_test_execution_model.md | 3 ++ .../0009_soak_and_breakpoint_test_design.md | 39 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 docs/adr/0009_soak_and_breakpoint_test_design.md diff --git a/CLAUDE.md b/CLAUDE.md index d4fec99..6181da0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -86,6 +86,9 @@ dotnet test TournamentAPI.IntegrationTests --filter "FullyQualifiedName~Tourname # Run load tests (Docker required, manual only, not part of CI) dotnet test TournamentAPI.LoadTests --filter "Category=Load" +# Run soak and breakpoint tests (Docker required, about 15 minutes, manual only, not part of CI) +dotnet test TournamentAPI.LoadTests --filter "Category=Soak" + # Run benchmarks (must be Release) dotnet run --project TournamentAPI.Benchmarks --configuration Release ``` diff --git a/docs/adr/0008_load_test_execution_model.md b/docs/adr/0008_load_test_execution_model.md index bf09768..6ab5a90 100644 --- a/docs/adr/0008_load_test_execution_model.md +++ b/docs/adr/0008_load_test_execution_model.md @@ -49,7 +49,10 @@ The API runs in-process, and load is generated as an open model with a fixed arr The baseline was measured on a developer machine on 29/09/2026. At 50 req/s the list query had p95 55 ms, p99 104 ms, a maximum of 225 ms and a payload of 3,026 bytes. In the ramp, 100 req/s gave p95 325 ms, p99 488 ms and a maximum of 689 ms, and the run recovered to p95 34 ms after ramping back down. Budgets are set with headroom above those figures: for the list query, p95 150 ms, p99 300 ms and a maximum of 1,000 ms. Ramp windows have their own limits, because peak load is expected to run slower than steady load. +Headroom over the measured values is 3.5x for p95, 3.5x for p99 and 5x for the maximum. Floors of 100 ms, 250 ms and 1,000 ms keep a tiny baseline from producing a limit nobody can meet. Payload limits sit between 0.6x and 1.6x of the measured mean. Where a step varied a lot between runs, the baseline is the worst value over at least three runs. On the mixed browsing load, latency collapsed between 25 and 50 requests per second, with p99 going from about 50 ms to 1,900 ms. + Numbers from the in-process host are comparable between runs on the same machine. They are not production latency figures. ## Related ADRs - ADR #0001 Rate Limiting Algorithm for API Requests: the concurrency limiter and token bucket tests exercise the chained global limiter that ADR decided on. Running them showed the per-IP limiter was attached as an endpoint policy before routing and never took effect, so it was moved into the global limiter. +- ADR #0009 Soak and Breakpoint Test Design: adds phased spike, soak and breakpoint runs on top of this execution model. diff --git a/docs/adr/0009_soak_and_breakpoint_test_design.md b/docs/adr/0009_soak_and_breakpoint_test_design.md new file mode 100644 index 0000000..aec12e6 --- /dev/null +++ b/docs/adr/0009_soak_and_breakpoint_test_design.md @@ -0,0 +1,39 @@ +# Title +Soak and Breakpoint Test Design + +# Date +29/09/2026 + +## Status +Accepted + +## Context +The load tests covered steady traffic, a phased ramp and writes under contention, but nothing checked a sudden burst, a long run, or where the API stops coping. Two design questions came up. The API runs in-process with the limiters turned off, so overload shows up as queueing and slow responses, not as errors. And a soak run has to show drift over time without turning into a set of unrelated short runs. + +## Considered Options + +### For how the breakpoint is found +1. Step through fixed rates in one run and read the breakpoint from per step p95 after the run (chosen) + - Pros: Works when overload appears as latency instead of errors. Every rate keeps its own stats, so a failure message can list p95 for each rate and show the whole curve. A safety abort on a very high error rate still stops a runaway run. + - Cons: The run always goes through every rate, even after the API has clearly broken. A noisy step ends the search early, so the breakpoint can read low. +2. Abort the run as soon as the error budget breaks + - Pros: Stops early and spends no time above the break. + - Cons: The in-process host queues under overload instead of failing, so the run may never abort and would report no breakpoint at all. + +### For how the soak is run +1. One 12 minute run with a warm up phase and six named two minute segments, plus managed memory sampled after forced collections (chosen) + - Pros: One host and one container live for the whole run, so a leak or slow degradation shows up as a trend between segments. Naming steps by segment reuses the phase pattern from the ramp test. + - Cons: The run takes about 12.5 minutes, so it stays manual and carries its own trait (Category=Soak). Each step gets a check per segment, and a busy machine can fail a single segment. The memory check only sees the managed heap of the test process, which includes the API, and the 150 MB growth limit has not been calibrated against a measured run. +2. Separate short runs, one per segment + - Pros: Every segment starts from a clean state. + - Cons: A fresh host per run hides exactly the drift a soak exists to find. + +## Decision +The breakpoint test steps a heavy query through 25, 50, 75, 100, 125 and 150 requests per second, 15 seconds each after a warm up. The breakpoint is the highest rate whose p95 stays under 800 ms with no failures. On the development machine it was 125 requests per second in two runs, and 150 broke the ceiling both times (p95 1,425 ms and 4,116 ms). The test asserts a floor of 50 requests per second. + +The spike test uses the same heavy query. A cheap list query showed a p95 of 22 ms even at 200 requests per second, which proves nothing. The heavy query goes from 25 to 150 requests per second for 10 seconds and back. The burst p95 reached about 1.4 s, and p95 returned to baseline after a 15 second drain. Budgets come from the worst of three runs with the same headroom as the other tests. + +The soak runs the mixed browsing load at the existing rates and compares the last segment with the first. Soak and breakpoint tests carry Category=Soak only, so `Category=Load` never picks them up. + +## Related ADRs +- ADR #0008 Load Test Execution Model: keeps the in-process host, the open arrival model, serialized tests and budgets measured from a baseline. This ADR adds phased spike, soak and breakpoint runs on top of it.