Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ dotnet test TournamentAPI.IntegrationTests --filter "FullyQualifiedName~Tourname
# Run load tests (Docker required, manual only, not part of CI)
dotnet test TournamentAPI.LoadTests --filter "Category=Load"

# Run soak and breakpoint tests (Docker required, about 15 minutes, manual only, not part of CI)
dotnet test TournamentAPI.LoadTests --filter "Category=Soak"

# Run benchmarks (must be Release)
dotnet run --project TournamentAPI.Benchmarks --configuration Release
```
Expand Down
96 changes: 96 additions & 0 deletions TournamentAPI.LoadTests/BreakpointLoadTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
using NBomber.Contracts;
using NBomber.Contracts.Stats;
using NBomber.CSharp;
using TournamentQueries = TournamentAPI.Shared.QueryExamples.Queries.Tournaments;

namespace TournamentAPI.LoadTests;

[Trait("Category", "Soak")]
public class BreakpointLoadTests : BaseLoadTest, IClassFixture<MediumDatasetWebAppFactory>
{
private static readonly LoadPhase WarmUp = new(
"breakpoint_warm_up",
LoadTestBudgets.Breakpoint.RatesPerSecond[0],
TimeSpan.FromSeconds(LoadTestBudgets.Breakpoint.WarmUpSeconds));

private static readonly LoadPhase[] RatePhases = LoadTestBudgets.Breakpoint.RatesPerSecond
.Select(rate => new LoadPhase(
PhaseName(rate),
rate,
TimeSpan.FromSeconds(LoadTestBudgets.Breakpoint.PhaseSeconds)))
.ToArray();

private static readonly LoadPhase[] Phases = [WarmUp, .. RatePhases];

public BreakpointLoadTests(MediumDatasetWebAppFactory factory) : base(factory)
{
}

[Fact]
public void Api_ShouldSustainAtLeastTheDocumentedRate_BeforeLatencyBreaks()
{
// Arrange
using var client = CreateClient();
var scenario = GraphQLLoadScenario.Create("browse_tournaments_breakpoint", async context =>
await GraphQLLoadStep.RunAsync(
LoadPhases.NameAt(Phases, context.GetScenarioTimerTime()),
context,
client.HttpClient,
TournamentQueries.GetAllWithBracketAndMatches,
bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments")))
.WithoutWarmUp()
.WithLoadSimulations(LoadPhases.ToSimulations(Phases))
.WithThresholds(
LoadThresholds.AbortOnErrors(
LoadTestBudgets.Breakpoint.SafetyMaxFailPercent,
LoadTestBudgets.Breakpoint.SafetyAbortWhenErrorCount).ToNBomberThreshold());

// Act
var result = NBomberRunner
.RegisterScenarios(scenario)
.Run();

// Assert
var stats = result.ScenarioStats.Get(scenario.ScenarioName);
var breakpoint = FindBreakpoint(stats);

Assert.True(
breakpoint >= LoadTestBudgets.Breakpoint.MinBreakpointRatePerSecond,
$"The breakpoint is {breakpoint} req/s, below the documented floor of {LoadTestBudgets.Breakpoint.MinBreakpointRatePerSecond} req/s. p95 per rate: {DescribeRates(stats)}");
}

private static string PhaseName(int rate) => $"breakpoint_rate_{rate}";

private static int FindBreakpoint(ScenarioStats stats)
{
var breakpoint = 0;

foreach (var phase in RatePhases)
{
if (!stats.StepStats.Exists(phase.Name))
{
break;
}

var step = stats.StepStats.Get(phase.Name);
var isHealthy = step.Fail.Request.Count == 0
&& step.Ok.Latency.Percent95 <= LoadTestBudgets.Breakpoint.CeilingP95Ms;

if (!isHealthy)
{
break;
}

breakpoint = phase.RatePerSecond;
}

return breakpoint;
}

private static string DescribeRates(ScenarioStats stats)
{
return string.Join(", ", RatePhases.Select(phase => stats.StepStats.Exists(phase.Name)
? $"{phase.RatePerSecond} req/s = {stats.StepStats.Get(phase.Name).Ok.Latency.Percent95}ms"
: $"{phase.RatePerSecond} req/s = not reached"));
}
}
32 changes: 32 additions & 0 deletions TournamentAPI.LoadTests/LoadPhases.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using NBomber.Contracts;
using NBomber.CSharp;

namespace TournamentAPI.LoadTests;

internal sealed record LoadPhase(string Name, int RatePerSecond, TimeSpan Duration);

internal static class LoadPhases
{
public static LoadSimulation[] ToSimulations(IEnumerable<LoadPhase> phases)
{
return phases
.Select(phase => Simulation.Inject(phase.RatePerSecond, TimeSpan.FromSeconds(1), phase.Duration))
.ToArray();
}

public static string NameAt(IReadOnlyList<LoadPhase> phases, TimeSpan elapsed)
{
var phaseEnd = TimeSpan.Zero;

foreach (var phase in phases)
{
phaseEnd += phase.Duration;
if (elapsed < phaseEnd)
{
return phase.Name;
}
}

return phases[^1].Name;
}
}
51 changes: 51 additions & 0 deletions TournamentAPI.LoadTests/LoadTestBudgets.cs
Original file line number Diff line number Diff line change
Expand Up @@ -166,4 +166,55 @@ public static class Ramp
public const double MaxTopToBottomP95Ratio = 6;
public const double MaxRecoveryP95Ratio = 2;
}

public static class Spike
{
public const int BaselineRatePerSecond = 25;
public const int BurstRatePerSecond = 150;

public const int WarmUpSeconds = 15;
public const int BaselineSeconds = 15;
public const int BurstSeconds = 10;
public const int DrainSeconds = 15;

public const double BaselineP95Ms = 280;
public const double BaselineP99Ms = 340;
public const double BaselineMaxMs = 1_000;

public const double BurstP95Ms = 5_030;
public const double BurstP99Ms = 6_860;
public const double BurstMaxMs = 13_920;

public const double MinMeanPayloadBytes = 3_250;
public const double MaxMeanPayloadBytes = 8_680;

public const double MaxRecoveryP95Ratio = 2;
public const double RecoveryP95FloorMs = 100;
}

public static class Soak
{
public const int WarmUpSeconds = 30;
public const int SegmentSeconds = 120;
public const int SegmentCount = 6;

public const double MaxDriftP95Ratio = 2;
public const double DriftP95FloorMs = 50;

public const long MaxManagedMemoryGrowthBytes = 150L * 1024 * 1024;
}

public static class Breakpoint
{
public static readonly int[] RatesPerSecond = [25, 50, 75, 100, 125, 150];

public const int WarmUpSeconds = 15;
public const int PhaseSeconds = 15;

public const double CeilingP95Ms = 800;
public const int MinBreakpointRatePerSecond = 50;

public const double SafetyMaxFailPercent = 50;
public const int SafetyAbortWhenErrorCount = 3;
}
}
29 changes: 23 additions & 6 deletions TournamentAPI.LoadTests/LoadThresholds.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,25 +13,31 @@ internal sealed class LoadThreshold
private readonly Expression<Func<StepStats, bool>>? _stepCheck;
private readonly Func<ScenarioStats, bool>? _compiledScenarioCheck;
private readonly Func<StepStats, bool>? _compiledStepCheck;
private readonly int? _abortWhenErrorCount;

private LoadThreshold(
string name,
string? stepName,
Expression<Func<ScenarioStats, bool>>? scenarioCheck,
Expression<Func<StepStats, bool>>? stepCheck)
Expression<Func<StepStats, bool>>? stepCheck,
int? abortWhenErrorCount = null)
{
Name = name;
_stepName = stepName;
_scenarioCheck = scenarioCheck;
_stepCheck = stepCheck;
_abortWhenErrorCount = abortWhenErrorCount;
_compiledScenarioCheck = scenarioCheck?.Compile();
_compiledStepCheck = stepCheck?.Compile();
}

public string Name { get; }

public static LoadThreshold ForScenario(string name, Expression<Func<ScenarioStats, bool>> check)
=> new(name, null, check, null);
public static LoadThreshold ForScenario(
string name,
Expression<Func<ScenarioStats, bool>> check,
int? abortWhenErrorCount = null)
=> new(name, null, check, null, abortWhenErrorCount);

public static LoadThreshold ForStep(string name, string stepName, Expression<Func<StepStats, bool>> check)
=> new(name, stepName, null, check);
Expand All @@ -45,9 +51,14 @@ public bool IsSatisfiedBy(ScenarioStats stats)

public Threshold ToNBomberThreshold()
{
return _stepName is null
? Threshold.Create(_scenarioCheck!)
: Threshold.Create(_stepName, _stepCheck!);
if (_stepName is not null)
{
return Threshold.Create(_stepName, _stepCheck!);
}

return _abortWhenErrorCount is { } abortWhenErrorCount
? Threshold.Create(_scenarioCheck!, abortWhenErrorCount: abortWhenErrorCount)
: Threshold.Create(_scenarioCheck!);
}
}

Expand All @@ -58,6 +69,12 @@ public static LoadThreshold ErrorBudget(double maxFailPercent)
$"error budget: fail rate <= {maxFailPercent}%",
s => s.Fail.Request.Percent <= maxFailPercent);

public static LoadThreshold AbortOnErrors(double maxFailPercent, int abortWhenErrorCount)
=> LoadThreshold.ForScenario(
$"safety abort: fail rate <= {maxFailPercent}%",
s => s.Fail.Request.Percent <= maxFailPercent,
abortWhenErrorCount);

public static LoadThreshold TailLatency(string stepName, double p95Ms, double p99Ms, double maxMs)
=> LoadThreshold.ForStep(
$"tail latency [{stepName}]: p95 <= {p95Ms}ms, p99 <= {p99Ms}ms, max <= {maxMs}ms",
Expand Down
Loading
Loading