diff --git a/CLAUDE.md b/CLAUDE.md index 13cea0b..d4fec99 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ TournamentAPI is a GraphQL API for running tournaments: creating them, managing Modular monolith: one ASP.NET Core deployable, one shared database, feature folders as the module boundaries. -- Rate limiting - Applied to `/graphql` only: a global concurrency limiter, plus a per-IP token bucket +- Rate limiting - One chained global limiter that runs before routing, so it covers every endpoint: a per-IP token bucket first, then a global concurrency limiter (ADR 0001) ## File naming convention @@ -83,8 +83,8 @@ dotnet test TournamentAPI.IntegrationTests --filter "TournamentQueryTests" # Run a single test method dotnet test TournamentAPI.IntegrationTests --filter "FullyQualifiedName~TournamentQueryTests.GetTournaments_ReturnsAllTournamentsWithTotalCount" -# Run load tests -dotnet test TournamentAPI.LoadTests +# Run load tests (Docker required, manual only, not part of CI) +dotnet test TournamentAPI.LoadTests --filter "Category=Load" # Run benchmarks (must be Release) dotnet run --project TournamentAPI.Benchmarks --configuration Release diff --git a/TournamentAPI.IntegrationTests/IntegrationTestWebAppFactory.cs b/TournamentAPI.IntegrationTests/IntegrationTestWebAppFactory.cs index 9ae63ff..2871f1e 100644 --- a/TournamentAPI.IntegrationTests/IntegrationTestWebAppFactory.cs +++ b/TournamentAPI.IntegrationTests/IntegrationTestWebAppFactory.cs @@ -1,11 +1,16 @@ using Hangfire; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.RateLimiting; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using System.Threading.RateLimiting; using Testcontainers.MsSql; using TournamentAPI.Data; using TournamentAPI.IntegrationTests.Infrastructure; @@ -38,6 +43,14 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) services.AddHangfire(config => config.UseSqlServerStorage(connectionString)); services.RemoveAll(); + + services.RemoveAll>(); + + services.AddRateLimiter(options => + { + options.GlobalLimiter = PartitionedRateLimiter.Create(_ => + RateLimitPartition.GetNoLimiter("no-limit")); + }); }); } diff --git a/TournamentAPI.LoadTests/BaseLoadTest.cs b/TournamentAPI.LoadTests/BaseLoadTest.cs index 2f554b1..9afb966 100644 --- a/TournamentAPI.LoadTests/BaseLoadTest.cs +++ b/TournamentAPI.LoadTests/BaseLoadTest.cs @@ -15,6 +15,9 @@ protected BaseLoadTest(WebApplicationFactory factory) protected TestClient CreateClient() { var httpClient = _factory.CreateClient(); - return new TestClient(httpClient); + var testClient = new TestClient(httpClient); + testClient.SetApiKey(LoadTestWebAppFactory.ApiKey); + + return testClient; } } diff --git a/TournamentAPI.LoadTests/ConcurrencyLimiterTests.cs b/TournamentAPI.LoadTests/ConcurrencyLimiterTests.cs index a070b82..42c8559 100644 --- a/TournamentAPI.LoadTests/ConcurrencyLimiterTests.cs +++ b/TournamentAPI.LoadTests/ConcurrencyLimiterTests.cs @@ -1,53 +1,45 @@ using NBomber.CSharp; -using TournamentAPI.Shared.Models; +using TournamentAPI.Configuration.Extensions; namespace TournamentAPI.LoadTests; -public class ConcurrencyLimiterTests : BaseLoadTest, IClassFixture +[Trait("Category", "Load")] +public class ConcurrencyLimiterTests : BaseLoadTest, IClassFixture { - public ConcurrencyLimiterTests(LoadTestWebAppFactory factory) : base(factory) + public ConcurrencyLimiterTests(ConcurrencyLimiterWebAppFactory factory) : base(factory) { } [Fact] - public void ConcurrencyLimiter_Should_LimitConcurrentRequests() + public void ConcurrencyLimiter_Should_RejectRequests_AboveThePermitLimit() { // Arrange - var client = CreateClient(); - - var scenario = Scenario.Create("concurrency_limiter", async _ => - { - var response = await client.ExecuteQueryAsync( - Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches - ); - - await Task.Delay(1000); - - if (response.HasErrors && - response.Errors?.Any(e => e.Extensions?.ContainsKey("statusCode") == true && - (int)e.Extensions["statusCode"] == 429) == true) - { - return Response.Fail(statusCode: "429", message: "Rate limited"); - } - - return response.HasErrors ? Response.Fail() : Response.Ok(); - }) + using var client = CreateClient(); + + var scenario = Scenario.Create("concurrent_tournament_browsing", async context => + await GraphQLLoadStep.RunAsync( + LoadTestBudgets.ListTournaments.Step, + context, + client.HttpClient, + Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) .WithoutWarmUp() .WithLoadSimulations( - Simulation.KeepConstant(copies: 150, during: TimeSpan.FromSeconds(5)) + Simulation.Inject(rate: 400, interval: TimeSpan.FromSeconds(1), during: TimeSpan.FromSeconds(5)) ); // Act - var stats = NBomberRunner.RegisterScenarios(scenario).Run(); + var stats = LoadTestRun.Execute( + scenario, + LoadThresholds.StatusMix("200", minOkPercent: 0, allowedFailCodes: "429")); // Assert - Assert.True(stats.ScenarioStats[0].Ok.Request.Count >= 100); - Assert.True(stats.ScenarioStats[0].Fail.Request.Count > 0); - - Assert.Single(stats.ScenarioStats[0].Fail.StatusCodes); - var rateLimitedRequests = stats.ScenarioStats[0].Fail.StatusCodes.Single(); + Assert.True(stats.Ok.Request.Count >= RateLimiterExtensions.ConcurrencyPermitLimit); + Assert.True(stats.Ok.Request.Count > RateLimiterExtensions.TokenBucketLimit); + Assert.True(stats.Fail.Request.Count > 0); + var rateLimitedRequests = Assert.Single(stats.Fail.StatusCodes); Assert.Equal("429", rateLimitedRequests.StatusCode); - Assert.Equal(stats.ScenarioStats[0].Fail.Request.Count, rateLimitedRequests.Count); + Assert.Equal(stats.Fail.Request.Count, rateLimitedRequests.Count); } } diff --git a/TournamentAPI.LoadTests/ConcurrencyLimiterWebAppFactory.cs b/TournamentAPI.LoadTests/ConcurrencyLimiterWebAppFactory.cs new file mode 100644 index 0000000..320bda5 --- /dev/null +++ b/TournamentAPI.LoadTests/ConcurrencyLimiterWebAppFactory.cs @@ -0,0 +1,23 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.AspNetCore.TestHost; +using TournamentAPI.Data; + +namespace TournamentAPI.LoadTests; + +public class ConcurrencyLimiterWebAppFactory : RateLimiterOverrideWebAppFactory +{ + public static readonly TimeSpan DbContextCreationDelay = TimeSpan.FromMilliseconds(300); + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + base.ConfigureWebHost(builder); + builder.ConfigureTestServices(services => + services.DelayDbContextCreation(DbContextCreationDelay)); + } + + protected override void ConfigureRateLimiter(RateLimiterOptions options) + { + UseOnlyConcurrencyLimiter(options); + } +} diff --git a/TournamentAPI.LoadTests/DelayingDbContextFactory.cs b/TournamentAPI.LoadTests/DelayingDbContextFactory.cs new file mode 100644 index 0000000..f03099c --- /dev/null +++ b/TournamentAPI.LoadTests/DelayingDbContextFactory.cs @@ -0,0 +1,58 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +namespace TournamentAPI.LoadTests; + +internal sealed class DelayingDbContextFactory(IDbContextFactory inner, TimeSpan delay) + : IDbContextFactory where TContext : DbContext +{ + public TContext CreateDbContext() + { + Thread.Sleep(delay); + return inner.CreateDbContext(); + } + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + await Task.Delay(delay, cancellationToken); + return await inner.CreateDbContextAsync(cancellationToken); + } +} + +internal static class DelayingDbContextFactoryExtensions +{ + private const string InnerFactoryKey = "delaying-db-context-factory-inner"; + + public static IServiceCollection DelayDbContextCreation( + this IServiceCollection services, + TimeSpan delay) where TContext : DbContext + { + var original = services.Last(d => d.ServiceType == typeof(IDbContextFactory)); + services.Remove(original); + + services.Add(new ServiceDescriptor( + typeof(IDbContextFactory), + InnerFactoryKey, + (serviceProvider, _) => CreateOriginal(serviceProvider, original), + original.Lifetime)); + + services.Add(new ServiceDescriptor( + typeof(IDbContextFactory), + serviceProvider => new DelayingDbContextFactory( + serviceProvider.GetRequiredKeyedService>(InnerFactoryKey), + delay), + original.Lifetime)); + + return services; + } + + private static object CreateOriginal(IServiceProvider serviceProvider, ServiceDescriptor original) + { + if (original.ImplementationFactory is not null) + { + return original.ImplementationFactory(serviceProvider); + } + + return ActivatorUtilities.CreateInstance(serviceProvider, original.ImplementationType!); + } +} diff --git a/TournamentAPI.LoadTests/GraphQLLoadStep.cs b/TournamentAPI.LoadTests/GraphQLLoadStep.cs new file mode 100644 index 0000000..8b8137d --- /dev/null +++ b/TournamentAPI.LoadTests/GraphQLLoadStep.cs @@ -0,0 +1,105 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using NBomber.Contracts; +using NBomber.CSharp; + +namespace TournamentAPI.LoadTests; + +internal static class GraphQLLoadStep +{ + public const string BodyCheckFailedCode = "BODY_CHECK_FAILED"; + public const string InvalidJsonCode = "INVALID_JSON"; + public const string GraphQLErrorCode = "GRAPHQL_ERROR"; + + public static Task> RunAsync( + string stepName, + IScenarioContext context, + HttpClient client, + string query, + object? variables = null, + Func? bodyCheck = null, + IReadOnlySet? expectedRejections = null, + string? bearerToken = null) + { + return Step.Run(stepName, context, async () => + { + using var request = new HttpRequestMessage(HttpMethod.Post, "/graphql") + { + Content = JsonContent.Create(new { query, variables }) + }; + + if (bearerToken is not null) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); + } + + using var response = await client.SendAsync(request); + var body = await response.Content.ReadAsByteArrayAsync(); + var (code, isSuccess) = Classify(response.IsSuccessStatusCode, (int)response.StatusCode, body, bodyCheck); + + return isSuccess || expectedRejections?.Contains(code) == true + ? Response.Ok(statusCode: code, sizeBytes: body.Length) + : Response.Fail(statusCode: code, sizeBytes: body.Length); + }); + } + + public static Func NonEmptyConnection(string field) + { + return data => data.TryGetProperty(field, out var connection) + && connection.TryGetProperty("edges", out var edges) + && edges.ValueKind == JsonValueKind.Array + && edges.GetArrayLength() > 0; + } + + private static (string Code, bool IsSuccess) Classify( + bool isSuccessStatusCode, + int statusCode, + byte[] body, + Func? bodyCheck) + { + var httpCode = statusCode.ToString(); + + if (!isSuccessStatusCode) + { + return (httpCode, false); + } + + try + { + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + + if (root.TryGetProperty("errors", out var errors) + && errors.ValueKind == JsonValueKind.Array + && errors.GetArrayLength() > 0) + { + return (ReadErrorCode(errors[0]), false); + } + + var hasData = root.TryGetProperty("data", out var data); + if (bodyCheck is not null && (!hasData || !bodyCheck(data))) + { + return (BodyCheckFailedCode, false); + } + + return (httpCode, true); + } + catch (JsonException) + { + return (InvalidJsonCode, false); + } + } + + private static string ReadErrorCode(JsonElement error) + { + if (error.TryGetProperty("extensions", out var extensions) + && extensions.TryGetProperty("code", out var code) + && code.ValueKind == JsonValueKind.String) + { + return code.GetString() ?? GraphQLErrorCode; + } + + return GraphQLErrorCode; + } +} diff --git a/TournamentAPI.LoadTests/LatencyTests.cs b/TournamentAPI.LoadTests/LatencyTests.cs index f8091a6..e37f6bc 100644 --- a/TournamentAPI.LoadTests/LatencyTests.cs +++ b/TournamentAPI.LoadTests/LatencyTests.cs @@ -1,65 +1,126 @@ +using NBomber.Contracts; using NBomber.CSharp; -using TournamentAPI.Shared.Models; +using ListBudget = TournamentAPI.LoadTests.LoadTestBudgets.ListTournaments; namespace TournamentAPI.LoadTests; +[Trait("Category", "Load")] public class LatencyTests : BaseLoadTest, IClassFixture { + private const string SteadyBaseline = "steady_baseline"; + private const string SteadyPeak = "steady_peak"; + private const string RecoveredBaseline = "recovered_baseline"; + + private sealed record RampPhase(string Name, int RatePerSecond, TimeSpan Duration, bool IsRamp); + + private static readonly RampPhase[] RampPhases = + [ + new("ramp_to_baseline", LoadTestBudgets.Ramp.BaselineRatePerSecond, TimeSpan.FromSeconds(10), true), + new(SteadyBaseline, LoadTestBudgets.Ramp.BaselineRatePerSecond, TimeSpan.FromSeconds(15), false), + new("ramp_to_mid", LoadTestBudgets.Ramp.MidRatePerSecond, TimeSpan.FromSeconds(10), true), + new("steady_mid", LoadTestBudgets.Ramp.MidRatePerSecond, TimeSpan.FromSeconds(15), false), + new("ramp_to_peak", LoadTestBudgets.Ramp.PeakRatePerSecond, TimeSpan.FromSeconds(10), true), + new(SteadyPeak, LoadTestBudgets.Ramp.PeakRatePerSecond, TimeSpan.FromSeconds(15), false), + new("ramp_down_to_baseline", LoadTestBudgets.Ramp.BaselineRatePerSecond, TimeSpan.FromSeconds(10), true), + new(RecoveredBaseline, LoadTestBudgets.Ramp.BaselineRatePerSecond, TimeSpan.FromSeconds(15), false) + ]; + public LatencyTests(LatencyWebAppFactory factory) : base(factory) { } [Fact] - public void Api_ShouldMeetP95Latency_UnderNormalLoad() + public void Api_ShouldMeetLatencyBudget_UnderNormalLoad() { // Arrange - var client = CreateClient(); - var scenario = Scenario.Create("latency_baseline", async _ => - { - var response = await client.ExecuteQueryAsync( - Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches); - - return response.HasErrors ? Response.Fail() : Response.Ok(); - }) + using var client = CreateClient(); + var scenario = Scenario.Create("browse_tournaments", async context => + await GraphQLLoadStep.RunAsync( + ListBudget.Step, + context, + client.HttpClient, + Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) .WithWarmUpDuration(TimeSpan.FromSeconds(10)) - .WithLoadSimulations(Simulation.KeepConstant(copies: 10, during: TimeSpan.FromSeconds(30))); + .WithLoadSimulations(Simulation.Inject( + rate: LoadTestBudgets.Load.NormalRatePerSecond, + interval: TimeSpan.FromSeconds(1), + during: TimeSpan.FromSeconds(30))); - // Act - var stats = NBomberRunner - .RegisterScenarios(scenario) - .Run(); - - // Assert - var p95Latency = stats.ScenarioStats[0].Ok.Latency.Percent95; - Assert.True(p95Latency < 500); - Assert.Equal(0, stats.ScenarioStats[0].Fail.Request.Count); + // Act & Assert + LoadTestRun.Execute( + scenario, + LoadThresholds.ErrorBudget(ListBudget.MaxErrorPercent), + LoadThresholds.StatusMix("200", ListBudget.MinOkStatusPercent), + LoadThresholds.StepHealth(ListBudget.Step, ListBudget.MaxErrorPercent), + LoadThresholds.TailLatency(ListBudget.Step, ListBudget.P95Ms, ListBudget.P99Ms, ListBudget.MaxMs), + LoadThresholds.PayloadGuard(ListBudget.Step, ListBudget.MinMeanPayloadBytes, ListBudget.MaxMeanPayloadBytes)); } [Fact] - public void Api_ShouldDegradeGracefully_AsLoadIncreases() + public void Api_ShouldDegradeGracefully_AsLoadIncreases_AndRecover_AsItDrops() { // Arrange - var client = CreateClient(); - var scenario = Scenario.Create("latency_under_load", async _ => + using var client = CreateClient(); + var scenario = Scenario.Create("browse_tournaments_ramp", async context => + await GraphQLLoadStep.RunAsync( + PhaseAt(context.GetScenarioTimerTime()), + context, + client.HttpClient, + Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) + .WithoutWarmUp() + .WithLoadSimulations(RampPhases + .Select(phase => phase.IsRamp + ? Simulation.RampingInject(phase.RatePerSecond, TimeSpan.FromSeconds(1), phase.Duration) + : Simulation.Inject(phase.RatePerSecond, TimeSpan.FromSeconds(1), phase.Duration)) + .ToArray()); + + var thresholds = new List { - var response = await client.ExecuteQueryAsync( - Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches); - return response.HasErrors ? Response.Fail() : Response.Ok(); - }) - .WithWarmUpDuration(TimeSpan.FromSeconds(10)) - .WithLoadSimulations( - Simulation.RampingConstant(copies: 5, during: TimeSpan.FromSeconds(20)), - Simulation.RampingConstant(copies: 15, during: TimeSpan.FromSeconds(20)), - Simulation.RampingConstant(copies: 20, during: TimeSpan.FromSeconds(20)) - ); + LoadThresholds.ErrorBudget(ListBudget.MaxErrorPercent), + LoadThresholds.StatusMix("200", ListBudget.MinOkStatusPercent) + }; + + foreach (var phase in RampPhases.Where(p => !p.IsRamp)) + { + thresholds.Add(LoadThresholds.StepHealth(phase.Name, ListBudget.MaxErrorPercent)); + thresholds.Add(LoadThresholds.TailLatency(phase.Name, LoadTestBudgets.Ramp.P95PerLevelMs, LoadTestBudgets.Ramp.P99Ms, LoadTestBudgets.Ramp.MaxMs)); + thresholds.Add(LoadThresholds.PayloadGuard(phase.Name, ListBudget.MinMeanPayloadBytes, ListBudget.MaxMeanPayloadBytes)); + } // Act - var stats = NBomberRunner - .RegisterScenarios(scenario) - .Run(); + var stats = LoadTestRun.Execute(scenario, thresholds.ToArray()); // Assert - Assert.Equal(0, stats.ScenarioStats[0].Fail.Request.Count); - Assert.True(stats.ScenarioStats[0].Ok.Latency.Percent95 < 800); + var baselineP95 = P95(stats, SteadyBaseline); + var peakP95 = P95(stats, SteadyPeak); + var recoveredP95 = P95(stats, RecoveredBaseline); + + Assert.True( + peakP95 <= baselineP95 * LoadTestBudgets.Ramp.MaxTopToBottomP95Ratio, + $"p95 at {LoadTestBudgets.Ramp.PeakRatePerSecond} req/s ({peakP95}ms) exceeds {LoadTestBudgets.Ramp.MaxTopToBottomP95Ratio}x the p95 at {LoadTestBudgets.Ramp.BaselineRatePerSecond} req/s ({baselineP95}ms)"); + Assert.True( + recoveredP95 <= baselineP95 * LoadTestBudgets.Ramp.MaxRecoveryP95Ratio, + $"p95 after dropping back to {LoadTestBudgets.Ramp.BaselineRatePerSecond} req/s ({recoveredP95}ms) exceeds {LoadTestBudgets.Ramp.MaxRecoveryP95Ratio}x the initial p95 ({baselineP95}ms)"); + } + + private static string PhaseAt(TimeSpan elapsed) + { + var phaseEnd = TimeSpan.Zero; + + foreach (var phase in RampPhases) + { + phaseEnd += phase.Duration; + if (elapsed < phaseEnd) + { + return phase.Name; + } + } + + return RampPhases[^1].Name; } + + private static double P95(NBomber.Contracts.Stats.ScenarioStats stats, string stepName) + => stats.StepStats.Get(stepName).Ok.Latency.Percent95; } diff --git a/TournamentAPI.LoadTests/LatencyWebAppFactory.cs b/TournamentAPI.LoadTests/LatencyWebAppFactory.cs index c8d8755..c3e679e 100644 --- a/TournamentAPI.LoadTests/LatencyWebAppFactory.cs +++ b/TournamentAPI.LoadTests/LatencyWebAppFactory.cs @@ -1,31 +1,11 @@ -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.RateLimiting; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using System.Threading.RateLimiting; namespace TournamentAPI.LoadTests; -public class LatencyWebAppFactory : LoadTestWebAppFactory +public class LatencyWebAppFactory : RateLimiterOverrideWebAppFactory { - protected override void ConfigureWebHost(IWebHostBuilder builder) + protected override void ConfigureRateLimiter(RateLimiterOptions options) { - base.ConfigureWebHost(builder); - builder.ConfigureTestServices(services => - { - services.RemoveAll>(); - - services.AddRateLimiter(options => - { - options.GlobalLimiter = PartitionedRateLimiter.Create(_ => - RateLimitPartition.GetNoLimiter("no-limit")); - - options.AddPolicy("IpBasedTokenBucket", _ => - RateLimitPartition.GetNoLimiter("no-limit")); - }); - }); + DisableLimiters(options); } } diff --git a/TournamentAPI.LoadTests/LoadTestAssemblyAttributes.cs b/TournamentAPI.LoadTests/LoadTestAssemblyAttributes.cs new file mode 100644 index 0000000..2171200 --- /dev/null +++ b/TournamentAPI.LoadTests/LoadTestAssemblyAttributes.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/TournamentAPI.LoadTests/LoadTestBudgets.cs b/TournamentAPI.LoadTests/LoadTestBudgets.cs new file mode 100644 index 0000000..6fdde74 --- /dev/null +++ b/TournamentAPI.LoadTests/LoadTestBudgets.cs @@ -0,0 +1,38 @@ +namespace TournamentAPI.LoadTests; + +internal static class LoadTestBudgets +{ + public static class ListTournaments + { + public const string Step = "list_tournaments"; + + public const double MaxErrorPercent = 0; + public const double MinOkStatusPercent = 100; + + public const double P95Ms = 150; + public const double P99Ms = 300; + public const double MaxMs = 1000; + + public const double MinMeanPayloadBytes = 2_000; + public const double MaxMeanPayloadBytes = 5_000; + } + + public static class Load + { + public const int NormalRatePerSecond = 50; + } + + public static class Ramp + { + public const int BaselineRatePerSecond = 25; + public const int MidRatePerSecond = 75; + public const int PeakRatePerSecond = 100; + + public const double P95PerLevelMs = 800; + public const double P99Ms = 1_000; + public const double MaxMs = 3_000; + + public const double MaxTopToBottomP95Ratio = 6; + public const double MaxRecoveryP95Ratio = 2; + } +} diff --git a/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs b/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs index 8be9409..ac2f858 100644 --- a/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs +++ b/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs @@ -1,9 +1,12 @@ +using Hangfire; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; using Testcontainers.MsSql; using TournamentAPI.Data; @@ -11,6 +14,8 @@ namespace TournamentAPI.LoadTests; public class LoadTestWebAppFactory : WebApplicationFactory, IAsyncLifetime { + public const string ApiKey = "pk_load_test_9f3a7c1e5b2d4086"; + private readonly MsSqlContainer _dbContainer = new MsSqlBuilder() .WithImage("mcr.microsoft.com/azure-sql-edge:latest") .WithPassword("Your_password123") @@ -18,15 +23,23 @@ public class LoadTestWebAppFactory : WebApplicationFactory, IAsyncLifet protected override void ConfigureWebHost(IWebHostBuilder builder) { + builder.ConfigureAppConfiguration((_, configuration) => + configuration.AddInMemoryCollection(new Dictionary + { + ["ApiKey:Value"] = ApiKey + })); + builder.ConfigureTestServices(services => { services.RemoveAll(typeof(IDbContextFactory)); - services.AddDbContextFactory(options => - { - var connectionString = _dbContainer.GetConnectionString() + ";Initial Catalog=TournamentLoadTestDb"; - options.UseSqlServer(connectionString); - }); + var connectionString = _dbContainer.GetConnectionString() + ";Initial Catalog=TournamentLoadTestDb"; + + services.AddDbContextFactory(options => options.UseSqlServer(connectionString)); + + services.AddHangfire(config => config.UseSqlServerStorage(connectionString)); + + services.RemoveAll(); }); } diff --git a/TournamentAPI.LoadTests/LoadThresholds.cs b/TournamentAPI.LoadTests/LoadThresholds.cs new file mode 100644 index 0000000..c69b52a --- /dev/null +++ b/TournamentAPI.LoadTests/LoadThresholds.cs @@ -0,0 +1,132 @@ +using System.Linq.Expressions; +using NBomber.Contracts; +using NBomber.Contracts.Stats; +using NBomber.CSharp; +using Xunit; + +namespace TournamentAPI.LoadTests; + +internal sealed class LoadThreshold +{ + private readonly string? _stepName; + private readonly Expression>? _scenarioCheck; + private readonly Expression>? _stepCheck; + private readonly Func? _compiledScenarioCheck; + private readonly Func? _compiledStepCheck; + + private LoadThreshold( + string name, + string? stepName, + Expression>? scenarioCheck, + Expression>? stepCheck) + { + Name = name; + _stepName = stepName; + _scenarioCheck = scenarioCheck; + _stepCheck = stepCheck; + _compiledScenarioCheck = scenarioCheck?.Compile(); + _compiledStepCheck = stepCheck?.Compile(); + } + + public string Name { get; } + + public static LoadThreshold ForScenario(string name, Expression> check) + => new(name, null, check, null); + + public static LoadThreshold ForStep(string name, string stepName, Expression> check) + => new(name, stepName, null, check); + + public bool IsSatisfiedBy(ScenarioStats stats) + { + return _stepName is null + ? _compiledScenarioCheck!(stats) + : _compiledStepCheck!(stats.StepStats.Get(_stepName)); + } + + public Threshold ToNBomberThreshold() + { + return _stepName is null + ? Threshold.Create(_scenarioCheck!) + : Threshold.Create(_stepName, _stepCheck!); + } +} + +internal static class LoadThresholds +{ + public static LoadThreshold ErrorBudget(double maxFailPercent) + => LoadThreshold.ForScenario( + $"error budget: fail rate <= {maxFailPercent}%", + s => s.Fail.Request.Percent <= maxFailPercent); + + public static LoadThreshold TailLatency(string stepName, double p95Ms, double p99Ms, double maxMs) + => LoadThreshold.ForStep( + $"tail latency [{stepName}]: p95 <= {p95Ms}ms, p99 <= {p99Ms}ms, max <= {maxMs}ms", + stepName, + s => s.Ok.Latency.Percent95 <= p95Ms + && s.Ok.Latency.Percent99 <= p99Ms + && s.Ok.Latency.MaxMs <= maxMs); + + public static LoadThreshold StatusMix(string okCode, double minOkPercent, params string[] allowedFailCodes) + => LoadThreshold.ForScenario( + $"status mix: {okCode} >= {minOkPercent}% of requests, failures only in [{string.Join(", ", allowedFailCodes)}]", + s => IsStatusMixSatisfied(s, okCode, minOkPercent, allowedFailCodes)); + + public static LoadThreshold PayloadGuard(string stepName, double minMeanBytes, double maxMeanBytes) + => LoadThreshold.ForStep( + $"payload guard [{stepName}]: mean response between {minMeanBytes} and {maxMeanBytes} bytes", + stepName, + s => s.Ok.DataTransfer.MeanBytes >= minMeanBytes && s.Ok.DataTransfer.MeanBytes <= maxMeanBytes); + + public static LoadThreshold StepHealth(string stepName, double maxFailPercent) + => LoadThreshold.ForStep( + $"step health [{stepName}]: fail rate <= {maxFailPercent}%", + stepName, + s => s.Fail.Request.Percent <= maxFailPercent); + + public static bool IsStatusMixSatisfied( + ScenarioStats stats, + string okCode, + double minOkPercent, + string[] allowedFailCodes) + { + var total = stats.Ok.Request.Count + stats.Fail.Request.Count; + if (total == 0) + { + return false; + } + + var okCount = stats.Ok.StatusCodes.Find(okCode)?.Count ?? 0; + var hasUnexpectedFailure = stats.Fail.StatusCodes.Any(c => !allowedFailCodes.Contains(c.StatusCode)); + + return okCount * 100.0 / total >= minOkPercent && !hasUnexpectedFailure; + } +} + +internal static class LoadTestRun +{ + public static ScenarioStats Execute(ScenarioProps scenario, params LoadThreshold[] thresholds) + { + var withThresholds = thresholds.Length == 0 + ? scenario + : scenario.WithThresholds(thresholds.Select(t => t.ToNBomberThreshold()).ToArray()); + + var result = NBomberRunner + .RegisterScenarios(withThresholds) + .Run(); + + var stats = result.ScenarioStats.Get(scenario.ScenarioName); + AssertPassed(stats, thresholds); + + return stats; + } + + public static void AssertPassed(ScenarioStats stats, IEnumerable thresholds) + { + var failed = thresholds.Where(t => !t.IsSatisfiedBy(stats)).Select(t => t.Name).ToList(); + + if (failed.Count > 0) + { + Assert.Fail($"Load thresholds failed for '{stats.ScenarioName}':{Environment.NewLine}- {string.Join($"{Environment.NewLine}- ", failed)}"); + } + } +} diff --git a/TournamentAPI.LoadTests/RateLimiterOverrideWebAppFactory.cs b/TournamentAPI.LoadTests/RateLimiterOverrideWebAppFactory.cs new file mode 100644 index 0000000..10ba635 --- /dev/null +++ b/TournamentAPI.LoadTests/RateLimiterOverrideWebAppFactory.cs @@ -0,0 +1,48 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using System.Threading.RateLimiting; +using TournamentAPI.Configuration.Extensions; + +namespace TournamentAPI.LoadTests; + +public abstract class RateLimiterOverrideWebAppFactory : LoadTestWebAppFactory +{ + protected abstract void ConfigureRateLimiter(RateLimiterOptions options); + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + base.ConfigureWebHost(builder); + builder.ConfigureTestServices(services => + { + services.RemoveAll>(); + + services.AddRateLimiter(options => + { + options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + ConfigureRateLimiter(options); + }); + }); + } + + protected static void DisableLimiters(RateLimiterOptions options) + { + options.GlobalLimiter = PartitionedRateLimiter.Create(_ => + RateLimitPartition.GetNoLimiter("no-limit")); + } + + protected static void UseOnlyConcurrencyLimiter(RateLimiterOptions options) + { + options.GlobalLimiter = RateLimiterExtensions.CreateConcurrencyLimiter(); + } + + protected static void UseOnlyIpTokenBucketLimiter(RateLimiterOptions options) + { + options.GlobalLimiter = RateLimiterExtensions.CreateIpTokenBucketLimiter(); + } +} diff --git a/TournamentAPI.LoadTests/TokenBucketTests.cs b/TournamentAPI.LoadTests/TokenBucketTests.cs index c9cf434..1fbfd76 100644 --- a/TournamentAPI.LoadTests/TokenBucketTests.cs +++ b/TournamentAPI.LoadTests/TokenBucketTests.cs @@ -1,11 +1,12 @@ using NBomber.CSharp; -using TournamentAPI.Shared.Models; +using TournamentAPI.Configuration.Extensions; namespace TournamentAPI.LoadTests; -public class TokenBucketTests : BaseLoadTest, IClassFixture +[Trait("Category", "Load")] +public class TokenBucketTests : BaseLoadTest, IClassFixture { - public TokenBucketTests(LoadTestWebAppFactory factory) : base(factory) + public TokenBucketTests(TokenBucketWebAppFactory factory) : base(factory) { } @@ -13,40 +14,34 @@ public TokenBucketTests(LoadTestWebAppFactory factory) : base(factory) public void TokenBucket_Should_AllowBurst_Then_Reject() { // Arrange - var client = CreateClient(); - - var scenario = Scenario.Create("token_bucket_burst", async _ => - { - var response = await client.ExecuteQueryAsync( - Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches); - - if (response.HasErrors && - response.Errors?.Any(e => e.Extensions?.ContainsKey("statusCode") == true && - (int)e.Extensions["statusCode"] == 429) == true) - { - return Response.Fail(statusCode: "429", message: "Rate limited"); - } - - return response.HasErrors ? Response.Fail() : Response.Ok(); - }) + using var client = CreateClient(); + + var scenario = Scenario.Create("tournament_browsing_burst", async context => + await GraphQLLoadStep.RunAsync( + LoadTestBudgets.ListTournaments.Step, + context, + client.HttpClient, + Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) .WithoutWarmUp() .WithLoadSimulations( Simulation.Inject(rate: 300, interval: TimeSpan.FromSeconds(1), during: TimeSpan.FromSeconds(2)) ); // Act - var stats = NBomberRunner - .RegisterScenarios(scenario) - .Run(); + var stats = LoadTestRun.Execute( + scenario, + LoadThresholds.StatusMix("200", minOkPercent: 0, allowedFailCodes: "429")); // Assert - Assert.InRange(stats.ScenarioStats[0].Ok.Request.Count, 90, 130); - Assert.True(stats.ScenarioStats[0].Fail.Request.Count > 0); - - Assert.Single(stats.ScenarioStats[0].Fail.StatusCodes); - var rateLimitedRequests = stats.ScenarioStats[0].Fail.StatusCodes.Single(); + Assert.InRange( + stats.Ok.Request.Count, + RateLimiterExtensions.TokenBucketLimit - 10, + RateLimiterExtensions.TokenBucketLimit + 30); + Assert.True(stats.Fail.Request.Count > 0); + var rateLimitedRequests = Assert.Single(stats.Fail.StatusCodes); Assert.Equal("429", rateLimitedRequests.StatusCode); - Assert.Equal(stats.ScenarioStats[0].Fail.Request.Count, rateLimitedRequests.Count); + Assert.Equal(stats.Fail.Request.Count, rateLimitedRequests.Count); } } diff --git a/TournamentAPI.LoadTests/TokenBucketWebAppFactory.cs b/TournamentAPI.LoadTests/TokenBucketWebAppFactory.cs new file mode 100644 index 0000000..744c321 --- /dev/null +++ b/TournamentAPI.LoadTests/TokenBucketWebAppFactory.cs @@ -0,0 +1,11 @@ +using Microsoft.AspNetCore.RateLimiting; + +namespace TournamentAPI.LoadTests; + +public class TokenBucketWebAppFactory : RateLimiterOverrideWebAppFactory +{ + protected override void ConfigureRateLimiter(RateLimiterOptions options) + { + UseOnlyIpTokenBucketLimiter(options); + } +} diff --git a/TournamentAPI/Configuration/Extensions/RateLimiterExtensions.cs b/TournamentAPI/Configuration/Extensions/RateLimiterExtensions.cs index 537d9e0..d9e05e6 100644 --- a/TournamentAPI/Configuration/Extensions/RateLimiterExtensions.cs +++ b/TournamentAPI/Configuration/Extensions/RateLimiterExtensions.cs @@ -4,6 +4,11 @@ namespace TournamentAPI.Configuration.Extensions; internal static class RateLimiterExtensions { + internal const int ConcurrencyPermitLimit = 100; + internal const int TokenBucketLimit = 100; + internal const int TokenBucketTokensPerPeriod = 50; + internal static readonly TimeSpan TokenBucketReplenishmentPeriod = TimeSpan.FromMinutes(1); + internal static IServiceCollection AddApplicationRateLimiting(this IServiceCollection services) { services.AddRateLimiter(options => @@ -25,36 +30,45 @@ internal static IServiceCollection AddApplicationRateLimiting(this IServiceColle return ValueTask.CompletedTask; }; options.GlobalLimiter = PartitionedRateLimiter.CreateChained( - PartitionedRateLimiter.Create(_ => - RateLimitPartition.GetConcurrencyLimiter( - "GlobalConcurrencyLimiter", - _ => new ConcurrencyLimiterOptions - { - PermitLimit = 100, - QueueProcessingOrder = QueueProcessingOrder.OldestFirst, - QueueLimit = 0 - } - ) - ) - ); - options.AddPolicy("IpBasedTokenBucket", httpContext => - { - var clientIp = httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"; - - return RateLimitPartition.GetTokenBucketLimiter( - clientIp, - _ => new TokenBucketRateLimiterOptions - { - TokenLimit = 100, - TokensPerPeriod = 50, - ReplenishmentPeriod = TimeSpan.FromMinutes(1), - QueueProcessingOrder = QueueProcessingOrder.OldestFirst, - QueueLimit = 0 - } - ); - }); + CreateIpTokenBucketLimiter(), + CreateConcurrencyLimiter()); }); return services; } + + internal static PartitionedRateLimiter CreateIpTokenBucketLimiter() + { + return PartitionedRateLimiter.Create(httpContext => + { + var clientIp = httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + + return RateLimitPartition.GetTokenBucketLimiter( + clientIp, + _ => new TokenBucketRateLimiterOptions + { + TokenLimit = TokenBucketLimit, + TokensPerPeriod = TokenBucketTokensPerPeriod, + ReplenishmentPeriod = TokenBucketReplenishmentPeriod, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + QueueLimit = 0 + } + ); + }); + } + + internal static PartitionedRateLimiter CreateConcurrencyLimiter() + { + return PartitionedRateLimiter.Create(_ => + RateLimitPartition.GetConcurrencyLimiter( + "GlobalConcurrencyLimiter", + _ => new ConcurrencyLimiterOptions + { + PermitLimit = ConcurrencyPermitLimit, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + QueueLimit = 0 + } + ) + ); + } } diff --git a/TournamentAPI/Program.cs b/TournamentAPI/Program.cs index 3536989..ef6eaac 100644 --- a/TournamentAPI/Program.cs +++ b/TournamentAPI/Program.cs @@ -105,15 +105,13 @@ { ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse }) -.RequireAuthorization("HealthCheckPolicy") -.RequireRateLimiting("IpBasedTokenBucket"); +.RequireAuthorization("HealthCheckPolicy"); app.MapGraphQL() .WithOptions(options => { options.Tool.Enable = app.Environment.IsDevelopment(); - }) - .RequireRateLimiting("IpBasedTokenBucket"); + }); app.Run(); diff --git a/TournamentAPI/TournamentAPI.csproj b/TournamentAPI/TournamentAPI.csproj index 57614b6..eb3c864 100644 --- a/TournamentAPI/TournamentAPI.csproj +++ b/TournamentAPI/TournamentAPI.csproj @@ -9,6 +9,7 @@ + diff --git a/docs/adr/0008_load_test_execution_model.md b/docs/adr/0008_load_test_execution_model.md new file mode 100644 index 0000000..bf09768 --- /dev/null +++ b/docs/adr/0008_load_test_execution_model.md @@ -0,0 +1,55 @@ +# Title +Load Test Execution Model + +# Date +29/09/2026 + +## Status +Accepted + +## Context +The load tests in `TournamentAPI.LoadTests` had four problems. They ran in parallel, so p95 assertions competed for CPU. They used a closed model (`KeepConstant`), where a slow server lowers the request rate and hides its own latency. Their limits were guesses, and no threshold was tied to a measured baseline. One test also exercised the wrong limiter. We needed to settle where the API runs during a load test, how load is generated, how tests are scheduled, and where the limits come from. + +## Considered Options + +### For where the API runs +1. In-process `WebApplicationFactory` with a Testcontainers SQL Server (chosen) + - Pros: Same fixture style as the integration tests. No port or process management. Services can be overridden per test class (rate limiters, a delayed `IDbContextFactory` to hold concurrency permits, Hangfire removed). + - Cons: The load generator and the server share one process, one CPU and one thread pool. There is no network or Kestrel in the path, so absolute numbers are optimistic and the generator can starve the server. +2. Real Kestrel host in a separate process + - Pros: Closer to production. Generator and server no longer compete. + - Cons: Port and process management, and the per-test service overrides above become much harder. + +### For how load is generated +1. Open model with a fixed arrival rate, `Inject` and `RampingInject` (chosen) + - Pros: The request rate does not depend on server speed, so a slowdown shows up as latency and queueing instead of fewer requests. This matches independent API clients. + - Cons: Rates must be calibrated to capacity. In-flight requests are unbounded when the server is overloaded, and the in-process generator can fall behind the target rate. +2. Closed model with a fixed number of virtual users, `KeepConstant` and `RampingConstant` + - Pros: Bounded concurrency, simple to reason about. + - Cons: Slowdowns lower the request rate and flatter latency. + +### For how tests are scheduled +1. Serialize all load tests (chosen) + - Pros: One scenario at a time, so latency assertions are not distorted by another scenario's CPU use. + - Cons: Longer total run time, and each test class starts its own SQL container. +2. Leave xUnit's default parallelism + - Pros: Shorter wall-clock time. + - Cons: Scenarios compete for CPU, which makes the p95 assertions flaky. + +### For where the limits come from +1. Budgets measured from a baseline run, with headroom, kept in `LoadTestBudgets` (chosen) + - Pros: Limits reflect real behaviour. Every threshold type reads from one file. + - Cons: Numbers depend on the machine that took the baseline and need recalibrating when hardware or the query shape changes. +2. Limits from assumed targets (for example a fixed p95 of 500 ms) + - Pros: No baseline run needed. + - Cons: Either too loose to catch regressions or too tight to pass reliably. + +## Decision +The API runs in-process, and load is generated as an open model with a fixed arrival rate. Tests run serialized, and every limit comes from a measured baseline in `LoadTestBudgets`. + +The baseline was measured on a developer machine on 29/09/2026. At 50 req/s the list query had p95 55 ms, p99 104 ms, a maximum of 225 ms and a payload of 3,026 bytes. In the ramp, 100 req/s gave p95 325 ms, p99 488 ms and a maximum of 689 ms, and the run recovered to p95 34 ms after ramping back down. Budgets are set with headroom above those figures: for the list query, p95 150 ms, p99 300 ms and a maximum of 1,000 ms. Ramp windows have their own limits, because peak load is expected to run slower than steady load. + +Numbers from the in-process host are comparable between runs on the same machine. They are not production latency figures. + +## Related ADRs +- ADR #0001 Rate Limiting Algorithm for API Requests: the concurrency limiter and token bucket tests exercise the chained global limiter that ADR decided on. Running them showed the per-IP limiter was attached as an endpoint policy before routing and never took effect, so it was moved into the global limiter.