test: improve load test quality and fix per-IP rate limiting

CLAUDE.md

@@ -27,7 +27,7 @@ TournamentAPI is a GraphQL API for running tournaments: creating them, managing
 
 Modular monolith: one ASP.NET Core deployable, one shared database, feature folders as the module boundaries.
 
-- Rate limiting - Applied to `/graphql` only: a global concurrency limiter, plus a per-IP token bucket
+- Rate limiting - One chained global limiter that runs before routing, so it covers every endpoint: a per-IP token bucket first, then a global concurrency limiter (ADR 0001)
 
 ## File naming convention
 
@@ -83,8 +83,8 @@ dotnet test TournamentAPI.IntegrationTests --filter "TournamentQueryTests"
 # Run a single test method
 dotnet test TournamentAPI.IntegrationTests --filter "FullyQualifiedName~TournamentQueryTests.GetTournaments_ReturnsAllTournamentsWithTotalCount"
 
-# Run load tests
-dotnet test TournamentAPI.LoadTests
+# Run load tests (Docker required, manual only, not part of CI)
+dotnet test TournamentAPI.LoadTests --filter "Category=Load"
 
 # Run benchmarks (must be Release)
 dotnet run --project TournamentAPI.Benchmarks --configuration Release

TournamentAPI.IntegrationTests/IntegrationTestWebAppFactory.cs

@@ -1,11 +1,16 @@
 using Hangfire;
+using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.AspNetCore.TestHost;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
+using System.Threading.RateLimiting;
 using Testcontainers.MsSql;
 using TournamentAPI.Data;
 using TournamentAPI.IntegrationTests.Infrastructure;
@@ -14,7 +19,7 @@
 
 public class IntegrationTestWebAppFactory : WebApplicationFactory<Program>, IAsyncLifetime
 {
     private readonly MsSqlContainer _dbContainer = new MsSqlBuilder()
         .WithImage("mcr.microsoft.com/azure-sql-edge:latest")
         .WithPassword("Your_password123")
         .Build();
@@ -38,6 +43,14 @@
             services.AddHangfire(config => config.UseSqlServerStorage(connectionString));
 
             services.RemoveAll<IHostedService>();
+
+            services.RemoveAll<IConfigureOptions<RateLimiterOptions>>();
+
+            services.AddRateLimiter(options =>
+            {
+                options.GlobalLimiter = PartitionedRateLimiter.Create<HttpContext, string>(_ =>
+                    RateLimitPartition.GetNoLimiter("no-limit"));
+            });
         });
     }
 

TournamentAPI.LoadTests/BaseLoadTest.cs

@@ -15,6 +15,9 @@ protected BaseLoadTest(WebApplicationFactory<Program> factory)
     protected TestClient CreateClient()
     {
         var httpClient = _factory.CreateClient();
-        return new TestClient(httpClient);
+        var testClient = new TestClient(httpClient);
+        testClient.SetApiKey(LoadTestWebAppFactory.ApiKey);
+
+        return testClient;
     }
 }

TournamentAPI.LoadTests/ConcurrencyLimiterTests.cs

@@ -1,53 +1,45 @@
 using NBomber.CSharp;
-using TournamentAPI.Shared.Models;
+using TournamentAPI.Configuration.Extensions;
 
 namespace TournamentAPI.LoadTests;
 
-public class ConcurrencyLimiterTests : BaseLoadTest, IClassFixture<LoadTestWebAppFactory>
+[Trait("Category", "Load")]
+public class ConcurrencyLimiterTests : BaseLoadTest, IClassFixture<ConcurrencyLimiterWebAppFactory>
 {
-    public ConcurrencyLimiterTests(LoadTestWebAppFactory factory) : base(factory)
+    public ConcurrencyLimiterTests(ConcurrencyLimiterWebAppFactory factory) : base(factory)
     {
     }
 
     [Fact]
-    public void ConcurrencyLimiter_Should_LimitConcurrentRequests()
+    public void ConcurrencyLimiter_Should_RejectRequests_AboveThePermitLimit()
     {
         // Arrange
-        var client = CreateClient();
-
-        var scenario = Scenario.Create("concurrency_limiter", async _ =>
-        {
-            var response = await client.ExecuteQueryAsync<TournamentsResponse>(
-                Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches
-            );
-
-            await Task.Delay(1000);
-
-            if (response.HasErrors &&
-                response.Errors?.Any(e => e.Extensions?.ContainsKey("statusCode") == true &&
-                                         (int)e.Extensions["statusCode"] == 429) == true)
-            {
-                return Response.Fail(statusCode: "429", message: "Rate limited");
-            }
-
-            return response.HasErrors ? Response.Fail() : Response.Ok();
-        })
+        using var client = CreateClient();
+
+        var scenario = Scenario.Create("concurrent_tournament_browsing", async context =>
+            await GraphQLLoadStep.RunAsync(
+                LoadTestBudgets.ListTournaments.Step,
+                context,
+                client.HttpClient,
+                Shared.QueryExamples.Queries.Tournaments.GetAllWithBracketAndMatches,
+                bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments")))
         .WithoutWarmUp()
         .WithLoadSimulations(
-            Simulation.KeepConstant(copies: 150, during: TimeSpan.FromSeconds(5))
+            Simulation.Inject(rate: 400, interval: TimeSpan.FromSeconds(1), during: TimeSpan.FromSeconds(5))
         );
 
         // Act
-        var stats = NBomberRunner.RegisterScenarios(scenario).Run();
+        var stats = LoadTestRun.Execute(
+            scenario,
+            LoadThresholds.StatusMix("200", minOkPercent: 0, allowedFailCodes: "429"));
 
         // Assert
-        Assert.True(stats.ScenarioStats[0].Ok.Request.Count >= 100);
-        Assert.True(stats.ScenarioStats[0].Fail.Request.Count > 0);
-
-        Assert.Single(stats.ScenarioStats[0].Fail.StatusCodes);
-        var rateLimitedRequests = stats.ScenarioStats[0].Fail.StatusCodes.Single();
+        Assert.True(stats.Ok.Request.Count >= RateLimiterExtensions.ConcurrencyPermitLimit);
+        Assert.True(stats.Ok.Request.Count > RateLimiterExtensions.TokenBucketLimit);
+        Assert.True(stats.Fail.Request.Count > 0);
 
+        var rateLimitedRequests = Assert.Single(stats.Fail.StatusCodes);
         Assert.Equal("429", rateLimitedRequests.StatusCode);
-        Assert.Equal(stats.ScenarioStats[0].Fail.Request.Count, rateLimitedRequests.Count);
+        Assert.Equal(stats.Fail.Request.Count, rateLimitedRequests.Count);
     }
 }

TournamentAPI.LoadTests/ConcurrencyLimiterWebAppFactory.cs

@@ -0,0 +1,23 @@
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.RateLimiting;
+using Microsoft.AspNetCore.TestHost;
+using TournamentAPI.Data;
+
+namespace TournamentAPI.LoadTests;
+
+public class ConcurrencyLimiterWebAppFactory : RateLimiterOverrideWebAppFactory
+{
+    public static readonly TimeSpan DbContextCreationDelay = TimeSpan.FromMilliseconds(300);
+
+    protected override void ConfigureWebHost(IWebHostBuilder builder)
+    {
+        base.ConfigureWebHost(builder);
+        builder.ConfigureTestServices(services =>
+            services.DelayDbContextCreation<ApplicationDbContext>(DbContextCreationDelay));
+    }
+
+    protected override void ConfigureRateLimiter(RateLimiterOptions options)
+    {
+        UseOnlyConcurrencyLimiter(options);
+    }
+}

TournamentAPI.LoadTests/DelayingDbContextFactory.cs

@@ -0,0 +1,58 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace TournamentAPI.LoadTests;
+
+internal sealed class DelayingDbContextFactory<TContext>(IDbContextFactory<TContext> inner, TimeSpan delay)
+    : IDbContextFactory<TContext> where TContext : DbContext
+{
+    public TContext CreateDbContext()
+    {
+        Thread.Sleep(delay);
+        return inner.CreateDbContext();
+    }
+
+    public async Task<TContext> CreateDbContextAsync(CancellationToken cancellationToken = default)
+    {
+        await Task.Delay(delay, cancellationToken);
+        return await inner.CreateDbContextAsync(cancellationToken);
+    }
+}
+
+internal static class DelayingDbContextFactoryExtensions
+{
+    private const string InnerFactoryKey = "delaying-db-context-factory-inner";
+
+    public static IServiceCollection DelayDbContextCreation<TContext>(
+        this IServiceCollection services,
+        TimeSpan delay) where TContext : DbContext
+    {
+        var original = services.Last(d => d.ServiceType == typeof(IDbContextFactory<TContext>));
+        services.Remove(original);
+
+        services.Add(new ServiceDescriptor(
+            typeof(IDbContextFactory<TContext>),
+            InnerFactoryKey,
+            (serviceProvider, _) => CreateOriginal(serviceProvider, original),
+            original.Lifetime));
+
+        services.Add(new ServiceDescriptor(
+            typeof(IDbContextFactory<TContext>),
+            serviceProvider => new DelayingDbContextFactory<TContext>(
+                serviceProvider.GetRequiredKeyedService<IDbContextFactory<TContext>>(InnerFactoryKey),
+                delay),
+            original.Lifetime));
+
+        return services;
+    }
+
+    private static object CreateOriginal(IServiceProvider serviceProvider, ServiceDescriptor original)
+    {
+        if (original.ImplementationFactory is not null)
+        {
+            return original.ImplementationFactory(serviceProvider);
+        }
+
+        return ActivatorUtilities.CreateInstance(serviceProvider, original.ImplementationType!);
+    }
+}

TournamentAPI.LoadTests/GraphQLLoadStep.cs

@@ -0,0 +1,105 @@
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text.Json;
+using NBomber.Contracts;
+using NBomber.CSharp;
+
+namespace TournamentAPI.LoadTests;
+
+internal static class GraphQLLoadStep
+{
+    public const string BodyCheckFailedCode = "BODY_CHECK_FAILED";
+    public const string InvalidJsonCode = "INVALID_JSON";
+    public const string GraphQLErrorCode = "GRAPHQL_ERROR";
+
+    public static Task<Response<object>> RunAsync(
+        string stepName,
+        IScenarioContext context,
+        HttpClient client,
+        string query,
+        object? variables = null,
+        Func<JsonElement, bool>? bodyCheck = null,
+        IReadOnlySet<string>? expectedRejections = null,
+        string? bearerToken = null)
+    {
+        return Step.Run(stepName, context, async () =>
+        {
+            using var request = new HttpRequestMessage(HttpMethod.Post, "/graphql")
+            {
+                Content = JsonContent.Create(new { query, variables })
+            };
+
+            if (bearerToken is not null)
+            {
+                request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken);
+            }
+
+            using var response = await client.SendAsync(request);
+            var body = await response.Content.ReadAsByteArrayAsync();
+            var (code, isSuccess) = Classify(response.IsSuccessStatusCode, (int)response.StatusCode, body, bodyCheck);
+
+            return isSuccess || expectedRejections?.Contains(code) == true
+                ? Response.Ok(statusCode: code, sizeBytes: body.Length)
+                : Response.Fail(statusCode: code, sizeBytes: body.Length);
+        });
+    }
+
+    public static Func<JsonElement, bool> NonEmptyConnection(string field)
+    {
+        return data => data.TryGetProperty(field, out var connection)
+            && connection.TryGetProperty("edges", out var edges)
+            && edges.ValueKind == JsonValueKind.Array
+            && edges.GetArrayLength() > 0;
+    }
+
+    private static (string Code, bool IsSuccess) Classify(
+        bool isSuccessStatusCode,
+        int statusCode,
+        byte[] body,
+        Func<JsonElement, bool>? bodyCheck)
+    {
+        var httpCode = statusCode.ToString();
+
+        if (!isSuccessStatusCode)
+        {
+            return (httpCode, false);
+        }
+
+        try
+        {
+            using var document = JsonDocument.Parse(body);
+            var root = document.RootElement;
+
+            if (root.TryGetProperty("errors", out var errors)
+                && errors.ValueKind == JsonValueKind.Array
+                && errors.GetArrayLength() > 0)
+            {
+                return (ReadErrorCode(errors[0]), false);
+            }
+
+            var hasData = root.TryGetProperty("data", out var data);
+            if (bodyCheck is not null && (!hasData || !bodyCheck(data)))
+            {
+                return (BodyCheckFailedCode, false);
+            }
+
+            return (httpCode, true);
+        }
+        catch (JsonException)
+        {
+            return (InvalidJsonCode, false);
+        }
+    }
+
+    private static string ReadErrorCode(JsonElement error)
+    {
+        if (error.TryGetProperty("extensions", out var extensions)
+            && extensions.TryGetProperty("code", out var code)
+            && code.ValueKind == JsonValueKind.String)
+        {
+            return code.GetString() ?? GraphQLErrorCode;
+        }
+
+        return GraphQLErrorCode;
+    }
+}