diff --git a/TournamentAPI.LoadTests/AuthenticationLoadTests.cs b/TournamentAPI.LoadTests/AuthenticationLoadTests.cs new file mode 100644 index 0000000..672102d --- /dev/null +++ b/TournamentAPI.LoadTests/AuthenticationLoadTests.cs @@ -0,0 +1,148 @@ +using System.Text.Json; +using NBomber.CSharp; +using UserMutations = TournamentAPI.Shared.MutationExamples.Mutations.Users; +using UserQueries = TournamentAPI.Shared.QueryExamples.Queries.Users; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Load")] +public class AuthenticationLoadTests : BaseLoadTest, IClassFixture +{ + private const string RefreshTokenCookieName = "refreshToken="; + + private readonly MediumDatasetWebAppFactory _factory; + + public AuthenticationLoadTests(MediumDatasetWebAppFactory factory) : base(factory) + { + _factory = factory; + } + + [Fact] + public void AuthenticationFlow_ShouldMeetBudgets_UnderSteadyLogins() + { + // Arrange + using var client = CreateClient(); + var http = client.HttpClient; + var emails = _factory.UserEmails; + var password = _factory.UserPassword; + + var scenario = Scenario.Create("authenticate_and_refresh_session", async context => + { + var email = emails[(int)(context.InvocationNumber % emails.Count)]; + string? accessToken = null; + string? refreshCookie = null; + + var login = await GraphQLLoadStep.RunAsync( + StepNames.Login, + context, + http, + UserMutations.LoginUser, + new { input = new { email, password } }, + HasStringResult("loginUser"), + onResponse: (response, body) => + { + accessToken = ReadStringResult(body, "loginUser"); + refreshCookie = ReadRefreshTokenCookie(response); + }); + + if (login.IsError) + { + return login; + } + + var profile = await GraphQLLoadStep.RunAsync( + StepNames.ViewProfile, + context, + http, + UserQueries.GetMe, + bodyCheck: GraphQLLoadStep.NonNullField("me"), + bearerToken: accessToken); + + if (profile.IsError) + { + return profile; + } + + var refresh = await GraphQLLoadStep.RunAsync( + StepNames.RefreshToken, + context, + http, + UserMutations.RefreshToken, + bodyCheck: HasStringResult("refreshToken"), + refreshTokenCookie: refreshCookie, + onResponse: (response, body) => + { + accessToken = ReadStringResult(body, "refreshToken"); + refreshCookie = ReadRefreshTokenCookie(response); + }); + + if (refresh.IsError) + { + return refresh; + } + + return await GraphQLLoadStep.RunAsync( + StepNames.Logout, + context, + http, + UserMutations.LogoutUser, + bodyCheck: HasTrueResult("logoutUser"), + bearerToken: accessToken, + refreshTokenCookie: refreshCookie); + }) + .WithWarmUpDuration(TimeSpan.FromSeconds(10)) + .WithLoadSimulations(Simulation.Inject( + rate: LoadTestBudgets.Rates.AuthFlowPerSecond, + interval: TimeSpan.FromSeconds(1), + during: TimeSpan.FromSeconds(30))); + + // Act & Assert + LoadTestRun.Execute( + scenario, + LoadThresholds.ForSteps( + LoadTestDataSize.Medium, + "200", + StepNames.Login, + StepNames.ViewProfile, + StepNames.RefreshToken, + StepNames.Logout)); + } + + private static Func HasStringResult(string mutation) + { + return data => data.TryGetProperty(mutation, out var result) + && result.TryGetProperty("string", out var value) + && value.ValueKind == JsonValueKind.String; + } + + private static Func HasTrueResult(string mutation) + { + return data => data.TryGetProperty(mutation, out var result) + && result.TryGetProperty("boolean", out var value) + && value.ValueKind == JsonValueKind.True; + } + + private static string? ReadStringResult(byte[] body, string mutation) + { + using var document = JsonDocument.Parse(body); + + return document.RootElement.TryGetProperty("data", out var data) + && data.TryGetProperty(mutation, out var result) + && result.TryGetProperty("string", out var value) + && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + } + + private static string? ReadRefreshTokenCookie(HttpResponseMessage response) + { + if (!response.Headers.TryGetValues("Set-Cookie", out var cookies)) + { + return null; + } + + var cookie = cookies.FirstOrDefault(c => c.StartsWith(RefreshTokenCookieName, StringComparison.Ordinal)); + + return cookie?.Split(';')[0][RefreshTokenCookieName.Length..]; + } +} diff --git a/TournamentAPI.LoadTests/BaseLoadTest.cs b/TournamentAPI.LoadTests/BaseLoadTest.cs index 9afb966..912230e 100644 --- a/TournamentAPI.LoadTests/BaseLoadTest.cs +++ b/TournamentAPI.LoadTests/BaseLoadTest.cs @@ -14,7 +14,7 @@ protected BaseLoadTest(WebApplicationFactory factory) protected TestClient CreateClient() { - var httpClient = _factory.CreateClient(); + var httpClient = _factory.CreateClient(new WebApplicationFactoryClientOptions { HandleCookies = false }); var testClient = new TestClient(httpClient); testClient.SetApiKey(LoadTestWebAppFactory.ApiKey); diff --git a/TournamentAPI.LoadTests/DatasetWebAppFactory.cs b/TournamentAPI.LoadTests/DatasetWebAppFactory.cs new file mode 100644 index 0000000..02ad283 --- /dev/null +++ b/TournamentAPI.LoadTests/DatasetWebAppFactory.cs @@ -0,0 +1,27 @@ +using Microsoft.AspNetCore.Identity; +using TournamentAPI.Data; +using TournamentAPI.Data.Models; + +namespace TournamentAPI.LoadTests; + +public abstract class DatasetWebAppFactory : LatencyWebAppFactory +{ + protected abstract LoadTestDataSize Size { get; } + + public override string UserPassword => LoadTestDataSeeder.UserPassword; + + public override string TournamentSearchTerm => LoadTestDataSeeder.TournamentNamePrefix; + + protected override Task SeedAsync(ApplicationDbContext context, UserManager userManager) + => LoadTestDataSeeder.SeedAsync(context, userManager, Size); +} + +public class MediumDatasetWebAppFactory : DatasetWebAppFactory +{ + protected override LoadTestDataSize Size => LoadTestDataSize.Medium; +} + +public class LargeDatasetWebAppFactory : DatasetWebAppFactory +{ + protected override LoadTestDataSize Size => LoadTestDataSize.Large; +} diff --git a/TournamentAPI.LoadTests/GraphQLLoadStep.cs b/TournamentAPI.LoadTests/GraphQLLoadStep.cs index 8b8137d..146b0fc 100644 --- a/TournamentAPI.LoadTests/GraphQLLoadStep.cs +++ b/TournamentAPI.LoadTests/GraphQLLoadStep.cs @@ -20,7 +20,10 @@ public static Task> RunAsync( object? variables = null, Func? bodyCheck = null, IReadOnlySet? expectedRejections = null, - string? bearerToken = null) + string? bearerToken = null, + string? refreshTokenCookie = null, + Action? onResponse = null, + Func? classifyRejection = null) { return Step.Run(stepName, context, async () => { @@ -34,9 +37,15 @@ public static Task> RunAsync( request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); } + if (refreshTokenCookie is not null) + { + request.Headers.Add("Cookie", $"refreshToken={refreshTokenCookie}"); + } + using var response = await client.SendAsync(request); var body = await response.Content.ReadAsByteArrayAsync(); - var (code, isSuccess) = Classify(response.IsSuccessStatusCode, (int)response.StatusCode, body, bodyCheck); + onResponse?.Invoke(response, body); + var (code, isSuccess) = Classify(response.IsSuccessStatusCode, (int)response.StatusCode, body, bodyCheck, classifyRejection); return isSuccess || expectedRejections?.Contains(code) == true ? Response.Ok(statusCode: code, sizeBytes: body.Length) @@ -52,27 +61,48 @@ public static Func NonEmptyConnection(string field) && edges.GetArrayLength() > 0; } + public static Func NonNullField(string field) + { + return data => data.TryGetProperty(field, out var value) && value.ValueKind == JsonValueKind.Object; + } + + public static Func RejectionWhenErrorHasExtension(string extensionName, string statusCode) + { + return error => error.TryGetProperty("extensions", out var extensions) + && extensions.TryGetProperty(extensionName, out _) + ? statusCode + : null; + } + private static (string Code, bool IsSuccess) Classify( bool isSuccessStatusCode, int statusCode, byte[] body, - Func? bodyCheck) + Func? bodyCheck, + Func? classifyRejection) { var httpCode = statusCode.ToString(); - if (!isSuccessStatusCode) - { - return (httpCode, false); - } - try { using var document = JsonDocument.Parse(body); var root = document.RootElement; - if (root.TryGetProperty("errors", out var errors) + var hasErrors = root.TryGetProperty("errors", out var errors) && errors.ValueKind == JsonValueKind.Array - && errors.GetArrayLength() > 0) + && errors.GetArrayLength() > 0; + + if (hasErrors && classifyRejection?.Invoke(errors[0]) is { } rejection) + { + return (rejection, true); + } + + if (!isSuccessStatusCode) + { + return (httpCode, false); + } + + if (hasErrors) { return (ReadErrorCode(errors[0]), false); } @@ -87,7 +117,7 @@ private static (string Code, bool IsSuccess) Classify( } catch (JsonException) { - return (InvalidJsonCode, false); + return isSuccessStatusCode ? (InvalidJsonCode, false) : (httpCode, false); } } diff --git a/TournamentAPI.LoadTests/LoadTestBudgets.cs b/TournamentAPI.LoadTests/LoadTestBudgets.cs index 6fdde74..0184795 100644 --- a/TournamentAPI.LoadTests/LoadTestBudgets.cs +++ b/TournamentAPI.LoadTests/LoadTestBudgets.cs @@ -1,7 +1,111 @@ namespace TournamentAPI.LoadTests; +internal sealed record StepBudget( + string Step, + double P95Ms, + double P99Ms, + double MaxMs, + double? MinMeanBytes = null, + double? MaxMeanBytes = null); + +internal sealed record MeasuredBaseline(double P95Ms, double P99Ms, double MaxMs, double MeanKilobytes); + +internal static class StepNames +{ + public const string ListTournaments = "list_tournaments"; + public const string ViewTournament = "view_tournament"; + public const string SearchTournamentsByName = "search_tournaments_by_name"; + public const string SortTournaments = "sort_tournaments"; + public const string ViewOwnerHistory = "view_owner_history"; + public const string ViewParticipantsWonTournaments = "view_participants_won_tournaments"; + public const string ViewParticipantsPlayedTournaments = "view_participants_played_tournaments"; + public const string ViewParticipantsWonMatches = "view_participants_won_matches"; + public const string ViewBracketStandings = "view_bracket_standings"; + public const string ViewOwnTournamentHistory = "view_own_tournament_history"; + public const string ListTournamentsWithOversizedPage = "list_tournaments_with_oversized_page"; + public const string Login = "login"; + public const string ViewProfile = "view_profile"; + public const string RefreshToken = "refresh_token"; + public const string Logout = "logout"; +} + internal static class LoadTestBudgets { + public static class Rates + { + public const int ListTournamentsPerSecond = 15; + public const int ViewTournamentPerSecond = 6; + public const int DiscoverTournamentsPerSecond = 4; + public const int InsightPerSecond = 5; + public const int AuthFlowPerSecond = 5; + } + + public static class Steps + { + private const double P95Headroom = 3.5; + private const double P99Headroom = 3.5; + private const double MaxHeadroom = 5; + private const double PayloadFloorRatio = 0.6; + private const double PayloadCeilingRatio = 1.6; + private const double BytesPerKilobyte = 1_000; + + private const double P95FloorMs = 100; + private const double P99FloorMs = 250; + private const double MaxFloorMs = 1_000; + + private static readonly Dictionary<(LoadTestDataSize Size, string Step), MeasuredBaseline> Measured = new() + { + [(LoadTestDataSize.Default, StepNames.ListTournaments)] = new(7.8, 13.46, 48.05, 1.388), + [(LoadTestDataSize.Default, StepNames.ViewTournament)] = new(18.96, 23.97, 30.28, 0.592), + [(LoadTestDataSize.Default, StepNames.SearchTournamentsByName)] = new(8.26, 11.89, 12.15, 0.314), + [(LoadTestDataSize.Default, StepNames.SortTournaments)] = new(19.54, 21.01, 22.43, 2.678), + + [(LoadTestDataSize.Medium, StepNames.ListTournaments)] = new(8.56, 13.29, 35.31, 1.410), + [(LoadTestDataSize.Medium, StepNames.ViewTournament)] = new(23.52, 47.84, 60.71, 1.425), + [(LoadTestDataSize.Medium, StepNames.SearchTournamentsByName)] = new(8.49, 14.45, 35.30, 1.410), + [(LoadTestDataSize.Medium, StepNames.SortTournaments)] = new(25.81, 48.42, 61.24, 6.173), + [(LoadTestDataSize.Medium, StepNames.ViewOwnerHistory)] = new(124.61, 215.81, 241.48, 0.415), + [(LoadTestDataSize.Medium, StepNames.ViewParticipantsWonTournaments)] = new(220.67, 260.99, 263.62, 0.614), + [(LoadTestDataSize.Medium, StepNames.ViewParticipantsPlayedTournaments)] = new(193.15, 265.73, 324.24, 0.638), + [(LoadTestDataSize.Medium, StepNames.ViewParticipantsWonMatches)] = new(236.42, 321.02, 321.11, 0.583), + [(LoadTestDataSize.Medium, StepNames.ViewBracketStandings)] = new(72.58, 211.46, 249.80, 1.007), + [(LoadTestDataSize.Medium, StepNames.ViewOwnTournamentHistory)] = new(94.66, 208.77, 208.91, 0.487), + [(LoadTestDataSize.Medium, StepNames.ListTournamentsWithOversizedPage)] = new(14.54, 48.19, 147.34, 0.213), + [(LoadTestDataSize.Medium, StepNames.Login)] = new(537.60, 824.83, 970.28, 0.653), + [(LoadTestDataSize.Medium, StepNames.ViewProfile)] = new(18.18, 39.23, 47.57, 0.131), + [(LoadTestDataSize.Medium, StepNames.RefreshToken)] = new(84.42, 170.62, 215.82, 0.656), + [(LoadTestDataSize.Medium, StepNames.Logout)] = new(46.69, 83.07, 83.73, 0.039), + + [(LoadTestDataSize.Large, StepNames.ListTournaments)] = new(9.35, 12.01, 41.25, 1.416), + [(LoadTestDataSize.Large, StepNames.ViewTournament)] = new(30.34, 45.41, 51.63, 1.770), + [(LoadTestDataSize.Large, StepNames.SearchTournamentsByName)] = new(10.14, 22.27, 41.25, 1.416), + [(LoadTestDataSize.Large, StepNames.SortTournaments)] = new(42.40, 54.37, 74.04, 8.631), + [(LoadTestDataSize.Large, StepNames.ViewOwnerHistory)] = new(144.77, 237.18, 288.05, 0.571), + [(LoadTestDataSize.Large, StepNames.ViewParticipantsWonTournaments)] = new(244.74, 369.92, 486.91, 1.162), + [(LoadTestDataSize.Large, StepNames.ViewParticipantsPlayedTournaments)] = new(240.26, 408.32, 430.80, 1.216), + [(LoadTestDataSize.Large, StepNames.ViewParticipantsWonMatches)] = new(253.18, 411.39, 472.05, 1.110), + [(LoadTestDataSize.Large, StepNames.ViewBracketStandings)] = new(117.38, 237.82, 262.43, 1.910), + [(LoadTestDataSize.Large, StepNames.ViewOwnTournamentHistory)] = new(130.69, 231.04, 233.95, 0.940), + [(LoadTestDataSize.Large, StepNames.ListTournamentsWithOversizedPage)] = new(8.34, 29.39, 43.05, 0.213) + }; + + public static StepBudget For(LoadTestDataSize size, string step) + { + if (!Measured.TryGetValue((size, step), out var baseline)) + { + throw new InvalidOperationException($"No measured baseline for step '{step}' on the {size} dataset."); + } + + return new StepBudget( + step, + Math.Max(P95FloorMs, baseline.P95Ms * P95Headroom), + Math.Max(P99FloorMs, baseline.P99Ms * P99Headroom), + Math.Max(MaxFloorMs, baseline.MaxMs * MaxHeadroom), + baseline.MeanKilobytes * BytesPerKilobyte * PayloadFloorRatio, + baseline.MeanKilobytes * BytesPerKilobyte * PayloadCeilingRatio); + } + } + public static class ListTournaments { public const string Step = "list_tournaments"; diff --git a/TournamentAPI.LoadTests/LoadTestDataSeeder.cs b/TournamentAPI.LoadTests/LoadTestDataSeeder.cs new file mode 100644 index 0000000..0b93357 --- /dev/null +++ b/TournamentAPI.LoadTests/LoadTestDataSeeder.cs @@ -0,0 +1,172 @@ +using Microsoft.AspNetCore.Identity; +using Microsoft.EntityFrameworkCore; +using TournamentAPI.Data; +using TournamentAPI.Data.Models; + +namespace TournamentAPI.LoadTests; + +public enum LoadTestDataSize +{ + Default, + Medium, + Large +} + +internal static class LoadTestDataSeeder +{ + public const string UserPassword = "LoadTestPassword123!"; + public const string TournamentNamePrefix = "Load Tournament"; + + public static async Task SeedAsync( + ApplicationDbContext context, + UserManager userManager, + LoadTestDataSize size) + { + var (userCount, tournamentCount, participantsPerTournament) = size switch + { + LoadTestDataSize.Medium => (50, 25, 8), + LoadTestDataSize.Large => (200, 100, 16), + _ => throw new ArgumentOutOfRangeException(nameof(size), size, "Only Medium and Large are seeded here.") + }; + + for (var i = 0; i < userCount; i++) + { + var user = new ApplicationUser + { + UserName = $"loaduser{i}", + Email = $"loaduser{i}@load.test", + FirstName = $"LoadFirstName{i}", + LastName = $"LoadLastName{i}", + IsEmailPublic = true + }; + + var result = await userManager.CreateAsync(user, UserPassword); + if (!result.Succeeded) + { + var errors = string.Join(", ", result.Errors.Select(e => e.Description)); + throw new InvalidOperationException($"Failed to create user {user.UserName}: {errors}"); + } + } + + var users = await context.Users.OrderBy(u => u.Id).ToListAsync(); + var random = new Random(42); + + for (var i = 0; i < tournamentCount; i++) + { + var participants = users + .OrderBy(_ => random.Next()) + .Take(Math.Min(participantsPerTournament, users.Count)) + .ToList(); + + context.Tournaments.Add(BuildTournament(i, participants, participantsPerTournament, random)); + } + + await context.SaveChangesAsync(); + } + + private static Tournament BuildTournament( + int index, + IReadOnlyList participants, + int maxParticipants, + Random random) + { + var kind = index % 4; + var owner = participants[0]; + + var tournament = new Tournament + { + Name = $"{TournamentNamePrefix} {index}", + StartDate = DateTime.UtcNow.AddDays(random.Next(-30, 30)), + OwnerId = owner.Id, + Owner = owner, + MaxParticipants = maxParticipants, + Status = kind == 0 ? TournamentStatus.Open : kind == 2 ? TournamentStatus.Completed : TournamentStatus.Closed, + Format = kind == 3 ? TournamentFormat.RoundRobin : TournamentFormat.SingleElimination, + Participants = new List() + }; + + for (var slot = 0; slot < participants.Count; slot++) + { + tournament.Participants.Add(new TournamentParticipant + { + Tournament = tournament, + Participant = participants[slot], + SlotNumber = slot + 1 + }); + } + + if (kind == 0) + { + return tournament; + } + + tournament.Bracket = new Bracket { Matches = new List() }; + + if (kind == 3) + { + AddRoundRobinMatches(tournament.Bracket, participants, random); + } + else + { + AddFirstRoundMatches(tournament.Bracket, participants, random, playAll: kind == 2); + } + + if (kind == 2) + { + var champion = participants[0]; + tournament.ChampionId = champion.Id; + tournament.Champion = champion; + } + + return tournament; + } + + private static void AddFirstRoundMatches( + Bracket bracket, + IReadOnlyList participants, + Random random, + bool playAll) + { + for (var i = 0; i + 1 < participants.Count; i += 2) + { + var played = playAll || i % 4 == 0; + bracket.Matches.Add(NewMatch(bracket, 1, participants[i], participants[i + 1], played, random)); + } + } + + private static void AddRoundRobinMatches( + Bracket bracket, + IReadOnlyList participants, + Random random) + { + var pairing = 0; + + for (var i = 0; i < participants.Count; i++) + { + for (var j = i + 1; j < participants.Count; j++) + { + var played = pairing++ % 4 != 0; + bracket.Matches.Add(NewMatch(bracket, 1, participants[i], participants[j], played, random)); + } + } + } + + private static Match NewMatch( + Bracket bracket, + int round, + ApplicationUser player1, + ApplicationUser player2, + bool played, + Random random) + { + return new Match + { + Round = round, + Player1Id = player1.Id, + Player2Id = player2.Id, + WinnerId = played ? (random.Next(2) == 0 ? player1.Id : player2.Id) : null, + Status = played ? MatchStatus.Played : MatchStatus.Scheduled, + Bracket = bracket + }; + } +} diff --git a/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs b/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs index ac2f858..681b1fc 100644 --- a/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs +++ b/TournamentAPI.LoadTests/LoadTestWebAppFactory.cs @@ -1,5 +1,6 @@ using Hangfire; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.TestHost; using Microsoft.EntityFrameworkCore; @@ -9,6 +10,7 @@ using Microsoft.Extensions.Hosting; using Testcontainers.MsSql; using TournamentAPI.Data; +using TournamentAPI.Data.Models; namespace TournamentAPI.LoadTests; @@ -43,18 +45,50 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) }); } + public IReadOnlyList TournamentIds { get; private set; } = []; + + public IReadOnlyList RoundRobinTournamentIds { get; private set; } = []; + + public IReadOnlyList UserEmails { get; private set; } = []; + + public virtual string UserPassword => "Password123!"; + + public virtual string TournamentSearchTerm => "Cup"; + public async Task InitializeAsync() { await _dbContainer.StartAsync(); using var scope = Services.CreateScope(); var context = scope.ServiceProvider.GetRequiredService(); - var userManager = scope.ServiceProvider.GetRequiredService>(); + var userManager = scope.ServiceProvider.GetRequiredService>(); await context.Database.EnsureDeletedAsync(); await context.Database.EnsureCreatedAsync(); - await DatabaseSeeder.SeedAsync(context, userManager); + await SeedAsync(context, userManager); + + TournamentIds = await context.Tournaments + .AsNoTracking() + .OrderBy(t => t.Id) + .Select(t => t.Id) + .ToListAsync(); + + RoundRobinTournamentIds = await context.Tournaments + .AsNoTracking() + .Where(t => t.Format == TournamentFormat.RoundRobin && t.Bracket != null) + .OrderBy(t => t.Id) + .Select(t => t.Id) + .ToListAsync(); + + UserEmails = await context.Users + .AsNoTracking() + .OrderBy(u => u.Id) + .Select(u => u.Email!) + .ToListAsync(); } + protected virtual Task SeedAsync(ApplicationDbContext context, UserManager userManager) + => DatabaseSeeder.SeedAsync(context, userManager); + public new Task DisposeAsync() => _dbContainer.StopAsync(); } \ No newline at end of file diff --git a/TournamentAPI.LoadTests/LoadThresholds.cs b/TournamentAPI.LoadTests/LoadThresholds.cs index c69b52a..8ade61a 100644 --- a/TournamentAPI.LoadTests/LoadThresholds.cs +++ b/TournamentAPI.LoadTests/LoadThresholds.cs @@ -83,6 +83,38 @@ public static LoadThreshold StepHealth(string stepName, double maxFailPercent) stepName, s => s.Fail.Request.Percent <= maxFailPercent); + public static LoadThreshold[] ForBudget(StepBudget budget) + { + var thresholds = new List + { + StepHealth(budget.Step, 0), + TailLatency(budget.Step, budget.P95Ms, budget.P99Ms, budget.MaxMs) + }; + + if (budget.MinMeanBytes is { } minMeanBytes && budget.MaxMeanBytes is { } maxMeanBytes) + { + thresholds.Add(PayloadGuard(budget.Step, minMeanBytes, maxMeanBytes)); + } + + return thresholds.ToArray(); + } + + public static LoadThreshold[] ForSteps(LoadTestDataSize size, string okCode, params string[] steps) + { + var thresholds = new List + { + ErrorBudget(0), + StatusMix(okCode, 100) + }; + + foreach (var step in steps) + { + thresholds.AddRange(ForBudget(LoadTestBudgets.Steps.For(size, step))); + } + + return thresholds.ToArray(); + } + public static bool IsStatusMixSatisfied( ScenarioStats stats, string okCode, @@ -102,31 +134,58 @@ public static bool IsStatusMixSatisfied( } } +internal sealed record ScenarioRun(ScenarioProps Scenario, LoadThreshold[] Thresholds); + internal static class LoadTestRun { public static ScenarioStats Execute(ScenarioProps scenario, params LoadThreshold[] thresholds) { - var withThresholds = thresholds.Length == 0 - ? scenario - : scenario.WithThresholds(thresholds.Select(t => t.ToNBomberThreshold()).ToArray()); + var results = ExecuteAll(new ScenarioRun(scenario, thresholds)); + + return results[scenario.ScenarioName]; + } + + public static IReadOnlyDictionary ExecuteAll(params ScenarioRun[] runs) + { + var scenarios = runs + .Select(run => run.Thresholds.Length == 0 + ? run.Scenario + : run.Scenario.WithThresholds(run.Thresholds.Select(t => t.ToNBomberThreshold()).ToArray())) + .ToArray(); var result = NBomberRunner - .RegisterScenarios(withThresholds) + .RegisterScenarios(scenarios) .Run(); - var stats = result.ScenarioStats.Get(scenario.ScenarioName); - AssertPassed(stats, thresholds); + var stats = runs.ToDictionary( + run => run.Scenario.ScenarioName, + run => result.ScenarioStats.Get(run.Scenario.ScenarioName)); + + var failures = runs.SelectMany(run => FindFailures(stats[run.Scenario.ScenarioName], run.Thresholds)).ToList(); + + if (failures.Count > 0) + { + Assert.Fail($"Load thresholds failed:{Environment.NewLine}- {string.Join($"{Environment.NewLine}- ", failures)}"); + } return stats; } public static void AssertPassed(ScenarioStats stats, IEnumerable thresholds) { - var failed = thresholds.Where(t => !t.IsSatisfiedBy(stats)).Select(t => t.Name).ToList(); + var failures = FindFailures(stats, thresholds); - if (failed.Count > 0) + if (failures.Count > 0) { - Assert.Fail($"Load thresholds failed for '{stats.ScenarioName}':{Environment.NewLine}- {string.Join($"{Environment.NewLine}- ", failed)}"); + Assert.Fail($"Load thresholds failed:{Environment.NewLine}- {string.Join($"{Environment.NewLine}- ", failures)}"); } } + + private static List FindFailures(ScenarioStats stats, IEnumerable thresholds) + { + return thresholds + .Where(t => !t.IsSatisfiedBy(stats)) + .Select(t => $"[{stats.ScenarioName}] {t.Name}") + .ToList(); + } } diff --git a/TournamentAPI.LoadTests/TournamentBrowseLoadTests.cs b/TournamentAPI.LoadTests/TournamentBrowseLoadTests.cs new file mode 100644 index 0000000..b33bdca --- /dev/null +++ b/TournamentAPI.LoadTests/TournamentBrowseLoadTests.cs @@ -0,0 +1,120 @@ +using NBomber.CSharp; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Load")] +public abstract class TournamentBrowseLoadTestsBase : BaseLoadTest +{ + private static readonly TimeSpan WarmUp = TimeSpan.FromSeconds(10); + private static readonly TimeSpan Duration = TimeSpan.FromSeconds(30); + + private readonly LoadTestWebAppFactory _factory; + + protected TournamentBrowseLoadTestsBase(LoadTestWebAppFactory factory) : base(factory) + { + _factory = factory; + } + + protected abstract LoadTestDataSize Size { get; } + + [Fact] + public void Api_ShouldMeetBudgets_UnderMixedBrowsingTraffic() + { + // Arrange + using var client = CreateClient(); + var http = client.HttpClient; + var tournamentIds = _factory.TournamentIds; + var searchTerm = _factory.TournamentSearchTerm; + + var browseList = Scenario.Create("browse_tournament_list", async context => + await GraphQLLoadStep.RunAsync( + StepNames.ListTournaments, + context, + http, + Shared.QueryExamples.Queries.Tournaments.GetAllWithTotalCount, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments"))) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Simulation.Inject( + rate: LoadTestBudgets.Rates.ListTournamentsPerSecond, + interval: TimeSpan.FromSeconds(1), + during: Duration)); + + var viewDetails = Scenario.Create("view_tournament_details", async context => + await GraphQLLoadStep.RunAsync( + StepNames.ViewTournament, + context, + http, + Shared.QueryExamples.Queries.Tournaments.GetByIdWithParticipants, + new { id = tournamentIds[context.Random.Next(tournamentIds.Count)] }, + GraphQLLoadStep.NonNullField("tournamentById"))) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Simulation.Inject( + rate: LoadTestBudgets.Rates.ViewTournamentPerSecond, + interval: TimeSpan.FromSeconds(1), + during: Duration)); + + var discover = Scenario.Create("discover_tournaments", async context => + { + var search = await GraphQLLoadStep.RunAsync( + StepNames.SearchTournamentsByName, + context, + http, + Shared.QueryExamples.Queries.Tournaments.GetAllWithNameFilter, + new { nameFilter = searchTerm }, + GraphQLLoadStep.NonEmptyConnection("tournaments")); + + if (search.IsError) + { + return search; + } + + return await GraphQLLoadStep.RunAsync( + StepNames.SortTournaments, + context, + http, + Shared.QueryExamples.Queries.Tournaments.GetAllWithSorting, + bodyCheck: GraphQLLoadStep.NonEmptyConnection("tournaments")); + }) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Simulation.Inject( + rate: LoadTestBudgets.Rates.DiscoverTournamentsPerSecond, + interval: TimeSpan.FromSeconds(1), + during: Duration)); + + // Act & Assert + LoadTestRun.ExecuteAll( + new ScenarioRun(browseList, LoadThresholds.ForSteps(Size, "200", StepNames.ListTournaments)), + new ScenarioRun(viewDetails, LoadThresholds.ForSteps(Size, "200", StepNames.ViewTournament)), + new ScenarioRun(discover, LoadThresholds.ForSteps(Size, "200", StepNames.SearchTournamentsByName, StepNames.SortTournaments))); + } +} + +public class DefaultDatasetTournamentBrowseLoadTests + : TournamentBrowseLoadTestsBase, IClassFixture +{ + public DefaultDatasetTournamentBrowseLoadTests(LatencyWebAppFactory factory) : base(factory) + { + } + + protected override LoadTestDataSize Size => LoadTestDataSize.Default; +} + +public class MediumDatasetTournamentBrowseLoadTests + : TournamentBrowseLoadTestsBase, IClassFixture +{ + public MediumDatasetTournamentBrowseLoadTests(MediumDatasetWebAppFactory factory) : base(factory) + { + } + + protected override LoadTestDataSize Size => LoadTestDataSize.Medium; +} + +public class LargeDatasetTournamentBrowseLoadTests + : TournamentBrowseLoadTestsBase, IClassFixture +{ + public LargeDatasetTournamentBrowseLoadTests(LargeDatasetWebAppFactory factory) : base(factory) + { + } + + protected override LoadTestDataSize Size => LoadTestDataSize.Large; +} diff --git a/TournamentAPI.LoadTests/TournamentInsightsLoadTests.cs b/TournamentAPI.LoadTests/TournamentInsightsLoadTests.cs new file mode 100644 index 0000000..b896f83 --- /dev/null +++ b/TournamentAPI.LoadTests/TournamentInsightsLoadTests.cs @@ -0,0 +1,149 @@ +using System.Text.Json; +using NBomber.Contracts; +using NBomber.CSharp; +using TournamentQueries = TournamentAPI.Shared.QueryExamples.Queries.Tournaments; +using UserQueries = TournamentAPI.Shared.QueryExamples.Queries.Users; + +namespace TournamentAPI.LoadTests; + +[Trait("Category", "Load")] +public abstract class TournamentInsightsLoadTestsBase : BaseLoadTest +{ + private const string PageLimitRejectedCode = "PAGE_SIZE_LIMIT_EXCEEDED"; + private const int LoggedInUsers = 10; + + private static readonly TimeSpan WarmUp = TimeSpan.FromSeconds(10); + private static readonly TimeSpan Duration = TimeSpan.FromSeconds(30); + + private readonly LoadTestWebAppFactory _factory; + + protected TournamentInsightsLoadTestsBase(LoadTestWebAppFactory factory) : base(factory) + { + _factory = factory; + } + + protected abstract LoadTestDataSize Size { get; } + + [Fact] + public void Api_ShouldMeetBudgets_ForExpensiveQueryShapes() + { + // Arrange + using var client = CreateClient(); + var http = client.HttpClient; + var tournamentIds = _factory.TournamentIds; + var roundRobinIds = _factory.RoundRobinTournamentIds; + + var ownerHistory = InspectTournament( + "inspect_owner_history", StepNames.ViewOwnerHistory, http, TournamentQueries.GetByIdWithOwnerTournamentHistory, tournamentIds); + + var participantsWonTournaments = InspectTournament( + "inspect_participants_won_tournaments", StepNames.ViewParticipantsWonTournaments, http, TournamentQueries.GetByIdWithParticipantsWonTournaments, tournamentIds); + + var participantsPlayedTournaments = InspectTournament( + "inspect_participants_played_tournaments", StepNames.ViewParticipantsPlayedTournaments, http, TournamentQueries.GetByIdWithParticipantsPlayedTournaments, tournamentIds); + + var participantsWonMatches = InspectTournament( + "inspect_participants_won_matches", StepNames.ViewParticipantsWonMatches, http, TournamentQueries.GetByIdWithParticipantsWonMatches, tournamentIds); + + var bracketStandings = InspectTournament( + "inspect_bracket_standings", StepNames.ViewBracketStandings, http, TournamentQueries.GetByIdWithBracketStandings, roundRobinIds, HasStandings); + + VirtualUserSession[] sessions = []; + + var ownHistory = Scenario.Create("inspect_own_tournament_history", async context => + { + var session = VirtualUserSessions.Pick(sessions, context.InvocationNumber); + + return await GraphQLLoadStep.RunAsync( + StepNames.ViewOwnTournamentHistory, + context, + session.Client, + UserQueries.GetMeWithTournamentHistory, + bodyCheck: GraphQLLoadStep.NonNullField("me"), + bearerToken: session.AccessToken); + }) + .WithInit(async _ => + { + sessions = await VirtualUserSessions.LoginAsync(CreateClient, _factory.UserEmails, _factory.UserPassword, LoggedInUsers); + }) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Steady()); + + var oversizedPage = Scenario.Create("request_oversized_page", async context => + await GraphQLLoadStep.RunAsync( + StepNames.ListTournamentsWithOversizedPage, + context, + http, + TournamentQueries.GetAllWithExcessivePageSize, + classifyRejection: GraphQLLoadStep.RejectionWhenErrorHasExtension("maxAllowedItems", PageLimitRejectedCode))) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Steady()); + + // Act & Assert + LoadTestRun.ExecuteAll( + new ScenarioRun(ownerHistory, LoadThresholds.ForSteps(Size, "200", StepNames.ViewOwnerHistory)), + new ScenarioRun(participantsWonTournaments, LoadThresholds.ForSteps(Size, "200", StepNames.ViewParticipantsWonTournaments)), + new ScenarioRun(participantsPlayedTournaments, LoadThresholds.ForSteps(Size, "200", StepNames.ViewParticipantsPlayedTournaments)), + new ScenarioRun(participantsWonMatches, LoadThresholds.ForSteps(Size, "200", StepNames.ViewParticipantsWonMatches)), + new ScenarioRun(bracketStandings, LoadThresholds.ForSteps(Size, "200", StepNames.ViewBracketStandings)), + new ScenarioRun(ownHistory, LoadThresholds.ForSteps(Size, "200", StepNames.ViewOwnTournamentHistory)), + new ScenarioRun(oversizedPage, LoadThresholds.ForSteps(Size, PageLimitRejectedCode, StepNames.ListTournamentsWithOversizedPage))); + } + + private static ScenarioProps InspectTournament( + string scenarioName, + string stepName, + HttpClient http, + string query, + IReadOnlyList tournamentIds, + Func? bodyCheck = null) + { + return Scenario.Create(scenarioName, async context => + await GraphQLLoadStep.RunAsync( + stepName, + context, + http, + query, + new { id = tournamentIds[context.Random.Next(tournamentIds.Count)] }, + bodyCheck ?? GraphQLLoadStep.NonNullField("tournamentById"))) + .WithWarmUpDuration(WarmUp) + .WithLoadSimulations(Steady()); + } + + private static LoadSimulation Steady() + { + return Simulation.Inject( + rate: LoadTestBudgets.Rates.InsightPerSecond, + interval: TimeSpan.FromSeconds(1), + during: Duration); + } + + private static bool HasStandings(JsonElement data) + { + return data.TryGetProperty("tournamentById", out var tournament) + && tournament.TryGetProperty("bracket", out var bracket) + && bracket.TryGetProperty("standings", out var standings) + && standings.ValueKind == JsonValueKind.Array + && standings.GetArrayLength() > 0; + } +} + +public class MediumDatasetTournamentInsightsLoadTests + : TournamentInsightsLoadTestsBase, IClassFixture +{ + public MediumDatasetTournamentInsightsLoadTests(MediumDatasetWebAppFactory factory) : base(factory) + { + } + + protected override LoadTestDataSize Size => LoadTestDataSize.Medium; +} + +public class LargeDatasetTournamentInsightsLoadTests + : TournamentInsightsLoadTestsBase, IClassFixture +{ + public LargeDatasetTournamentInsightsLoadTests(LargeDatasetWebAppFactory factory) : base(factory) + { + } + + protected override LoadTestDataSize Size => LoadTestDataSize.Large; +} diff --git a/TournamentAPI.LoadTests/VirtualUserSessions.cs b/TournamentAPI.LoadTests/VirtualUserSessions.cs new file mode 100644 index 0000000..3fb9989 --- /dev/null +++ b/TournamentAPI.LoadTests/VirtualUserSessions.cs @@ -0,0 +1,37 @@ +using TournamentAPI.Shared.Helpers; +using TournamentAPI.Shared.Models; + +namespace TournamentAPI.LoadTests; + +internal sealed record VirtualUserSession(HttpClient Client, string AccessToken); + +internal static class VirtualUserSessions +{ + public static async Task LoginAsync( + Func createClient, + IReadOnlyList emails, + string password, + int count) + { + var sessions = new List(); + + foreach (var email in emails.Take(count)) + { + var client = createClient(); + + var response = await client.ExecuteMutationAsync( + Shared.MutationExamples.Mutations.Users.LoginUser, + new { input = new { email, password } }); + + var accessToken = response.Data?.LoginUser?.String + ?? throw new InvalidOperationException($"Login failed for {email}: {response.Errors?.FirstOrDefault()?.Message}"); + + sessions.Add(new VirtualUserSession(client.HttpClient, accessToken)); + } + + return sessions.ToArray(); + } + + public static VirtualUserSession Pick(VirtualUserSession[] sessions, long invocationNumber) + => sessions[(int)(invocationNumber % sessions.Length)]; +}