diff --git a/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/analysis.md b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/analysis.md new file mode 100644 index 0000000..acf7644 --- /dev/null +++ b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/analysis.md @@ -0,0 +1,90 @@ +# Run Analysis + +Run: [run.json](run.json) + +Results: [result.json](result.json) + +This document is generated by AI from the run metadata, machine results, and +retained diagnostics. + +## Purpose + +Locate the local capacity ceiling of the `kq-bench` harness on the +`north-star-steady-v1` execution-time distribution (250 ms average, 475 ms p95, +646 ms p99) by scaling producers, workers, worker concurrency, and ready +partitions. + +## Observations + +Capacity probes at 3,000 RPS with 4 workers and 16 ready partitions: + +| Concurrency | Active worker RPS | Per worker | Drain | Queue p50 | +| ---: | ---: | ---: | ---: | ---: | +| 100 | 529.7 | 132 | 93.3 s | 44.6 s | +| 250 | 1,136.2 | 284 | 32.8 s | 15.8 s | +| 500 | 2,006.3 | 502 | 9.9 s | 3.5 s | +| 1000 | 2,742.1 | 686 | 1.9 s | 501 ms | + +Arrival-rate scaling: + +| Requested RPS | Producers | Workers x concurrency | Partitions | Enqueue RPS | Worker RPS | Drain | Queue p50 / p95 / p99 | Host busy | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 500 | 4 | 4 x 100 | 4 | 500.0 | 473.5 | 1.7 s | 438 / 982 / 1,295 ms | n/a | +| 5,000 | 8 | 8 x 1000 | 16 | 4,995 | 4,613 | 1.7 s | 475 / 984 / 1,206 ms | 68% | +| 8,000 | 10 | 12 x 1000 | 24 | 7,992 | 7,395 | 1.6 s | 475 / 997 / 1,211 ms | 87% | +| 12,000 | 15 | 18 x 1000 | 32 | 11,984 | 11,114 | 1.6 s | 479 / 1,001 / 1,214 ms | 99% | +| 16,000 | 20 | 24 x 1000 | 32 | 15,391 | 12,446 | 5.0 s | 700 / 4,087 / 5,233 ms | 100% | +| 16,000 | 16 | 12 x 2000 | 32 | 15,958 | 12,760 | 5.1 s | 813 / 4,770 / 6,967 ms | 100% | +| 20,000 | 20 | 15 x 2000 | 40 | 16,925 | 13,463 | 6.1 s | 1,025 / 7,962 / 10,367 ms | 100% | + +- 12,000 RPS was the highest sustainable tested rate; 16,000 RPS was the first + overloaded rate. Maximum observed worker throughput was 13,463 tasks/s. +- At 16,000 and 20,000 RPS producers also fell behind the requested rate + (15,391 and 16,925 enqueues/s). +- Steady-state CPU at 12,000 RPS, measured as cumulative process CPU time over + a 20 s window: Kafka Docker VM 4.29 cores, producers 2.75, harness Python + 1.73, workers 0.77, other host processes 1.03. +- Harness Python resident memory reached about 2.6 GB during the 45 s + 12,000 RPS point. +- All 2,815,000 measured tasks across all points completed with zero missing, + duplicate, outstanding, or dead-lettered IDs. +- Measured execution averaged 250 to 252 ms with a 645 to 657 ms p99. Execution + maximum rose from about 1,494 ms to 1,805 ms once the host was saturated. + +## Interpretation + +- The ceiling is host CPU saturation, not a kq-specific limit. Workers used + 0.77 cores for over 11,000 tasks/s; the harness's own Python accounting and + producer-side event emission consumed several cores of the same 12-core host. +- Each harness producer calls `Enqueue` serially, so every task is an + individual `acks=all` produce request. Hypothesis: this drives a large share + of producer and Kafka CPU; the lean-probe run measures concurrent `Enqueue` + separately. +- Queue p50 of about 475 ms is nearly independent of arrival rate at + concurrency 1000. It reflects worker generation behavior rather than backlog; + the queue-latency diagnosis run investigates it. +- The rising execution maximum at saturated points suggests timer overshoot + from CPU contention. + +## Suggestions + +1. Rework the harness before using it above about 10,000 RPS: concurrent or + batched producer enqueues, compact or sampled per-task events, and + streaming accounting instead of in-memory rescans every second. +2. Record per-process CPU and memory in the harness so saturated runs are + identified automatically. +3. Compare against the lean-probe capacity run, which removes harness overhead + on the same host. + +## Caveats + +- Each point ran once on one arm64 laptop with a single Dockerized Kafka broker + (RF=1, 12 CPUs and 8 GB assigned to the Docker VM). Load generators, workers, + and Kafka shared the host. +- Points used 20 to 45 s arrival windows; longer runs were not attempted + because harness memory grows with event count. +- Host and process CPU peaks come from 2 s samples of `top`, `docker stats`, + and `ps`; `ps` CPU is a decaying average. The 12,000 RPS breakdown uses + cumulative CPU time and is more reliable. +- The harness was invoked from source because the configured package index + rejected the `hatchling` build dependency; the harness code was unchanged. diff --git a/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/result.json b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/result.json new file mode 100644 index 0000000..ffb6650 --- /dev/null +++ b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/result.json @@ -0,0 +1,556 @@ +{ + "capacity": { + "highest_sustainable_tested_rps": 12000, + "first_overloaded_tested_rps": 16000, + "maximum_observed_active_worker_rps": 13462.589, + "maximum_observed_enqueue_rps": 16925.084 + }, + "cpu_breakdown_12k_rps": { + "source_point": "cpu12k", + "method": "per-process cumulative CPU time delta over a 20 s steady-state window, divided by wall time", + "cores": { + "kafka_docker_vm": 4.29, + "producers": 2.75, + "harness_python": 1.73, + "workers": 0.77, + "mover_and_observer": 0.0, + "other_host": 1.03 + } + }, + "correctness": { + "produced": 2815000, + "completed": 2815000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0 + }, + "points": [ + { + "label": "smoke", + "requested_rps": 500, + "duration_seconds": 30, + "producers": 4, + "workers": 4, + "worker_concurrency": 100, + "ready_partitions": 4, + "achieved_enqueue_rps": 499.955, + "active_worker_rps": 473.519, + "drain_ms": 1699.773, + "sustained": true, + "queue_p50_ms": 437.519, + "queue_p95_ms": 982.153, + "queue_p99_ms": 1294.791, + "delivery_p50_ms": 688.988, + "delivery_p95_ms": 1280.133, + "delivery_p99_ms": 1579.452, + "enqueue_p50_ms": 0.689, + "enqueue_p99_ms": 2.77, + "execution_average_ms": 251.598, + "execution_p99_ms": 656.692, + "execution_max_ms": 1494.014, + "correctness": { + "completed": 15000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 15000 + }, + "sampled_peaks": null, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /kqperf --set workload.arrival.target_rps=500 --set workload.arrival.duration_seconds=30 --set topology.producers=4 --set topology.workers=4 --set topology.worker_concurrency=100 --set topology.ready_partitions=4" + }, + { + "label": "capC100", + "requested_rps": 3000, + "duration_seconds": 20, + "producers": 6, + "workers": 4, + "worker_concurrency": 100, + "ready_partitions": 16, + "achieved_enqueue_rps": 2998.839, + "active_worker_rps": 529.745, + "drain_ms": 93277.057, + "sustained": false, + "queue_p50_ms": 44575.481, + "queue_p95_ms": 86987.014, + "queue_p99_ms": 90402.088, + "delivery_p50_ms": 44840.198, + "delivery_p95_ms": 87243.293, + "delivery_p99_ms": 90627.816, + "enqueue_p50_ms": 0.315, + "enqueue_p99_ms": 0.944, + "execution_average_ms": 250.68, + "execution_p99_ms": 650.362, + "execution_max_ms": 1493.971, + "correctness": { + "completed": 60000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 60000 + }, + "sampled_peaks": { + "host_busy_percent": 50.6, + "kafka_container_cpu_percent": 216.7, + "kq_load_cpu_percent": 58, + "kq_load_rss_mb": 254, + "harness_python_cpu_percent": 29, + "harness_python_rss_mb": 270 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /capC100 --set workload.arrival.target_rps=3000 --set workload.arrival.duration_seconds=20 --set topology.producers=6 --set topology.workers=4 --set topology.worker_concurrency=100 --set topology.ready_partitions=16 --set timeouts.drain_seconds=240" + }, + { + "label": "capC250", + "requested_rps": 3000, + "duration_seconds": 20, + "producers": 6, + "workers": 4, + "worker_concurrency": 250, + "ready_partitions": 16, + "achieved_enqueue_rps": 2998.906, + "active_worker_rps": 1136.216, + "drain_ms": 32829.499, + "sustained": false, + "queue_p50_ms": 15778.688, + "queue_p95_ms": 30174.639, + "queue_p99_ms": 31711.795, + "delivery_p50_ms": 16030.769, + "delivery_p95_ms": 30418.353, + "delivery_p99_ms": 32002.521, + "enqueue_p50_ms": 0.36, + "enqueue_p99_ms": 1.054, + "execution_average_ms": 250.526, + "execution_p99_ms": 650.691, + "execution_max_ms": 1493.636, + "correctness": { + "completed": 60000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 60000 + }, + "sampled_peaks": { + "host_busy_percent": 69.2, + "kafka_container_cpu_percent": 316.5, + "kq_load_cpu_percent": 70, + "kq_load_rss_mb": 260, + "harness_python_cpu_percent": 24, + "harness_python_rss_mb": 269 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /capC250 --set workload.arrival.target_rps=3000 --set workload.arrival.duration_seconds=20 --set topology.producers=6 --set topology.workers=4 --set topology.worker_concurrency=250 --set topology.ready_partitions=16 --set timeouts.drain_seconds=240" + }, + { + "label": "capC500", + "requested_rps": 3000, + "duration_seconds": 20, + "producers": 6, + "workers": 4, + "worker_concurrency": 500, + "ready_partitions": 16, + "achieved_enqueue_rps": 2999.941, + "active_worker_rps": 2006.263, + "drain_ms": 9934.554, + "sustained": false, + "queue_p50_ms": 3508.021, + "queue_p95_ms": 10833.22, + "queue_p99_ms": 12374.796, + "delivery_p50_ms": 3762.021, + "delivery_p95_ms": 11081.333, + "delivery_p99_ms": 12631.386, + "enqueue_p50_ms": 0.291, + "enqueue_p99_ms": 0.741, + "execution_average_ms": 250.415, + "execution_p99_ms": 650.226, + "execution_max_ms": 1493.578, + "correctness": { + "completed": 60000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 60000 + }, + "sampled_peaks": { + "host_busy_percent": 75.0, + "kafka_container_cpu_percent": 260.68, + "kq_load_cpu_percent": 59, + "kq_load_rss_mb": 266, + "harness_python_cpu_percent": 38, + "harness_python_rss_mb": 263 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /capC500 --set workload.arrival.target_rps=3000 --set workload.arrival.duration_seconds=20 --set topology.producers=6 --set topology.workers=4 --set topology.worker_concurrency=500 --set topology.ready_partitions=16 --set timeouts.drain_seconds=240" + }, + { + "label": "capC1000", + "requested_rps": 3000, + "duration_seconds": 20, + "producers": 6, + "workers": 4, + "worker_concurrency": 1000, + "ready_partitions": 16, + "achieved_enqueue_rps": 2998.195, + "active_worker_rps": 2742.143, + "drain_ms": 1896.079, + "sustained": true, + "queue_p50_ms": 500.606, + "queue_p95_ms": 1072.521, + "queue_p99_ms": 1312.133, + "delivery_p50_ms": 748.943, + "delivery_p95_ms": 1367.854, + "delivery_p99_ms": 1634.258, + "enqueue_p50_ms": 0.369, + "enqueue_p99_ms": 1.397, + "execution_average_ms": 250.387, + "execution_p99_ms": 650.482, + "execution_max_ms": 1494.019, + "correctness": { + "completed": 60000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 60000 + }, + "sampled_peaks": { + "host_busy_percent": 54.7, + "kafka_container_cpu_percent": 286.67, + "kq_load_cpu_percent": 68, + "kq_load_rss_mb": 281, + "harness_python_cpu_percent": 97, + "harness_python_rss_mb": 266 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /capC1000 --set workload.arrival.target_rps=3000 --set workload.arrival.duration_seconds=20 --set topology.producers=6 --set topology.workers=4 --set topology.worker_concurrency=1000 --set topology.ready_partitions=16 --set timeouts.drain_seconds=240" + }, + { + "label": "r5k", + "requested_rps": 5000, + "duration_seconds": 20, + "producers": 8, + "workers": 8, + "worker_concurrency": 1000, + "ready_partitions": 16, + "achieved_enqueue_rps": 4994.773, + "active_worker_rps": 4613.395, + "drain_ms": 1684.682, + "sustained": true, + "queue_p50_ms": 475.407, + "queue_p95_ms": 984.087, + "queue_p99_ms": 1205.72, + "delivery_p50_ms": 724.801, + "delivery_p95_ms": 1284.392, + "delivery_p99_ms": 1520.589, + "enqueue_p50_ms": 0.337, + "enqueue_p99_ms": 1.082, + "execution_average_ms": 250.011, + "execution_p99_ms": 646.771, + "execution_max_ms": 1493.158, + "correctness": { + "completed": 100000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 100000 + }, + "sampled_peaks": { + "host_busy_percent": 67.6, + "kafka_container_cpu_percent": 189.5, + "kq_load_cpu_percent": 121, + "kq_load_rss_mb": 442, + "harness_python_cpu_percent": 53, + "harness_python_rss_mb": 393 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r5k --set workload.arrival.target_rps=5000 --set workload.arrival.duration_seconds=20 --set topology.producers=8 --set topology.workers=8 --set topology.worker_concurrency=1000 --set topology.ready_partitions=16 --set timeouts.drain_seconds=240" + }, + { + "label": "r8k", + "requested_rps": 8000, + "duration_seconds": 20, + "producers": 10, + "workers": 12, + "worker_concurrency": 1000, + "ready_partitions": 24, + "achieved_enqueue_rps": 7991.509, + "active_worker_rps": 7394.717, + "drain_ms": 1643.775, + "sustained": true, + "queue_p50_ms": 474.791, + "queue_p95_ms": 997.449, + "queue_p99_ms": 1210.845, + "delivery_p50_ms": 724.336, + "delivery_p95_ms": 1293.945, + "delivery_p99_ms": 1528.183, + "enqueue_p50_ms": 0.42, + "enqueue_p99_ms": 1.589, + "execution_average_ms": 249.801, + "execution_p99_ms": 646.675, + "execution_max_ms": 1493.748, + "correctness": { + "completed": 160000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 160000 + }, + "sampled_peaks": { + "host_busy_percent": 86.7, + "kafka_container_cpu_percent": 238.12, + "kq_load_cpu_percent": 232, + "kq_load_rss_mb": 602, + "harness_python_cpu_percent": 105, + "harness_python_rss_mb": 771 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r8k --set workload.arrival.target_rps=8000 --set workload.arrival.duration_seconds=20 --set topology.producers=10 --set topology.workers=12 --set topology.worker_concurrency=1000 --set topology.ready_partitions=24 --set timeouts.drain_seconds=240" + }, + { + "label": "r12k", + "requested_rps": 12000, + "duration_seconds": 20, + "producers": 15, + "workers": 18, + "worker_concurrency": 1000, + "ready_partitions": 32, + "achieved_enqueue_rps": 11984.156, + "active_worker_rps": 11114.172, + "drain_ms": 1593.796, + "sustained": true, + "queue_p50_ms": 479.039, + "queue_p95_ms": 1001.339, + "queue_p99_ms": 1213.749, + "delivery_p50_ms": 727.464, + "delivery_p95_ms": 1295.048, + "delivery_p99_ms": 1530.284, + "enqueue_p50_ms": 0.7, + "enqueue_p99_ms": 1.912, + "execution_average_ms": 249.566, + "execution_p99_ms": 644.815, + "execution_max_ms": 1496.496, + "correctness": { + "completed": 240000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 240000 + }, + "sampled_peaks": { + "host_busy_percent": 99.1, + "kafka_container_cpu_percent": 330.02, + "kq_load_cpu_percent": 428, + "kq_load_rss_mb": 890, + "harness_python_cpu_percent": 192, + "harness_python_rss_mb": 1138 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r12k --set workload.arrival.target_rps=12000 --set workload.arrival.duration_seconds=20 --set topology.producers=15 --set topology.workers=18 --set topology.worker_concurrency=1000 --set topology.ready_partitions=32 --set timeouts.drain_seconds=240" + }, + { + "label": "prof12k", + "requested_rps": 12000, + "duration_seconds": 40, + "producers": 15, + "workers": 18, + "worker_concurrency": 1000, + "ready_partitions": 32, + "achieved_enqueue_rps": 11983.926, + "active_worker_rps": 11384.692, + "drain_ms": 2143.885, + "sustained": true, + "queue_p50_ms": 477.783, + "queue_p95_ms": 1013.138, + "queue_p99_ms": 1293.222, + "delivery_p50_ms": 726.339, + "delivery_p95_ms": 1310.068, + "delivery_p99_ms": 1604.094, + "enqueue_p50_ms": 0.678, + "enqueue_p99_ms": 1.998, + "execution_average_ms": 249.5, + "execution_p99_ms": 644.864, + "execution_max_ms": 1804.656, + "correctness": { + "completed": 480000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 480000 + }, + "sampled_peaks": { + "host_busy_percent": 100.0, + "kafka_container_cpu_percent": 287.55, + "kq_load_cpu_percent": 313, + "kq_load_rss_mb": 904, + "harness_python_cpu_percent": 184, + "harness_python_rss_mb": 2368 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /prof12k --set workload.arrival.target_rps=12000 --set workload.arrival.duration_seconds=40 --set topology.producers=15 --set topology.workers=18 --set topology.worker_concurrency=1000 --set topology.ready_partitions=32 --set timeouts.drain_seconds=240" + }, + { + "label": "cpu12k", + "requested_rps": 12000, + "duration_seconds": 45, + "producers": 15, + "workers": 18, + "worker_concurrency": 1000, + "ready_partitions": 32, + "achieved_enqueue_rps": 11991.237, + "active_worker_rps": 11535.614, + "drain_ms": 1807.56, + "sustained": true, + "queue_p50_ms": 485.252, + "queue_p95_ms": 1029.463, + "queue_p99_ms": 1314.801, + "delivery_p50_ms": 734.58, + "delivery_p95_ms": 1327.889, + "delivery_p99_ms": 1620.303, + "enqueue_p50_ms": 0.714, + "enqueue_p99_ms": 2.22, + "execution_average_ms": 249.505, + "execution_p99_ms": 644.966, + "execution_max_ms": 1804.004, + "correctness": { + "completed": 540000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 540000 + }, + "sampled_peaks": { + "host_busy_percent": 100.0, + "kafka_container_cpu_percent": 340.58, + "kq_load_cpu_percent": 379, + "kq_load_rss_mb": 912, + "harness_python_cpu_percent": 196, + "harness_python_rss_mb": 2615 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /cpu12k --set workload.arrival.target_rps=12000 --set workload.arrival.duration_seconds=45 --set topology.producers=15 --set topology.workers=18 --set topology.worker_concurrency=1000 --set topology.ready_partitions=32 --set timeouts.drain_seconds=240" + }, + { + "label": "r16k", + "requested_rps": 16000, + "duration_seconds": 20, + "producers": 20, + "workers": 24, + "worker_concurrency": 1000, + "ready_partitions": 32, + "achieved_enqueue_rps": 15391.477, + "active_worker_rps": 12446.131, + "drain_ms": 4951.005, + "sustained": false, + "queue_p50_ms": 699.733, + "queue_p95_ms": 4086.928, + "queue_p99_ms": 5233.334, + "delivery_p50_ms": 966.663, + "delivery_p95_ms": 4397.416, + "delivery_p99_ms": 5521.904, + "enqueue_p50_ms": 0.882, + "enqueue_p99_ms": 2.623, + "execution_average_ms": 249.598, + "execution_p99_ms": 645.282, + "execution_max_ms": 1594.017, + "correctness": { + "completed": 320000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 320000 + }, + "sampled_peaks": { + "host_busy_percent": 100.0, + "kafka_container_cpu_percent": 322.76, + "kq_load_cpu_percent": 457, + "kq_load_rss_mb": 1171, + "harness_python_cpu_percent": 180, + "harness_python_rss_mb": 1496 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r16k --set workload.arrival.target_rps=16000 --set workload.arrival.duration_seconds=20 --set topology.producers=20 --set topology.workers=24 --set topology.worker_concurrency=1000 --set topology.ready_partitions=32 --set timeouts.drain_seconds=240" + }, + { + "label": "r16k-c2000", + "requested_rps": 16000, + "duration_seconds": 20, + "producers": 16, + "workers": 12, + "worker_concurrency": 2000, + "ready_partitions": 32, + "achieved_enqueue_rps": 15957.66, + "active_worker_rps": 12759.856, + "drain_ms": 5055.301, + "sustained": false, + "queue_p50_ms": 812.852, + "queue_p95_ms": 4770.124, + "queue_p99_ms": 6967.437, + "delivery_p50_ms": 1176.959, + "delivery_p95_ms": 5077.714, + "delivery_p99_ms": 7389.238, + "enqueue_p50_ms": 0.811, + "enqueue_p99_ms": 2.006, + "execution_average_ms": 249.593, + "execution_p99_ms": 645.302, + "execution_max_ms": 1594.749, + "correctness": { + "completed": 320000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 320000 + }, + "sampled_peaks": { + "host_busy_percent": 100.0, + "kafka_container_cpu_percent": 342.58, + "kq_load_cpu_percent": 355, + "kq_load_rss_mb": 846, + "harness_python_cpu_percent": 186, + "harness_python_rss_mb": 1569 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r16k-c2000 --set workload.arrival.target_rps=16000 --set workload.arrival.duration_seconds=20 --set topology.producers=16 --set topology.workers=12 --set topology.worker_concurrency=2000 --set topology.ready_partitions=32 --set timeouts.drain_seconds=240" + }, + { + "label": "r20k-c2000", + "requested_rps": 20000, + "duration_seconds": 20, + "producers": 20, + "workers": 15, + "worker_concurrency": 2000, + "ready_partitions": 40, + "achieved_enqueue_rps": 16925.084, + "active_worker_rps": 13462.589, + "drain_ms": 6102.466, + "sustained": false, + "queue_p50_ms": 1025.412, + "queue_p95_ms": 7961.55, + "queue_p99_ms": 10366.812, + "delivery_p50_ms": 1505.95, + "delivery_p95_ms": 8295.118, + "delivery_p99_ms": 10695.774, + "enqueue_p50_ms": 0.991, + "enqueue_p99_ms": 2.837, + "execution_average_ms": 249.539, + "execution_p99_ms": 644.287, + "execution_max_ms": 1791.135, + "correctness": { + "completed": 400000, + "dead_lettered": 0, + "duplicates": 0, + "missing": 0, + "outstanding": 0, + "produced": 400000 + }, + "sampled_peaks": { + "host_busy_percent": 100.0, + "kafka_container_cpu_percent": 347.37, + "kq_load_cpu_percent": 367, + "kq_load_rss_mb": 1043, + "harness_python_cpu_percent": 182, + "harness_python_rss_mb": 2003 + }, + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --output /r20k-c2000 --set workload.arrival.target_rps=20000 --set workload.arrival.duration_seconds=20 --set topology.producers=20 --set topology.workers=15 --set topology.worker_concurrency=2000 --set topology.ready_partitions=40 --set timeouts.drain_seconds=240" + } + ] +} diff --git a/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/run.json b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/run.json new file mode 100644 index 0000000..17d6b5b --- /dev/null +++ b/docs/performance/runs/2026-09-23-north-star-steady-v1-harness-capacity-33c95bf/run.json @@ -0,0 +1,95 @@ +{ + "run_id": "2026-09-23-north-star-steady-v1-harness-capacity-33c95bf", + "created_at": "2026-09-23T00:05:00Z", + "revision": "33c95bf74b5c", + "purpose": "Locate the local capacity ceiling of the kq-bench harness on the north-star execution-time distribution by scaling producers, workers, worker concurrency, and ready partitions.", + "scenario": { + "id": "north-star-steady-v1", + "version": 1 + }, + "profile": "ci-smoke", + "command": "kq-bench run --scenario north-star-steady-v1 --profile ci-smoke --set ; see result.json points[].command", + "environment": { + "architecture": "arm64", + "ci_provider": "", + "cpu_count": 12, + "go_version": "go version go1.27.0 darwin/arm64", + "kafka_version": "apache/kafka:4.3.1", + "memory_bytes": 22118400, + "operating_system": "Darwin", + "python_version": "3.13.12", + "runner": "rays-MacBook-Pro.local", + "docker_vm_cpus": 12, + "docker_vm_memory_bytes": 8216776704 + }, + "topology": { + "ready_partitions_sweep": [ + 4, + 16, + 24, + 32, + 40 + ], + "retry_partitions": 4, + "producers_sweep": [ + 4, + 6, + 8, + 10, + 15, + 16, + 20 + ], + "workers_sweep": [ + 4, + 8, + 12, + 15, + 18, + 24 + ], + "worker_concurrency_sweep": [ + 100, + 250, + 500, + 1000, + 2000 + ], + "movers": 1 + }, + "workload": { + "arrival": { + "target_rps_sweep": [ + 500, + 3000, + 5000, + 8000, + 12000, + 16000, + 20000 + ], + "duration_seconds_sweep": [ + 20, + 30, + 40, + 45 + ] + }, + "execution_time_ms": { + "average": 250, + "p95": 475, + "p99": 646 + }, + "random_seed": 1, + "failures": { + "rate": 0, + "mode": "none", + "duration_ms": 0 + }, + "warmup": { + "tasks": 10, + "id_base": 4294967296 + } + }, + "notes": "Every point used revision 33c95bf74b5c with no source changes. The harness was invoked from source (PYTHONPATH=bench) because the configured package index rejected the hatchling build dependency. Host and process CPU peaks were sampled every 2 s with top, docker stats, and ps; ps CPU is a decaying average and is approximate." +}