Gate startup performance across target sizes

Signed-off-by: irl-dan <97565471+irl-dan@users.noreply.github.com>

Author: irl-dan

.github/workflows/performance.yml

@@ -0,0 +1,43 @@
+name: Startup performance
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: performance-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  startup:
+    name: macOS startup envelope
+    runs-on: macos-15
+    timeout-minutes: 20
+    env:
+      MARGIN_STARTUP_RUNS: 10
+      MARGIN_STARTUP_WARMUPS: 3
+      MARGIN_STARTUP_VISIBLE_P95_LIMIT_MS: 1000
+      MARGIN_STARTUP_READY_P95_LIMIT_MS: 3000
+    steps:
+      - name: Check out source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Build release app
+        run: make release
+
+      - name: Measure startup matrix
+        run: Scripts/benchmark-startup-matrix.sh
+
+      - name: Upload measurements
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: startup-performance-${{ github.sha }}
+          path: build/benchmarks/startup-matrix/
+          if-no-files-found: error
+          retention-days: 90

Benchmarks/performance/LaunchBenchmark.swift

@@ -27,6 +27,8 @@ private struct Options {
     var warmups = 3
     var settleMilliseconds = 250
     var timeoutMilliseconds = 5_000
+    var visibleP95LimitMilliseconds: Double?
+    var readyP95LimitMilliseconds: Double?
 
     init(arguments: [String]) throws {
         var index = 0
@@ -48,14 +50,18 @@ private struct Options {
             case "--warmups": warmups = try Self.nonnegativeInteger(try value(after: option), option: option)
             case "--settle-ms": settleMilliseconds = try Self.nonnegativeInteger(try value(after: option), option: option)
             case "--timeout-ms": timeoutMilliseconds = try Self.positiveInteger(try value(after: option), option: option)
+            case "--visible-p95-limit-ms":
+                visibleP95LimitMilliseconds = try Self.positiveDouble(try value(after: option), option: option)
+            case "--ready-p95-limit-ms":
+                readyP95LimitMilliseconds = try Self.positiveDouble(try value(after: option), option: option)
             default: throw BenchmarkFailure.usage("Unknown option \(option).")
             }
             index += 2
         }
 
         guard !appPath.isEmpty, !documentPath.isEmpty, !outputPath.isEmpty else {
             throw BenchmarkFailure.usage(
-                "usage: launch-benchmark --app APP --document FILE --output JSON [--runs N] [--warmups N]"
+                "usage: launch-benchmark --app APP --document FILE_OR_DIRECTORY --output JSON [--runs N] [--warmups N] [--visible-p95-limit-ms N] [--ready-p95-limit-ms N]"
             )
         }
     }
@@ -73,10 +79,18 @@ private struct Options {
         }
         return value
     }
+
+    private static func positiveDouble(_ raw: String, option: String) throws -> Double {
+        guard let value = Double(raw), value.isFinite, value > 0 else {
+            throw BenchmarkFailure.invalidValue("\(option) expects a positive number.")
+        }
+        return value
+    }
 }
 
 private struct Sample {
     let launchMilliseconds: Double
+    let readyMilliseconds: Double
     let residentMemoryMiB: Double
 }
 
@@ -137,15 +151,18 @@ private struct BenchmarkReport: Encodable {
         let settleMilliseconds: Int
         let timeoutMilliseconds: Int
         let documentPath: String
+        let visibleP95LimitMilliseconds: Double?
+        let readyP95LimitMilliseconds: Double?
     }
 
-    let schema = "urn:margin:performance:v1"
+    let schema = "urn:margin:performance:v2"
     let measuredAt: String
     let measurement: String
     let system: System
     let artifact: Artifact
     let settings: Settings
     let launchMilliseconds: Statistics
+    let readyMilliseconds: Statistics
     let residentMemoryMiB: Statistics
 }
 
@@ -204,6 +221,11 @@ private func measureOnce(options: Options) throws -> Sample {
     let process = Process()
     process.executableURL = executable
     process.arguments = [options.documentPath]
+    let readyURL = FileManager.default.temporaryDirectory
+        .appendingPathComponent("margin-ready-\(UUID().uuidString)", isDirectory: false)
+    process.environment = ProcessInfo.processInfo.environment.merging([
+        "MARGIN_BENCHMARK_READY_FILE": readyURL.path,
+    ]) { _, benchmarkValue in benchmarkValue }
     process.standardOutput = FileHandle.nullDevice
     process.standardError = FileHandle.nullDevice
 
@@ -213,20 +235,34 @@ private func measureOnce(options: Options) throws -> Sample {
     } catch {
         throw BenchmarkFailure.launch("Could not launch \(executable.path): \(error.localizedDescription)")
     }
-    defer { stop(process) }
+    defer {
+        stop(process)
+        try? FileManager.default.removeItem(at: readyURL)
+    }
 
     let timeout = UInt64(options.timeoutMilliseconds) * 1_000_000
     var visibleAt: UInt64?
+    var readyAt: UInt64?
     while process.isRunning && DispatchTime.now().uptimeNanoseconds - start < timeout {
-        if isWindowVisible(for: process.processIdentifier) {
+        if visibleAt == nil, isWindowVisible(for: process.processIdentifier) {
             visibleAt = DispatchTime.now().uptimeNanoseconds
+        }
+        if readyAt == nil, FileManager.default.fileExists(atPath: readyURL.path) {
+            readyAt = DispatchTime.now().uptimeNanoseconds
+        }
+        if visibleAt != nil, readyAt != nil {
             break
         }
         usleep(1_000)
     }
     guard let visibleAt else {
         throw BenchmarkFailure.timeout(options.timeoutMilliseconds)
     }
+    guard let readyAt else {
+        throw BenchmarkFailure.measurement(
+            "Margin showed a window but did not finish loading the benchmark target within \(options.timeoutMilliseconds) ms."
+        )
+    }
 
     if options.settleMilliseconds > 0 {
         usleep(useconds_t(options.settleMilliseconds * 1_000))
@@ -237,6 +273,7 @@ private func measureOnce(options: Options) throws -> Sample {
     let residentMiB = try residentMemoryMiB(for: process.processIdentifier)
     return Sample(
         launchMilliseconds: Double(visibleAt - start) / 1_000_000,
+        readyMilliseconds: Double(readyAt - start) / 1_000_000,
         residentMemoryMiB: residentMiB
     )
 }
@@ -302,7 +339,7 @@ do {
     formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
     let report = BenchmarkReport(
         measuredAt: formatter.string(from: Date()),
-        measurement: "Warm direct-executable spawn to first on-screen layer-0 app window; RSS sampled after the settle interval.",
+        measurement: "Warm direct-executable spawn to first on-screen layer-0 app window and to completion of target loading and initial Markdown presentation; RSS sampled after the settle interval.",
         system: .init(
             operatingSystem: ProcessInfo.processInfo.operatingSystemVersionString,
             architecture: architecture,
@@ -319,9 +356,12 @@ do {
             warmupRuns: options.warmups,
             settleMilliseconds: options.settleMilliseconds,
             timeoutMilliseconds: options.timeoutMilliseconds,
-            documentPath: URL(fileURLWithPath: options.documentPath).standardizedFileURL.path
+            documentPath: URL(fileURLWithPath: options.documentPath).standardizedFileURL.path,
+            visibleP95LimitMilliseconds: options.visibleP95LimitMilliseconds,
+            readyP95LimitMilliseconds: options.readyP95LimitMilliseconds
         ),
         launchMilliseconds: Statistics(samples.map(\.launchMilliseconds)),
+        readyMilliseconds: Statistics(samples.map(\.readyMilliseconds)),
         residentMemoryMiB: Statistics(samples.map(\.residentMemoryMiB))
     )
 
@@ -336,6 +376,17 @@ do {
     )
     try data.write(to: outputURL, options: .atomic)
     FileHandle.standardOutput.write(data)
+
+    if let limit = options.visibleP95LimitMilliseconds, report.launchMilliseconds.p95 > limit {
+        throw BenchmarkFailure.measurement(
+            "Visible-window p95 \(report.launchMilliseconds.p95) ms exceeds the \(limit) ms limit."
+        )
+    }
+    if let limit = options.readyP95LimitMilliseconds, report.readyMilliseconds.p95 > limit {
+        throw BenchmarkFailure.measurement(
+            "Target-ready p95 \(report.readyMilliseconds.p95) ms exceeds the \(limit) ms limit."
+        )
+    }
 } catch {
     let message = (error as? BenchmarkFailure)?.description ?? error.localizedDescription
     fputs("launch-benchmark: \(message)\n", stderr)

CONTRIBUTING.md

@@ -64,7 +64,8 @@ opening a pull request.
 | AppKit UI or app behavior | `make test`, `make smoke` |
 | Comment or collaboration protocol | `make test`, `make test-linux`, `make eval` |
 | Directory transactions or agent workflow | `make eval-collaboration` |
-| Launch-path or packaging behavior | `make smoke`, `make benchmark`, `make package` |
+| Launch-path behavior | `make smoke`, `make benchmark-matrix` |
+| Packaging behavior | `make package` |
 | MarginBench implementation or public evidence | `make marginbench-test`, `make marginbench-preflight` |
 
 `make eval-preflight` and paid or remote benchmark runs are not routine pull

Docs/PERFORMANCE.md

@@ -11,10 +11,19 @@ make benchmark
 The default run performs three cache warm-ups and fifteen measured launches of the packaged release app. It records:
 
 - launch latency from spawning `Margin.app/Contents/MacOS/Margin` until its first on-screen layer-0 window is visible;
+- target-ready latency until the requested Markdown has been decoded, installed in the real editor with its initial syntax presentation, and made editable when permissions allow;
 - resident set size (RSS) 250 ms after that window appears;
 - logical app-bundle bytes and main-executable bytes.
 
-Results are written to `build/benchmarks/performance.json`. The launch value is an external window-visible proxy: it is reproducible without modifying production code, but it does not claim to measure the final compositor frame or every asynchronous document-loading operation.
+Results are written to `build/benchmarks/performance.json`. The visible-window value is an external proxy and does not claim to measure the final compositor frame. The target-ready value comes from an environment-gated marker emitted after initial document presentation and editor activation; ordinary app launches do not create the marker or perform benchmark I/O.
+
+Run the release-sized file and directory matrix with:
+
+```sh
+make benchmark-matrix
+```
+
+That command covers 4 KiB, 1 MiB, and 5 MiB Markdown files plus directories with 100 and 10,000 entries. A directory result includes root enumeration, initial `README.md` selection, document decoding, and initial Markdown presentation. The checked-in local p95 limits are 500 ms for the first visible window and 1,250 ms for target readiness.
 
 Override the sample size or input when investigating regressions:
 
@@ -27,6 +36,14 @@ make benchmark
 
 For clean comparisons, close other Margin instances, use the same hardware and power state, and compare release bundles built with the same macOS toolchain.
 
+## What GitHub Actions can establish
+
+The `Startup performance` workflow runs the complete matrix on one `macos-15` runner, publishes the table in the job summary, retains the raw JSON samples for 90 days, and fails when any case exceeds a 1,000 ms visible-window p95 or 3,000 ms target-ready p95. Keeping all cases in one job makes size comparisons share the same host and machine state.
+
+Those intentionally loose hosted-runner limits are regression alarms, not end-user guarantees. GitHub currently documents the standard public `macos-15` runner as an arm64 M1 VM with three CPUs and 7 GB RAM, but hosted runner load, virtualization, image revisions, thermal state, storage caches, and end-user hardware are outside Margin's control. A hardware-specific service-level guarantee would require a controlled physical or dedicated self-hosted Mac, a pinned OS and toolchain, repeated cold and warm samples, and ongoing calibration. See GitHub's [hosted-runner reference](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) and [self-hosted runner guidance](https://docs.github.com/en/actions/concepts/runners/self-hosted-runners).
+
+Margin therefore does **not** promise universal sub-200 ms startup. The current AppKit baseline and Margin measurements do not support that claim. The public statement is narrower: the release is continuously checked against the documented CI envelope, and the reference-machine measurements below are reproducible evidence for a specific system.
+
 ## Current reference and framework floor
 
 The following reference run used an Apple M1 Max on macOS 26.2, the release bundle, three warm-ups, fifteen measured launches, and the same external window-visible/RSS probe for every row.
@@ -132,3 +149,22 @@ Across 100 fresh CLI processes after three warmups, ordinary help measured
 / 8.477 ms**. The corresponding structured outputs are 12,716 and 19,797
 bytes, both below their hard bounds and independent of filesystem or network
 state.
+
+## v0.4.0 startup matrix baseline
+
+The v0.4.0 release candidate completed the new matrix on an Apple M1 Max with
+macOS 26.2. Every case used three warm-ups and ten measured direct launches.
+
+| Case | Visible median | Visible p95 | Ready median | Ready p95 |
+|---|---:|---:|---:|---:|
+| 4 KiB Markdown file | 323.903 ms | 403.134 ms | 354.102 ms | 442.868 ms |
+| 1 MiB Markdown file | 317.479 ms | 343.534 ms | 470.048 ms | 496.853 ms |
+| 5 MiB Markdown file | 319.724 ms | 366.418 ms | 920.244 ms | 983.171 ms |
+| Directory with 100 entries | 323.939 ms | 332.526 ms | 513.149 ms | 521.528 ms |
+| Directory with 10,000 entries | 320.360 ms | 327.736 ms | 1,030.775 ms | 1,060.048 ms |
+
+The nearly flat visible-window results confirm that file decoding and directory
+enumeration do not block the first window. Target readiness scales with the
+requested work and remains within the checked-in 1,250 ms local p95 envelope
+for this matrix. These warm-launch figures support “a few hundred milliseconds
+to a visible window” on the reference system, not a sub-200 ms claim.

Makefile

@@ -6,7 +6,7 @@ BUILD_SCRATCH_PATH ?= $(SCRATCH_ROOT)/command-line-tools
 TEST_SCRATCH_PATH ?= $(SCRATCH_ROOT)/xcode-15.4
 OUTPUT_DIR ?= $(PROJECT_DIR)/build
 
-.PHONY: debug test test-linux check-version marginbench-test marginbench-audit marginbench-preflight marginbench-control-preflight marginbench-neutral-preflight marginbench-remote-plan marginbench-linux-binary marginbench-package release package package-linux installer install smoke benchmark eval eval-preflight eval-collaboration clean
+.PHONY: debug test test-linux check-version marginbench-test marginbench-audit marginbench-preflight marginbench-control-preflight marginbench-neutral-preflight marginbench-remote-plan marginbench-linux-binary marginbench-package release package package-linux installer install smoke benchmark benchmark-matrix eval eval-preflight eval-collaboration clean
 
 check-version:
 	"$(PROJECT_DIR)/Scripts/check-version.sh"
@@ -155,6 +155,12 @@ benchmark: release
 	MARGIN_PERFORMANCE_SCRATCH_PATH="$(PROJECT_DIR)/.build/performance" \
 		"$(PROJECT_DIR)/Scripts/benchmark-performance.sh"
 
+benchmark-matrix: release
+	DEVELOPER_DIR="$(BUILD_DEVELOPER_DIR)" \
+	MARGIN_BUILD_OUTPUT_DIR="$(OUTPUT_DIR)" \
+	MARGIN_PERFORMANCE_SCRATCH_PATH="$(PROJECT_DIR)/.build/performance" \
+		"$(PROJECT_DIR)/Scripts/benchmark-startup-matrix.sh"
+
 eval: release
 	PYTHONDONTWRITEBYTECODE=1 \
 		python3 -m unittest discover -s "$(PROJECT_DIR)/Evals/cli/tests" -p 'test_*.py'

ROADMAP.md

@@ -20,6 +20,10 @@ directions rather than promises or dates.
   Apple Developer credentials.
 - Improve first-run installation and update guidance without adding a daemon or
   background service.
+- Investigate whether the real editable-window path can be brought below 200 ms
+  on the reference Mac. The v0.4.0 baseline is intentionally documented and
+  gated above that target; do not advertise sub-200 ms until repeatable evidence
+  supports it.
 - Add a restrained product screenshot and social preview to the public project.
 - Expand private MarginBench validation before making any general performance
   or agent-effectiveness claim.

Scripts/benchmark-performance.sh

@@ -11,9 +11,11 @@ SETTLE_MS="${MARGIN_BENCHMARK_SETTLE_MS:-250}"
 TIMEOUT_MS="${MARGIN_BENCHMARK_TIMEOUT_MS:-5000}"
 SCRATCH_PATH="${MARGIN_PERFORMANCE_SCRATCH_PATH:-$PROJECT_DIR/.build/performance}"
 RESULTS_PATH="${MARGIN_BENCHMARK_RESULTS:-$OUTPUT_DIR/benchmarks/performance.json}"
+VISIBLE_P95_LIMIT_MS="${MARGIN_BENCHMARK_VISIBLE_P95_LIMIT_MS:-}"
+READY_P95_LIMIT_MS="${MARGIN_BENCHMARK_READY_P95_LIMIT_MS:-}"
 
-if [[ ! -x "$APP_BUNDLE/Contents/MacOS/Margin" || ! -f "$DOCUMENT" ]]; then
-    print -u2 "Margin.app or benchmark document is missing. Run make release first."
+if [[ ! -x "$APP_BUNDLE/Contents/MacOS/Margin" || ! -e "$DOCUMENT" ]]; then
+    print -u2 "Margin.app or benchmark target is missing. Run make release first."
     exit 66
 fi
 
@@ -26,14 +28,19 @@ mkdir -p "$SCRATCH_PATH" "${RESULTS_PATH:h}"
 RUNNER="$SCRATCH_PATH/launch-benchmark"
 xcrun swiftc -O "$PROJECT_DIR/Benchmarks/performance/LaunchBenchmark.swift" -o "$RUNNER"
 
-"$RUNNER" \
-    --app "$APP_BUNDLE" \
-    --document "$DOCUMENT" \
-    --runs "$RUNS" \
-    --warmups "$WARMUPS" \
-    --settle-ms "$SETTLE_MS" \
-    --timeout-ms "$TIMEOUT_MS" \
+ARGS=(
+    --app "$APP_BUNDLE"
+    --document "$DOCUMENT"
+    --runs "$RUNS"
+    --warmups "$WARMUPS"
+    --settle-ms "$SETTLE_MS"
+    --timeout-ms "$TIMEOUT_MS"
     --output "$RESULTS_PATH"
+)
+[[ -n "$VISIBLE_P95_LIMIT_MS" ]] && ARGS+=(--visible-p95-limit-ms "$VISIBLE_P95_LIMIT_MS")
+[[ -n "$READY_P95_LIMIT_MS" ]] && ARGS+=(--ready-p95-limit-ms "$READY_P95_LIMIT_MS")
+
+"$RUNNER" "${ARGS[@]}"
 
 print "Results: $RESULTS_PATH"
 print "Allocated bundle size (KiB): $(/usr/bin/du -sk "$APP_BUNDLE" | /usr/bin/awk '{print $1}')"