diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea5fe25..1fc81cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,11 +76,12 @@ jobs: run: go run -p 2 golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./cmd/agent-heavy ./cmd/agent-remote ./cmd/queue-tool macos-tests: - name: macOS tests (${{ matrix.os }}) + name: macOS tests (${{ matrix.os }}, ${{ matrix.scope }}) strategy: fail-fast: false matrix: os: [macos-15, macos-15-intel] + scope: [core, https] runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: @@ -92,14 +93,15 @@ jobs: go-version-file: go.mod cache: true - name: Go unit, ownership, SQLite, real daemon, and HTTPS regressions - run: bash scripts/test-go-runtime.sh + run: bash scripts/test-go-runtime.sh '${{ matrix.scope }}' tooling-tests: - name: Tooling tests (${{ matrix.os }}) + name: Tooling tests (${{ matrix.os }}, shard ${{ matrix.shard }}/2) strategy: fail-fast: false matrix: os: [macos-15, macos-15-intel] + shard: [1, 2] runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: @@ -110,8 +112,8 @@ jobs: with: go-version-file: go.mod cache: true - - name: Full Go tooling and installer regressions - run: go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m + - name: Complete tooling and installer regression shard + run: bash scripts/test-go-tooling.sh '${{ matrix.shard }}' native-tests: name: Native tests (${{ matrix.os }}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e216679..6c0bfde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,10 @@ agent-heavy run --label "Agent Queue validation" -- bash scripts/validate.sh The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI runs separate required runtime, tooling/installer, and native test matrices on Apple Silicon and Intel Macs. Each keeps its 20-minute job budget; Go tests retain race detection, two package/test workers and the 12-minute test timeout. The runtime adapter dynamically enumerates the original Go scope and excludes only `internal/tooling`, which runs completely in its own matrix. Tooling emits per-test timings for timeout diagnosis. All matrices remain required by `ci-required`. +The tooling matrix uses two deterministic shards on each architecture, invoked with `bash scripts/test-go-tooling.sh 1` and `bash scripts/test-go-tooling.sh 2`. Their complete discovered test, example and fuzz-seed inventory is partitioned exactly once; malformed or incomplete enumeration fails before testing. Every shard remains required, with the same race detection, worker limits and time budgets. + +The runtime matrix independently runs `bash scripts/test-go-runtime.sh core` and `bash scripts/test-go-runtime.sh https` on both architectures. HTTPS runs the complete `internal/remote` package on its own runner so real cost-14 bcrypt does not compete with the backend suite on that host. Core includes every other dynamically discovered runtime package, including any future remote child packages. The two partitions cover the original runtime scope exactly once; absent required packages, malformed inventory and invalid selectors fail before testing. Omitting the selector retains the complete original runtime scope. Both partitions keep the existing race detection, workers, test timeout and job budget; both remain required by `ci-required`. + For focused development, these commands use only this checkout and private fixtures: ```sh diff --git a/docs/handoffs/bundle-owned-services.md b/docs/handoffs/bundle-owned-services.md index d21af07..02eafb5 100644 --- a/docs/handoffs/bundle-owned-services.md +++ b/docs/handoffs/bundle-owned-services.md @@ -8,10 +8,40 @@ Original incident evidence and the build-58/build-94 comparison are recorded in ## PR integration and release authorization -The user subsequently authorized pushing this implementation, resolving conflicts, getting [PR 41](https://github.com/Dunky13/agent-job-queue/pull/41) green, merging it and completing the signed release. Integration preserves main commit `5ab7db8` and its approved-repair and unattended-release changes alongside explicit-only setup. Integrated validation is running through queue job 2161 and the PR's required Apple Silicon/Intel CI matrices. The completed local evidence below records the earlier implementation snapshot; it does not certify the integrated or published artifact. +The user subsequently authorized pushing this implementation, resolving conflicts, getting [PR 41](https://github.com/Dunky13/agent-job-queue/pull/41) green, merging it and completing the signed release. Integration preserves main commit `5ab7db8` and its approved-repair and unattended-release changes alongside explicit-only setup. Queue job 2161 passed the complete integrated local validation pipeline, including Go race tests, native tests, universal packaging, actual private installed-runtime acceptance and DMG verification. Required Apple Silicon/Intel hosted CI and signed publication remain separate gates. The detailed local evidence below records the earlier implementation snapshot. Published build 118 uses source `5ab7db8`, which does not contain this PR's native Service Management adapter or bundle-owned service packaging. Its reported SentinelOne alert therefore does not qualify the changed implementation. The authoritative detection and a protected retest of the eventual signed candidate are still required to establish SentinelOne acceptance. +### Merged-source certification recovery + +PR 41 was squash-merged as `c3a690b27d593d97165ed7b5275c04f1b46ffcce`. Its file tree exactly matches the integrated branch, but GitHub selected author `Marc Went ` while the commit carries maintainer certifications for `Marc Went `. The GitHub signature is valid. Main CI run `40397632997`, commit-policy job `121834785287`, correctly failed with `missing author-matching DCO sign-off`. + +The published commit and failed witness are retained. Re-running the same SHA cannot correct that identity mismatch. Successor PR 42 establishes a new, properly signed and certified source commit without rewriting main or changing the DCO verifier. It records this recovery and the CI scheduling repair below without changing product code. Its squash merge must explicitly select author email `marc@went.io`, retain exactly one matching maintainer trailer, and pass post-merge author/signature verification. Required CI and the existing distribution pipeline must then qualify that new source before any release is claimed. Build 118 remains immutable and is not replaced. + +### Intel tooling budget recovery + +PR 42 run `40398515127`, Intel tooling job `121837837447`, reached the unchanged 12-minute package timeout after 720.078 seconds. The active stapling-retry parent had run for 89 seconds; its current subcase had run for one second. Other parallel tests were still queued. This was an aggregate workload failure, and `ci-required` correctly failed too. The failed job log remains the witness; no assertion failure or green result is inferred from it. + +The full tooling package now runs as two deterministic inventory shards on each architecture. Enumeration includes every top-level test, example and fuzz seed, rejects malformed, duplicate, partial or empty inventory, and assigns each executable name exactly once. Top-level selection keeps all subtests. Benchmarks remain excluded as in the prior command. Both shards retain race detection, two package/test workers, the 12-minute package timeout and the 20-minute job limit. The required aggregate job still requires the entire tooling matrix. Regression fixtures verify coverage, deterministic membership, exact flags, failure propagation and refusal before testing invalid input. No product code, release trust check or timeout is relaxed. + +Queue job 2162 passed the actual two-shard runner locally: 85 and 84 top-level tests, in 203.965 and 175.657 seconds respectively, with race detection and the original worker limits. Hosted results for both architectures remain required. + +### Packaged setup refusal recovery + +Universal-installer job `121857014460` in run `40403882465` passed packaging, signatures, Homebrew validation and packaged payload installation, then the app-bundle fixture hit the installer's safe kernel busy-lock refusal. This fixture performs a second installation while its real private resource controller is running. Its original one-attempt assumption differed from the explicit-refusal contract already exercised by the removal fixture. + +The package fixture now permits at most three attempts within its original single 30-second deadline. Only `EWOULDBLOCK` or `EAGAIN` with an absent prior pending marker, unchanged installation records and wrappers (including inode, mode and content), and an intact native-call history with no mutation can be retried. Any accepted mutation, retained pending state, uncertain witness, non-kernel error or cancellation fails immediately. Regression cases cover each refusal boundary and the attempt limit. Production locking and installation behavior are unchanged; the failed hosted witness is retained. + +Queue job 2163 passed lint, the focused refusal race tests, complete packaged installer acceptance and DMG verification with exit 0. The packaged payload and app-bundle tests passed in 9.09 and 6.62 seconds; the actual private outer-installer lifecycle passed under race in 67.459 seconds. The disk image detached successfully. These checks reuse the unchanged development artifact; native lifecycle calls remain injected, and fresh hosted CI is still required. + +### Intel HTTPS workload isolation + +A later notification for the same `05c1758` run identified a separate failure in Intel runtime job `121857014412`. The cold 16-request asset burst took 35.93 seconds; fourteen waiting requests received HTTP 429. The single-comparison cache assertion did not fail. The private lifecycle then hit its existing 30-second request deadlines, and an enabled reconfigure was killed at its 35-second fixture limit. Backend and remote packages shared that job, taking 562.952 and 406.182 seconds respectively. These are retained failed assertions, not a package timeout or a successful run. + +Race-instrumented cost-14 bcrypt competing with the backend workload is the leading explanation, not a proven Intel measurement. Isolated local queue job 2165 passed those same two race tests: the cold burst took 9.86 seconds and the lifecycle took 92.62 seconds in total. The local Mac is Apple Silicon; Intel execution is unavailable here and cannot be inferred from that result. + +CI now requires separate core and HTTPS runtime partitions on each architecture. HTTPS runs the entire remote package on a separate runner; core dynamically includes every other original runtime package except the separately tested tooling root. Future remote children and similarly named packages remain in core. The default `all` mode retains the previous complete runtime scope. Parser and workflow regressions verify exact coverage, unchanged race/worker/time limits, refusal before testing invalid or incomplete inventories, and error propagation. The partition assertions failed before the adapter change and passed afterward. Authentication configuration, bcrypt cost 14, cache behavior, admission bounds and service deadlines are unchanged. Fresh isolated hosted Intel results remain the acceptance gate. + Requirements carried into implementation: - Installation and upgrades perform setup explicitly. Opening an incomplete app shows instructions without rewriting files or automatically retrying recovery. diff --git a/internal/backend/install_test.go b/internal/backend/install_test.go index 5b23dba..f598537 100644 --- a/internal/backend/install_test.go +++ b/internal/backend/install_test.go @@ -2196,7 +2196,11 @@ func installTestPackaged(t *testing.T, appBundle bool) { runPackaged := func() error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - return install.execute(ctx) + retries, err := runPackagedSetup(ctx, f, install.execute) + if retries > 0 { + t.Logf("Packaged setup observed %d proven unaccepted busy refusals within its original deadline", retries) + } + return err } if err := runPackaged(); err != nil { t.Fatal(err) diff --git a/internal/backend/managed_runtime_fixture_test.go b/internal/backend/managed_runtime_fixture_test.go index 8b695bf..a381c3a 100644 --- a/internal/backend/managed_runtime_fixture_test.go +++ b/internal/backend/managed_runtime_fixture_test.go @@ -2,16 +2,108 @@ package backend import ( "context" + "crypto/sha256" "errors" + "fmt" "os" "os/exec" "path/filepath" + "reflect" "slices" "syscall" "testing" "time" + + "github.com/Dunky13/agent-job-queue/internal/appservices" ) +type packagedSetupFile struct { + present bool + mode os.FileMode + inode uint64 + digest [32]byte +} + +func packagedSetupWitness(install *installer) (map[string]packagedSetupFile, error) { + witness := map[string]packagedSetupFile{} + for _, filename := range []string{ + filepath.Join(install.root, "installation.json"), filepath.Join(install.root, "build-info.json"), + filepath.Join(install.root, "install-in-progress.json"), filepath.Join(install.root, "publication-custody.json"), + filepath.Join(install.home, ".local/bin/agent-heavy"), filepath.Join(install.home, ".local/bin/agent-remote"), + } { + info, err := os.Lstat(filename) + if errors.Is(err, os.ErrNotExist) { + witness[filename] = packagedSetupFile{} + continue + } + if err != nil { + return nil, err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil, errors.New("unsupported private setup file identity") + } + raw, err := appservices.ReadFile(install.home, filename, 4*1024*1024, false) + if err != nil { + return nil, err + } + witness[filename] = packagedSetupFile{true, info.Mode(), stat.Ino, sha256.Sum256(raw)} + } + return witness, nil +} + +// Only a kernel busy refusal with unchanged installation custody and no native +// mutation can be retried. The caller's original deadline spans every attempt. +func runPackagedSetup(ctx context.Context, fixture *installFixture, execute func(context.Context) error) (int, error) { + for attempt := 0; attempt < 3; attempt++ { + if err := ctx.Err(); err != nil { + return attempt, err + } + before, err := packagedSetupWitness(fixture.installer) + if err != nil { + return attempt, err + } + beforeCalls := make([][]string, len(fixture.calls)) + for index, args := range fixture.calls { + beforeCalls[index] = slices.Clone(args) + } + calls := len(beforeCalls) + err = execute(ctx) + if err == nil || !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) { + return attempt, err + } + after, witnessError := packagedSetupWitness(fixture.installer) + pending := filepath.Join(fixture.installer.root, "install-in-progress.json") + if witnessError != nil || before[pending].present || !reflect.DeepEqual(before, after) { + return attempt, fmt.Errorf("busy setup changed or retained mutation custody; no retry: %w", errors.Join(err, witnessError)) + } + if len(fixture.calls) < calls { + return attempt, fmt.Errorf("busy setup lost its action witness; no retry: %w", err) + } + for index, args := range beforeCalls { + if !slices.Equal(args, fixture.calls[index]) { + return attempt, fmt.Errorf("busy setup changed its action witness; no retry: %w", err) + } + } + for _, args := range fixture.calls[calls:] { + if len(args) > 2 && args[1] == "--service-management" && args[2] != "status" || + len(args) > 1 && args[1] == "--prepare-app-replacement" || + len(args) > 1 && args[0] == "launchctl" && args[1] != "print" { + return attempt, fmt.Errorf("busy setup accepted a service mutation; no retry: %w", err) + } + } + if attempt == 2 { + return attempt, fmt.Errorf("packaged setup remained busy within its original attempt bounds: %w", err) + } + select { + case <-ctx.Done(): + return attempt, ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } + panic("unreachable packaged setup attempts") +} + // Only native lifecycle and OS ownership readback are simulated. Readiness and // drain snapshots come from the actual packaged canonical helper and private DB. // The parent test owns all processes, independently of installer command scopes. diff --git a/internal/backend/packaged_setup_retry_test.go b/internal/backend/packaged_setup_retry_test.go new file mode 100644 index 0000000..d4cf826 --- /dev/null +++ b/internal/backend/packaged_setup_retry_test.go @@ -0,0 +1,112 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestPackagedSetupRetriesOnlyUnacceptedKernelBusy(t *testing.T) { + f := installTestFixture(t) + f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "status", "queue"}) + ctx := context.Background() + calls := 0 + retries, err := runPackagedSetup(ctx, f, func(received context.Context) error { + if received != ctx { + t.Fatal("retry changed the caller's operation budget") + } + calls++ + if calls == 1 { + f.calls = append(f.calls, []string{"launchctl", "print", "private-read"}) + return fmt.Errorf("private scheduling lock: %w", syscall.EWOULDBLOCK) + } + return nil + }) + if err != nil || retries != 1 || calls != 2 { + t.Fatal("safe refusal did not resume exactly once", retries, calls, err) + } +} + +func TestPackagedSetupNeverRetriesMutationOrUncertainHistory(t *testing.T) { + for _, phase := range []string{"existing-pending", "new-pending", "metadata", "wrapper", "same-content-replacement", "native", "menu", "launchctl", "lost-history", "changed-history"} { + t.Run(phase, func(t *testing.T) { + f := installTestFixture(t) + i := f.installer + f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "status", "queue"}) + pending := filepath.Join(i.root, "install-in-progress.json") + metadata := filepath.Join(i.root, "installation.json") + if phase == "existing-pending" { + integrationTestWrite(t, pending, []byte("retained pending identity")) + } + if phase == "same-content-replacement" { + integrationTestWrite(t, metadata, []byte("retained identity")) + } + calls := 0 + retries, err := runPackagedSetup(context.Background(), f, func(context.Context) error { + calls++ + switch phase { + case "new-pending": + integrationTestWrite(t, pending, []byte("accepted identity")) + case "metadata": + integrationTestWrite(t, metadata, []byte("changed identity")) + case "wrapper": + integrationTestWrite(t, filepath.Join(i.home, ".local/bin/agent-heavy"), []byte("changed wrapper")) + case "same-content-replacement": + if err := os.Rename(metadata, metadata+".retained"); err != nil { + t.Fatal(err) + } + integrationTestWrite(t, metadata, []byte("retained identity")) + case "native": + f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "unregister", "resources"}) + case "menu": + f.calls = append(f.calls, []string{"AgentQueue", "--prepare-app-replacement", i.app}) + case "launchctl": + f.calls = append(f.calls, []string{"launchctl", "bootout", "private-service"}) + case "lost-history": + f.calls = nil + case "changed-history": + f.calls[0][1] = "changed" + } + return syscall.EAGAIN + }) + if err == nil || retries != 0 || calls != 1 { + t.Fatal("accepted or uncertain operation was repeated", retries, calls, err) + } + }) + } +} + +func TestPackagedSetupPreservesFailureBoundsAndCancellation(t *testing.T) { + for _, failure := range []error{io.EOF, errors.New("Another installer or queue change is running")} { + f := installTestFixture(t) + calls := 0 + _, err := runPackagedSetup(context.Background(), f, func(context.Context) error { calls++; return failure }) + if !errors.Is(err, failure) || calls != 1 { + t.Fatal("non-kernel failure was repeated or replaced", calls, err) + } + } + f := installTestFixture(t) + calls := 0 + retries, err := runPackagedSetup(context.Background(), f, func(context.Context) error { calls++; return syscall.EWOULDBLOCK }) + if err == nil || calls != 3 || retries != 2 { + t.Fatal("busy attempts escaped their bound", retries, calls, err) + } + for _, alreadyCancelled := range []bool{false, true} { + f := installTestFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + calls := 0 + if alreadyCancelled { + cancel() + } + _, err := runPackagedSetup(ctx, f, func(context.Context) error { calls++; cancel(); return syscall.EWOULDBLOCK }) + if !errors.Is(err, context.Canceled) || calls > 1 || alreadyCancelled && calls != 0 { + t.Fatal("cancelled setup was retried", calls, err) + } + } +} diff --git a/internal/remote/admission_test.go b/internal/remote/admission_test.go index 098fb81..3ae66b1 100644 --- a/internal/remote/admission_test.go +++ b/internal/remote/admission_test.go @@ -80,6 +80,7 @@ func TestAuthAdmissionRealCaddyCacheAndDistinctGuesses(t *testing.T) { // assets and cached dashboard polling consume only that single comparison. var assets sync.WaitGroup start := make(chan struct{}) + started := time.Now() for i := range limits.Burst { assets.Add(1) go func() { @@ -94,6 +95,7 @@ func TestAuthAdmissionRealCaddyCacheAndDistinctGuesses(t *testing.T) { } close(start) assets.Wait() + t.Logf("Cold asset burst: %d requests, %d real bcrypt comparisons, %s elapsed, %s admission wait limit", limits.Burst, counter.calls.Load(), time.Since(started), gatewayResponseTimeout) if counter.calls.Load() != 1 { t.Fatal("cached polls recomputed bcrypt", counter.calls.Load()) } diff --git a/internal/remote/control_test.go b/internal/remote/control_test.go index b0d7340..2efa091 100644 --- a/internal/remote/control_test.go +++ b/internal/remote/control_test.go @@ -669,9 +669,10 @@ func runRemoteLifecycleAcceptance(t *testing.T, legacyExecutable string) { if credential != "" { request.SetBasicAuth(username, credential) } + started := time.Now() response, err := client.Do(request) if err != nil { - t.Fatal(err) + t.Fatalf("gateway request failed after %s within %s response budget: %v", time.Since(started), requestBudget, err) } defer response.Body.Close() data, _ := io.ReadAll(response.Body) diff --git a/internal/tooling/go_runtime_test.go b/internal/tooling/go_runtime_test.go index 1366406..765cf4b 100644 --- a/internal/tooling/go_runtime_test.go +++ b/internal/tooling/go_runtime_test.go @@ -14,7 +14,7 @@ import ( const goRuntimeModule = "github.com/Dunky13/agent-job-queue" -func goRuntimeFixture(t *testing.T, packages string, listStatus, testStatus string) (string, error, string) { +func goRuntimeFixture(t *testing.T, packages string, listStatus, testStatus string, selection ...string) (string, error, string) { t.Helper() root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "packages"), []byte(packages), 0600); err != nil { @@ -38,7 +38,7 @@ esac`) } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - cmd := exec.CommandContext(ctx, "/bin/bash", script) + cmd := exec.CommandContext(ctx, "/bin/bash", append([]string{script}, selection...)...) // Invocation from elsewhere must still enumerate the repository's packages. cmd.Dir = root cmd.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), @@ -72,6 +72,9 @@ func TestGoRuntimePartitionsCompleteScopeWithExactFlags(t *testing.T) { goRuntimeModule + "/remote/static", goRuntimeModule + "/cmd", goRuntimeModule + "/internal", + goRuntimeModule + "/internal/remote", + goRuntimeModule + "/internal/remote/child", + goRuntimeModule + "/internal/remote-extra", } root, err, output := goRuntimeFixture(t, strings.Join(packages, "\n")+"\n", "0", "0") if err != nil { @@ -92,7 +95,7 @@ func TestGoRuntimePartitionsCompleteScopeWithExactFlags(t *testing.T) { t.Fatalf("enumerated in %q, want repository %q", cwd, repository) } want := []string{"test", "-p", "2", "-parallel", "2", "-race"} - for _, index := range []int{0, 1, 3, 4, 5, 6, 7, 8, 9} { + for _, index := range []int{0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} { want = append(want, packages[index]) } want = append(want, "-timeout", "12m") @@ -101,10 +104,74 @@ func TestGoRuntimePartitionsCompleteScopeWithExactFlags(t *testing.T) { } } +func TestGoRuntimeCoreAndHTTPSCoverOriginalScopeExactlyOnce(t *testing.T) { + packages := []string{ + goRuntimeModule + "/internal/tooling", goRuntimeModule + "/internal/backend", + goRuntimeModule + "/internal/remote", goRuntimeModule + "/internal/remote/child", + goRuntimeModule + "/internal/remote-extra", goRuntimeModule + "/cmd/new-command", + } + selected := map[string]int{} + for _, suite := range []struct { + name string + want []string + }{ + {"core", []string{packages[1], packages[3], packages[4], packages[5]}}, + {"https", []string{packages[2]}}, + } { + root, err, output := goRuntimeFixture(t, strings.Join(packages, "\n"), "0", "0", suite.name) + if err != nil { + t.Fatalf("%s: %v: %s", suite.name, err, output) + } + want := append([]string{"test", "-p", "2", "-parallel", "2", "-race"}, suite.want...) + want = append(want, "-timeout", "12m") + if got := goRuntimeArguments(t, root, "test"); !reflect.DeepEqual(got, want) { + t.Fatalf("%s changed scope or flags: got %q, want %q", suite.name, got, want) + } + for _, name := range suite.want { + selected[name]++ + } + } + for _, name := range packages[1:] { + if selected[name] != 1 { + t.Fatal("original runtime scope omitted or duplicated", name, selected[name]) + } + } +} + +func TestGoRuntimeRejectsInvalidSelectionBeforeEnumeration(t *testing.T) { + for _, selection := range [][]string{{""}, {"unknown"}, {"--run=Nothing"}, {"core", "https"}} { + root, err, output := goRuntimeFixture(t, "unused", "0", "0", selection...) + if err == nil || !strings.Contains(output, "Usage:") { + t.Fatalf("invalid selection accepted: %q: %v: %s", selection, err, output) + } + for _, file := range []string{"list-args", "test-args"} { + if _, err := os.Stat(filepath.Join(root, file)); !os.IsNotExist(err) { + t.Fatal("invalid selection reached Go", selection, file, err) + } + } + } +} + +func TestGoRuntimeRejectsIncompleteSelectedScope(t *testing.T) { + for _, suite := range []struct{ name, packages string }{ + {"core", goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/remote"}, + {"https", goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/backend"}, + } { + root, err, output := goRuntimeFixture(t, suite.packages, "0", "0", suite.name) + if err == nil || !strings.Contains(output, "nonempty runtime suite") { + t.Fatalf("incomplete %s accepted: %v: %s", suite.name, err, output) + } + if _, err := os.Stat(filepath.Join(root, "test-args")); !os.IsNotExist(err) { + t.Fatal("incomplete partition reached tests", suite.name, err) + } + } +} + func TestGoRuntimeFailsBeforeTestingInvalidEnumeration(t *testing.T) { runtime := goRuntimeModule + "/internal/backend" tooling := goRuntimeModule + "/internal/tooling" - valid := runtime + "\n" + tooling + "\n" + https := goRuntimeModule + "/internal/remote" + valid := runtime + "\n" + tooling + "\n" + https + "\n" for _, test := range []struct { name, packages, status, diagnostic string }{ @@ -112,9 +179,11 @@ func TestGoRuntimeFailsBeforeTestingInvalidEnumeration(t *testing.T) { {"partial enumeration error", valid, "23", "fixture enumeration failed"}, {"empty", "", "0", "Unexpected Go package"}, {"missing tooling", runtime, "0", "requires tooling"}, + {"missing HTTPS", runtime + "\n" + tooling, "0", "requires tooling and HTTPS"}, {"empty runtime", tooling, "0", "nonempty runtime suite"}, {"duplicate runtime", valid + runtime, "0", "Duplicate Go package"}, {"duplicate tooling", valid + tooling, "0", "Duplicate Go package"}, + {"duplicate HTTPS", valid + https, "0", "Duplicate Go package"}, {"blank record", runtime + "\n\n" + tooling, "0", "Unexpected Go package"}, {"option", valid + "-run=Nothing", "0", "Unexpected Go package"}, {"foreign module", valid + "example.com/other/internal/tooling", "0", "Unexpected Go package"}, @@ -146,13 +215,15 @@ func TestGoRuntimeFailsBeforeTestingInvalidEnumeration(t *testing.T) { } func TestGoRuntimePreservesTestFailure(t *testing.T) { - packages := goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/backend\n" - root, err, output := goRuntimeFixture(t, packages, "0", "37") - var exit *exec.ExitError - if !errors.As(err, &exit) || exit.ExitCode() != 37 { - t.Fatalf("go test failure lost: %v: %s", err, output) - } - if got := goRuntimeArguments(t, root, "test"); len(got) == 0 || got[0] != "test" { - t.Fatalf("runtime test was not invoked: %q", got) + packages := goRuntimeModule + "/internal/tooling\n" + goRuntimeModule + "/internal/backend\n" + goRuntimeModule + "/internal/remote\n" + for _, suite := range []string{"all", "core", "https"} { + root, err, output := goRuntimeFixture(t, packages, "0", "37", suite) + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 37 { + t.Fatalf("%s go test failure lost: %v: %s", suite, err, output) + } + if got := goRuntimeArguments(t, root, "test"); len(got) == 0 || got[0] != "test" { + t.Fatalf("%s runtime test was not invoked: %q", suite, got) + } } } diff --git a/internal/tooling/go_tooling_test.go b/internal/tooling/go_tooling_test.go new file mode 100644 index 0000000..70c36da --- /dev/null +++ b/internal/tooling/go_tooling_test.go @@ -0,0 +1,144 @@ +package tooling + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + "time" +) + +func toolingShardFixture(t *testing.T, inventory string, shard []string, listStatus, testStatus string) (string, error, string) { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "inventory"), []byte(inventory), 0600); err != nil { + t.Fatal(err) + } + testTool(t, root, "go", ` +[ "$1" = test ] || exit 99 +case " $* " in +*" -list "*) + pwd > "$TOOLING_SHARD_ROOT/cwd" + printf '%s\0' "$@" > "$TOOLING_SHARD_ROOT/list-args" + cat "$TOOLING_SHARD_ROOT/inventory" + exit "$TOOLING_SHARD_LIST_STATUS";; +*) + printf '%s\0' "$@" > "$TOOLING_SHARD_ROOT/test-args" + exit "$TOOLING_SHARD_TEST_STATUS";; +esac`) + script, err := filepath.Abs("../../scripts/test-go-tooling.sh") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "/bin/bash", append([]string{script}, shard...)...) + command.Dir = root + command.Env = append(os.Environ(), "PATH="+root+string(os.PathListSeparator)+os.Getenv("PATH"), + "TOOLING_SHARD_ROOT="+root, "TOOLING_SHARD_LIST_STATUS="+listStatus, "TOOLING_SHARD_TEST_STATUS="+testStatus) + output, err := command.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("tooling shard did not finish: %v: %s", ctx.Err(), output) + } + return root, err, string(output) +} + +const toolingInventorySummary = "ok\tgithub.com/Dunky13/agent-job-queue/internal/tooling\t0.001s\n" + +func TestGoToolingShardsCoverCompleteInventoryExactlyOnce(t *testing.T) { + names := []string{"TestZulu", "ExampleOutput", "FuzzInputs", "TestAlpha", "TestFuturePackageAddition"} + inventory := strings.Join(names, "\n") + "\nBenchmarkIgnored\n" + toolingInventorySummary + coverage := map[string]int{} + for _, shard := range []string{"1", "2"} { + root, err, output := toolingShardFixture(t, inventory, []string{shard}, "0", "0") + if err != nil { + t.Fatalf("shard %s failed: %v: %s", shard, err, output) + } + if got, want := goRuntimeArguments(t, root, "list"), []string{"test", "-p", "2", "./internal/tooling", "-list", "."}; !reflect.DeepEqual(got, want) { + t.Fatalf("inventory scope: %q", got) + } + args := goRuntimeArguments(t, root, "test") + if len(args) != 12 || !reflect.DeepEqual(args[:9], []string{"test", "-p", "2", "-parallel", "2", "-race", "-v", "./internal/tooling", "-run"}) || !reflect.DeepEqual(args[10:], []string{"-timeout", "12m"}) { + t.Fatalf("race, package, worker or timeout contract changed: %q", args) + } + pattern, err := regexp.Compile(args[9]) + if err != nil || !strings.HasPrefix(args[9], "^(") || !strings.HasSuffix(args[9], ")$") { + t.Fatal("shard selector is not an exact top-level inventory", args[9], err) + } + if pattern.MatchString("BenchmarkIgnored") || pattern.MatchString("TestAlphaUnlistedSuffix") { + t.Fatal("selector escaped its exact inventory") + } + for _, name := range names { + if pattern.MatchString(name) { + coverage[name]++ + } + } + reversed := []string{"TestFuturePackageAddition", "TestAlpha", "FuzzInputs", "ExampleOutput", "TestZulu"} + other, err, output := toolingShardFixture(t, strings.Join(reversed, "\n")+"\n"+toolingInventorySummary, []string{shard}, "0", "0") + if err != nil || goRuntimeArguments(t, other, "test")[9] != args[9] { + t.Fatal("inventory order changed deterministic membership", err, output) + } + cwd, err := os.ReadFile(filepath.Join(root, "cwd")) + repository, pathError := filepath.Abs("../..") + if err != nil || pathError != nil || strings.TrimSpace(string(cwd)) != repository { + t.Fatal("enumeration did not use the repository", err, pathError) + } + } + for _, name := range names { + if coverage[name] != 1 { + t.Fatalf("%s appears in %d shards, want exactly one", name, coverage[name]) + } + } +} + +func TestGoToolingShardsRejectIncompleteOrUnsafeInventory(t *testing.T) { + valid := "TestAlpha\nTestBeta\n" + toolingInventorySummary + for _, entry := range []struct{ name, inventory, status string }{ + {"list error", "", "23"}, + {"partial list error", valid, "23"}, + {"empty", "", "0"}, + {"missing summary", "TestAlpha\nTestBeta\n", "0"}, + {"duplicate", "TestAlpha\nTestAlpha\n" + toolingInventorySummary, "0"}, + {"malformed name", "TestAlpha -run=Other\nTestBeta\n" + toolingInventorySummary, "0"}, + {"blank record", "TestAlpha\n\nTestBeta\n" + toolingInventorySummary, "0"}, + {"foreign package", "TestAlpha\nTestBeta\nok example.com/foreign 0.001s\n", "0"}, + {"duplicate summary", valid + toolingInventorySummary, "0"}, + {"test after summary", valid + "TestUnaccounted\n", "0"}, + {"benchmark only", "BenchmarkOnly\n" + toolingInventorySummary, "0"}, + {"empty second shard", "TestOnly\n" + toolingInventorySummary, "0"}, + } { + t.Run(entry.name, func(t *testing.T) { + root, err, output := toolingShardFixture(t, entry.inventory, []string{"1"}, entry.status, "0") + var exited *exec.ExitError + if !errors.As(err, &exited) || entry.status == "23" && exited.ExitCode() != 23 { + t.Fatalf("invalid enumeration did not fail closed: %v: %s", err, output) + } + if _, err := os.Stat(filepath.Join(root, "test-args")); !os.IsNotExist(err) { + t.Fatal("partial tooling tests ran after invalid inventory", err) + } + }) + } +} + +func TestGoToolingShardsPreserveFailureAndRejectInvalidSelection(t *testing.T) { + valid := "TestAlpha\nTestBeta\n" + toolingInventorySummary + _, err, output := toolingShardFixture(t, valid, []string{"1"}, "0", "37") + var exited *exec.ExitError + if !errors.As(err, &exited) || exited.ExitCode() != 37 { + t.Fatal("test failure status was lost", err, output) + } + for _, args := range [][]string{nil, {"0"}, {"3"}, {"01"}, {"1", "2"}, {"-run=Other"}} { + root, err, output := toolingShardFixture(t, valid, args, "0", "0") + if err == nil { + t.Fatal("invalid shard accepted", args, output) + } + if _, err := os.Stat(filepath.Join(root, "list-args")); !os.IsNotExist(err) { + t.Fatal("invalid shard reached enumeration", args, err) + } + } +} diff --git a/internal/tooling/workflow_test.go b/internal/tooling/workflow_test.go index ebcf381..807321b 100644 --- a/internal/tooling/workflow_test.go +++ b/internal/tooling/workflow_test.go @@ -30,7 +30,9 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { Strategy struct { FailFast *bool `yaml:"fail-fast"` Matrix struct { - OS []string `yaml:"os"` + OS []string `yaml:"os"` + Shard []int `yaml:"shard"` + Scope []string `yaml:"scope"` } `yaml:"matrix"` } `yaml:"strategy"` Steps []validationStep `yaml:"steps"` @@ -40,8 +42,8 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { t.Fatal(err) } suites := map[string]string{ - "macos-tests": "bash scripts/test-go-runtime.sh", - "tooling-tests": "go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m", + "macos-tests": "bash scripts/test-go-runtime.sh '${{ matrix.scope }}'", + "tooling-tests": "bash scripts/test-go-tooling.sh '${{ matrix.shard }}'", "native-tests": "bash scripts/test-native.sh", } for name, command := range suites { @@ -52,6 +54,12 @@ func TestMacOSGoAndNativeValidationHaveIndependentBudgets(t *testing.T) { if job.Timeout != 20 || job.ContinueOnError || job.If != "" || job.RunsOn != "${{ matrix.os }}" || job.Strategy.FailFast == nil || *job.Strategy.FailFast || !reflect.DeepEqual(job.Strategy.Matrix.OS, []string{"macos-15", "macos-15-intel"}) { t.Fatalf("%s changed the existing budget or architecture requirements", name) } + if name == "tooling-tests" && !reflect.DeepEqual(job.Strategy.Matrix.Shard, []int{1, 2}) { + t.Fatal("both tooling shards must run on every required architecture") + } + if name == "macos-tests" && !reflect.DeepEqual(job.Strategy.Matrix.Scope, []string{"core", "https"}) { + t.Fatal("complete core and HTTPS partitions must run independently on every required architecture") + } found := false for _, step := range job.Steps { if step.ContinueOnError || step.If != "" { diff --git a/scripts/test-go-runtime.sh b/scripts/test-go-runtime.sh index 31fe15d..4576f7f 100755 --- a/scripts/test-go-runtime.sh +++ b/scripts/test-go-runtime.sh @@ -2,13 +2,25 @@ set -euo pipefail cd "$(dirname "$0")/.." +if [[ $# -gt 1 ]]; then + printf 'Usage: %s [all|core|https]\n' "$0" >&2 + exit 1 +fi +scope=${1-all} +case "$scope" in + all|core|https) ;; + *) printf 'Usage: %s [all|core|https]\n' "$0" >&2; exit 1 ;; +esac + # Keep the original CI scope dynamic, and stop before testing on list errors. packages=$(go list -p 2 ./cmd/... ./internal/... ./packaging/homebrew ./remote/static) module=github.com/Dunky13/agent-job-queue tooling="$module/internal/tooling" +https="$module/internal/remote" runtime=() seen=() found_tooling=false +found_https=false while IFS= read -r package || [[ -n "$package" ]]; do case "$package" in "$module"/cmd|"$module"/cmd/*|"$module"/internal|"$module"/internal/*|"$module"/packaging/homebrew|"$module"/remote/static) ;; @@ -30,12 +42,19 @@ while IFS= read -r package || [[ -n "$package" ]]; do seen+=("$package") if [[ "$package" == "$tooling" ]]; then found_tooling=true + elif [[ "$package" == "$https" ]]; then + found_https=true + if [[ "$scope" != core ]]; then + runtime+=("$package") + fi else - runtime+=("$package") + if [[ "$scope" != https ]]; then + runtime+=("$package") + fi fi done <<< "$packages" -if [[ "$found_tooling" != true || ${#runtime[@]} -eq 0 ]]; then - printf 'Go package enumeration requires tooling and a nonempty runtime suite.\n' >&2 +if [[ "$found_tooling" != true || "$found_https" != true || ${#runtime[@]} -eq 0 ]]; then + printf 'Go package enumeration requires tooling and HTTPS packages and a nonempty runtime suite.\n' >&2 exit 1 fi go test -p 2 -parallel 2 -race "${runtime[@]}" -timeout 12m diff --git a/scripts/test-go-tooling.sh b/scripts/test-go-tooling.sh new file mode 100755 index 0000000..11e2307 --- /dev/null +++ b/scripts/test-go-tooling.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")/.." +[[ $# == 1 && ( $1 == 1 || $1 == 2 ) ]] || { + printf 'Usage: test-go-tooling.sh SHARD (1 or 2)\n' >&2 + exit 1 +} +shard=$1 +package=github.com/Dunky13/agent-job-queue/internal/tooling +# Discover every top-level test, example and fuzz seed. Enumeration failures or +# malformed/duplicate records must stop before any partial suite is executed. +inventory=$(go test -p 2 ./internal/tooling -list .) +tests=() +seen=() +summary=false +while IFS= read -r name || [[ -n "$name" ]]; do + if [[ $name =~ ^(Test|Example|Fuzz|Benchmark)[a-zA-Z0-9_]*$ ]]; then + [[ $summary == false ]] || { printf 'Test after enumeration summary.\n' >&2; exit 1; } + for previous in ${seen[@]+"${seen[@]}"}; do + [[ $previous != "$name" ]] || { printf 'Duplicate Go test: %s\n' "$name" >&2; exit 1; } + done + seen+=("$name") + case "$name" in Benchmark*) ;; *) tests+=("$name");; esac + else + read -r status listed detail <<< "$name" + if [[ $summary != false || $status != ok || $listed != "$package" || -z $detail ]]; then + printf 'Unexpected Go test inventory record: %s\n' "$name" >&2 + exit 1 + fi + summary=true + fi +done <<< "$inventory" +[[ $summary == true && ${#tests[@]} -ge 2 ]] || { + printf 'Tooling enumeration requires a complete nonempty two-shard suite.\n' >&2 + exit 1 +} +sorted=$(printf '%s\n' "${tests[@]}" | LC_ALL=C sort) +selected=() +index=0 +while IFS= read -r name; do + if (( index % 2 + 1 == shard )); then selected+=("$name"); fi + index=$((index + 1)) +done <<< "$sorted" +[[ ${#selected[@]} -gt 0 ]] || { printf 'Empty tooling shard.\n' >&2; exit 1; } +printf 'Tooling shard %s/2: %s of %s top-level tests\n' "$shard" "${#selected[@]}" "${#tests[@]}" +printf ' %s\n' "${selected[@]}" +pattern=$(IFS='|'; printf '%s' "${selected[*]}") +go test -p 2 -parallel 2 -race -v ./internal/tooling -run "^($pattern)$" -timeout 12m