fix(ci): qualify managed services release

.github/workflows/ci.yml

@@ -76,11 +76,12 @@ jobs:
         run: go run -p 2 golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./cmd/agent-heavy ./cmd/agent-remote ./cmd/queue-tool
 
   macos-tests:
-    name: macOS tests (${{ matrix.os }})
+    name: macOS tests (${{ matrix.os }}, ${{ matrix.scope }})
     strategy:
       fail-fast: false
       matrix:
         os: [macos-15, macos-15-intel]
+        scope: [core, https]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     steps:
@@ -92,14 +93,15 @@ jobs:
           go-version-file: go.mod
           cache: true
       - name: Go unit, ownership, SQLite, real daemon, and HTTPS regressions
-        run: bash scripts/test-go-runtime.sh
+        run: bash scripts/test-go-runtime.sh '${{ matrix.scope }}'
 
   tooling-tests:
-    name: Tooling tests (${{ matrix.os }})
+    name: Tooling tests (${{ matrix.os }}, shard ${{ matrix.shard }}/2)
     strategy:
       fail-fast: false
       matrix:
         os: [macos-15, macos-15-intel]
+        shard: [1, 2]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     steps:
@@ -110,8 +112,8 @@ jobs:
         with:
           go-version-file: go.mod
           cache: true
-      - name: Full Go tooling and installer regressions
-        run: go test -p 2 -parallel 2 -race -v ./internal/tooling -timeout 12m
+      - name: Complete tooling and installer regression shard
+        run: bash scripts/test-go-tooling.sh '${{ matrix.shard }}'
 
   native-tests:
     name: Native tests (${{ matrix.os }})

CONTRIBUTING.md

@@ -17,6 +17,10 @@ agent-heavy run --label "Agent Queue validation" -- bash scripts/validate.sh
 
 The batch checks source portability, Go formatting and vet, known reachable Go vulnerabilities, shell and workflow lint, Go race tests, standalone trusted tap tests, Linux compilation of CI tools, browser schema and interaction tests, native regressions, and the universal installer. Hosted CI runs separate required runtime, tooling/installer, and native test matrices on Apple Silicon and Intel Macs. Each keeps its 20-minute job budget; Go tests retain race detection, two package/test workers and the 12-minute test timeout. The runtime adapter dynamically enumerates the original Go scope and excludes only `internal/tooling`, which runs completely in its own matrix. Tooling emits per-test timings for timeout diagnosis. All matrices remain required by `ci-required`.
 
+The tooling matrix uses two deterministic shards on each architecture, invoked with `bash scripts/test-go-tooling.sh 1` and `bash scripts/test-go-tooling.sh 2`. Their complete discovered test, example and fuzz-seed inventory is partitioned exactly once; malformed or incomplete enumeration fails before testing. Every shard remains required, with the same race detection, worker limits and time budgets.
+
+The runtime matrix independently runs `bash scripts/test-go-runtime.sh core` and `bash scripts/test-go-runtime.sh https` on both architectures. HTTPS runs the complete `internal/remote` package on its own runner so real cost-14 bcrypt does not compete with the backend suite on that host. Core includes every other dynamically discovered runtime package, including any future remote child packages. The two partitions cover the original runtime scope exactly once; absent required packages, malformed inventory and invalid selectors fail before testing. Omitting the selector retains the complete original runtime scope. Both partitions keep the existing race detection, workers, test timeout and job budget; both remain required by `ci-required`.
+
 For focused development, these commands use only this checkout and private fixtures:
 
 ```sh

docs/handoffs/bundle-owned-services.md

@@ -8,10 +8,40 @@ Original incident evidence and the build-58/build-94 comparison are recorded in
 
 ## PR integration and release authorization
 
-The user subsequently authorized pushing this implementation, resolving conflicts, getting [PR 41](https://github.com/Dunky13/agent-job-queue/pull/41) green, merging it and completing the signed release. Integration preserves main commit `5ab7db8` and its approved-repair and unattended-release changes alongside explicit-only setup. Integrated validation is running through queue job 2161 and the PR's required Apple Silicon/Intel CI matrices. The completed local evidence below records the earlier implementation snapshot; it does not certify the integrated or published artifact.
+The user subsequently authorized pushing this implementation, resolving conflicts, getting [PR 41](https://github.com/Dunky13/agent-job-queue/pull/41) green, merging it and completing the signed release. Integration preserves main commit `5ab7db8` and its approved-repair and unattended-release changes alongside explicit-only setup. Queue job 2161 passed the complete integrated local validation pipeline, including Go race tests, native tests, universal packaging, actual private installed-runtime acceptance and DMG verification. Required Apple Silicon/Intel hosted CI and signed publication remain separate gates. The detailed local evidence below records the earlier implementation snapshot.
 
 Published build 118 uses source `5ab7db8`, which does not contain this PR's native Service Management adapter or bundle-owned service packaging. Its reported SentinelOne alert therefore does not qualify the changed implementation. The authoritative detection and a protected retest of the eventual signed candidate are still required to establish SentinelOne acceptance.
 
+### Merged-source certification recovery
+
+PR 41 was squash-merged as `c3a690b27d593d97165ed7b5275c04f1b46ffcce`. Its file tree exactly matches the integrated branch, but GitHub selected author `Marc Went <Dunky13@users.noreply.github.com>` while the commit carries maintainer certifications for `Marc Went <marc@went.io>`. The GitHub signature is valid. Main CI run `40397632997`, commit-policy job `121834785287`, correctly failed with `missing author-matching DCO sign-off`.
+
+The published commit and failed witness are retained. Re-running the same SHA cannot correct that identity mismatch. Successor PR 42 establishes a new, properly signed and certified source commit without rewriting main or changing the DCO verifier. It records this recovery and the CI scheduling repair below without changing product code. Its squash merge must explicitly select author email `marc@went.io`, retain exactly one matching maintainer trailer, and pass post-merge author/signature verification. Required CI and the existing distribution pipeline must then qualify that new source before any release is claimed. Build 118 remains immutable and is not replaced.
+
+### Intel tooling budget recovery
+
+PR 42 run `40398515127`, Intel tooling job `121837837447`, reached the unchanged 12-minute package timeout after 720.078 seconds. The active stapling-retry parent had run for 89 seconds; its current subcase had run for one second. Other parallel tests were still queued. This was an aggregate workload failure, and `ci-required` correctly failed too. The failed job log remains the witness; no assertion failure or green result is inferred from it.
+
+The full tooling package now runs as two deterministic inventory shards on each architecture. Enumeration includes every top-level test, example and fuzz seed, rejects malformed, duplicate, partial or empty inventory, and assigns each executable name exactly once. Top-level selection keeps all subtests. Benchmarks remain excluded as in the prior command. Both shards retain race detection, two package/test workers, the 12-minute package timeout and the 20-minute job limit. The required aggregate job still requires the entire tooling matrix. Regression fixtures verify coverage, deterministic membership, exact flags, failure propagation and refusal before testing invalid input. No product code, release trust check or timeout is relaxed.
+
+Queue job 2162 passed the actual two-shard runner locally: 85 and 84 top-level tests, in 203.965 and 175.657 seconds respectively, with race detection and the original worker limits. Hosted results for both architectures remain required.
+
+### Packaged setup refusal recovery
+
+Universal-installer job `121857014460` in run `40403882465` passed packaging, signatures, Homebrew validation and packaged payload installation, then the app-bundle fixture hit the installer's safe kernel busy-lock refusal. This fixture performs a second installation while its real private resource controller is running. Its original one-attempt assumption differed from the explicit-refusal contract already exercised by the removal fixture.
+
+The package fixture now permits at most three attempts within its original single 30-second deadline. Only `EWOULDBLOCK` or `EAGAIN` with an absent prior pending marker, unchanged installation records and wrappers (including inode, mode and content), and an intact native-call history with no mutation can be retried. Any accepted mutation, retained pending state, uncertain witness, non-kernel error or cancellation fails immediately. Regression cases cover each refusal boundary and the attempt limit. Production locking and installation behavior are unchanged; the failed hosted witness is retained.
+
+Queue job 2163 passed lint, the focused refusal race tests, complete packaged installer acceptance and DMG verification with exit 0. The packaged payload and app-bundle tests passed in 9.09 and 6.62 seconds; the actual private outer-installer lifecycle passed under race in 67.459 seconds. The disk image detached successfully. These checks reuse the unchanged development artifact; native lifecycle calls remain injected, and fresh hosted CI is still required.
+
+### Intel HTTPS workload isolation
+
+A later notification for the same `05c1758` run identified a separate failure in Intel runtime job `121857014412`. The cold 16-request asset burst took 35.93 seconds; fourteen waiting requests received HTTP 429. The single-comparison cache assertion did not fail. The private lifecycle then hit its existing 30-second request deadlines, and an enabled reconfigure was killed at its 35-second fixture limit. Backend and remote packages shared that job, taking 562.952 and 406.182 seconds respectively. These are retained failed assertions, not a package timeout or a successful run.
+
+Race-instrumented cost-14 bcrypt competing with the backend workload is the leading explanation, not a proven Intel measurement. Isolated local queue job 2165 passed those same two race tests: the cold burst took 9.86 seconds and the lifecycle took 92.62 seconds in total. The local Mac is Apple Silicon; Intel execution is unavailable here and cannot be inferred from that result.
+
+CI now requires separate core and HTTPS runtime partitions on each architecture. HTTPS runs the entire remote package on a separate runner; core dynamically includes every other original runtime package except the separately tested tooling root. Future remote children and similarly named packages remain in core. The default `all` mode retains the previous complete runtime scope. Parser and workflow regressions verify exact coverage, unchanged race/worker/time limits, refusal before testing invalid or incomplete inventories, and error propagation. The partition assertions failed before the adapter change and passed afterward. Authentication configuration, bcrypt cost 14, cache behavior, admission bounds and service deadlines are unchanged. Fresh isolated hosted Intel results remain the acceptance gate.
+
 Requirements carried into implementation:
 
 - Installation and upgrades perform setup explicitly. Opening an incomplete app shows instructions without rewriting files or automatically retrying recovery.

internal/backend/install_test.go

@@ -2196,7 +2196,11 @@ func installTestPackaged(t *testing.T, appBundle bool) {
 	runPackaged := func() error {
 		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
-		return install.execute(ctx)
+		retries, err := runPackagedSetup(ctx, f, install.execute)
+		if retries > 0 {
+			t.Logf("Packaged setup observed %d proven unaccepted busy refusals within its original deadline", retries)
+		}
+		return err
 	}
 	if err := runPackaged(); err != nil {
 		t.Fatal(err)

internal/backend/managed_runtime_fixture_test.go

@@ -2,16 +2,108 @@ package backend
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"reflect"
 	"slices"
 	"syscall"
 	"testing"
 	"time"
+
+	"github.com/Dunky13/agent-job-queue/internal/appservices"
 )
 
+type packagedSetupFile struct {
+	present bool
+	mode    os.FileMode
+	inode   uint64
+	digest  [32]byte
+}
+
+func packagedSetupWitness(install *installer) (map[string]packagedSetupFile, error) {
+	witness := map[string]packagedSetupFile{}
+	for _, filename := range []string{
+		filepath.Join(install.root, "installation.json"), filepath.Join(install.root, "build-info.json"),
+		filepath.Join(install.root, "install-in-progress.json"), filepath.Join(install.root, "publication-custody.json"),
+		filepath.Join(install.home, ".local/bin/agent-heavy"), filepath.Join(install.home, ".local/bin/agent-remote"),
+	} {
+		info, err := os.Lstat(filename)
+		if errors.Is(err, os.ErrNotExist) {
+			witness[filename] = packagedSetupFile{}
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		stat, ok := info.Sys().(*syscall.Stat_t)
+		if !ok {
+			return nil, errors.New("unsupported private setup file identity")
+		}
+		raw, err := appservices.ReadFile(install.home, filename, 4*1024*1024, false)
+		if err != nil {
+			return nil, err
+		}
+		witness[filename] = packagedSetupFile{true, info.Mode(), stat.Ino, sha256.Sum256(raw)}
+	}
+	return witness, nil
+}
+
+// Only a kernel busy refusal with unchanged installation custody and no native
+// mutation can be retried. The caller's original deadline spans every attempt.
+func runPackagedSetup(ctx context.Context, fixture *installFixture, execute func(context.Context) error) (int, error) {
+	for attempt := 0; attempt < 3; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return attempt, err
+		}
+		before, err := packagedSetupWitness(fixture.installer)
+		if err != nil {
+			return attempt, err
+		}
+		beforeCalls := make([][]string, len(fixture.calls))
+		for index, args := range fixture.calls {
+			beforeCalls[index] = slices.Clone(args)
+		}
+		calls := len(beforeCalls)
+		err = execute(ctx)
+		if err == nil || !errors.Is(err, syscall.EWOULDBLOCK) && !errors.Is(err, syscall.EAGAIN) {
+			return attempt, err
+		}
+		after, witnessError := packagedSetupWitness(fixture.installer)
+		pending := filepath.Join(fixture.installer.root, "install-in-progress.json")
+		if witnessError != nil || before[pending].present || !reflect.DeepEqual(before, after) {
+			return attempt, fmt.Errorf("busy setup changed or retained mutation custody; no retry: %w", errors.Join(err, witnessError))
+		}
+		if len(fixture.calls) < calls {
+			return attempt, fmt.Errorf("busy setup lost its action witness; no retry: %w", err)
+		}
+		for index, args := range beforeCalls {
+			if !slices.Equal(args, fixture.calls[index]) {
+				return attempt, fmt.Errorf("busy setup changed its action witness; no retry: %w", err)
+			}
+		}
+		for _, args := range fixture.calls[calls:] {
+			if len(args) > 2 && args[1] == "--service-management" && args[2] != "status" ||
+				len(args) > 1 && args[1] == "--prepare-app-replacement" ||
+				len(args) > 1 && args[0] == "launchctl" && args[1] != "print" {
+				return attempt, fmt.Errorf("busy setup accepted a service mutation; no retry: %w", err)
+			}
+		}
+		if attempt == 2 {
+			return attempt, fmt.Errorf("packaged setup remained busy within its original attempt bounds: %w", err)
+		}
+		select {
+		case <-ctx.Done():
+			return attempt, ctx.Err()
+		case <-time.After(25 * time.Millisecond):
+		}
+	}
+	panic("unreachable packaged setup attempts")
+}
+
 // Only native lifecycle and OS ownership readback are simulated. Readiness and
 // drain snapshots come from the actual packaged canonical helper and private DB.
 // The parent test owns all processes, independently of installer command scopes.

internal/backend/packaged_setup_retry_test.go

@@ -0,0 +1,112 @@
+package backend
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+func TestPackagedSetupRetriesOnlyUnacceptedKernelBusy(t *testing.T) {
+	f := installTestFixture(t)
+	f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "status", "queue"})
+	ctx := context.Background()
+	calls := 0
+	retries, err := runPackagedSetup(ctx, f, func(received context.Context) error {
+		if received != ctx {
+			t.Fatal("retry changed the caller's operation budget")
+		}
+		calls++
+		if calls == 1 {
+			f.calls = append(f.calls, []string{"launchctl", "print", "private-read"})
+			return fmt.Errorf("private scheduling lock: %w", syscall.EWOULDBLOCK)
+		}
+		return nil
+	})
+	if err != nil || retries != 1 || calls != 2 {
+		t.Fatal("safe refusal did not resume exactly once", retries, calls, err)
+	}
+}
+
+func TestPackagedSetupNeverRetriesMutationOrUncertainHistory(t *testing.T) {
+	for _, phase := range []string{"existing-pending", "new-pending", "metadata", "wrapper", "same-content-replacement", "native", "menu", "launchctl", "lost-history", "changed-history"} {
+		t.Run(phase, func(t *testing.T) {
+			f := installTestFixture(t)
+			i := f.installer
+			f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "status", "queue"})
+			pending := filepath.Join(i.root, "install-in-progress.json")
+			metadata := filepath.Join(i.root, "installation.json")
+			if phase == "existing-pending" {
+				integrationTestWrite(t, pending, []byte("retained pending identity"))
+			}
+			if phase == "same-content-replacement" {
+				integrationTestWrite(t, metadata, []byte("retained identity"))
+			}
+			calls := 0
+			retries, err := runPackagedSetup(context.Background(), f, func(context.Context) error {
+				calls++
+				switch phase {
+				case "new-pending":
+					integrationTestWrite(t, pending, []byte("accepted identity"))
+				case "metadata":
+					integrationTestWrite(t, metadata, []byte("changed identity"))
+				case "wrapper":
+					integrationTestWrite(t, filepath.Join(i.home, ".local/bin/agent-heavy"), []byte("changed wrapper"))
+				case "same-content-replacement":
+					if err := os.Rename(metadata, metadata+".retained"); err != nil {
+						t.Fatal(err)
+					}
+					integrationTestWrite(t, metadata, []byte("retained identity"))
+				case "native":
+					f.calls = append(f.calls, []string{"AgentQueue", "--service-management", "unregister", "resources"})
+				case "menu":
+					f.calls = append(f.calls, []string{"AgentQueue", "--prepare-app-replacement", i.app})
+				case "launchctl":
+					f.calls = append(f.calls, []string{"launchctl", "bootout", "private-service"})
+				case "lost-history":
+					f.calls = nil
+				case "changed-history":
+					f.calls[0][1] = "changed"
+				}
+				return syscall.EAGAIN
+			})
+			if err == nil || retries != 0 || calls != 1 {
+				t.Fatal("accepted or uncertain operation was repeated", retries, calls, err)
+			}
+		})
+	}
+}
+
+func TestPackagedSetupPreservesFailureBoundsAndCancellation(t *testing.T) {
+	for _, failure := range []error{io.EOF, errors.New("Another installer or queue change is running")} {
+		f := installTestFixture(t)
+		calls := 0
+		_, err := runPackagedSetup(context.Background(), f, func(context.Context) error { calls++; return failure })
+		if !errors.Is(err, failure) || calls != 1 {
+			t.Fatal("non-kernel failure was repeated or replaced", calls, err)
+		}
+	}
+	f := installTestFixture(t)
+	calls := 0
+	retries, err := runPackagedSetup(context.Background(), f, func(context.Context) error { calls++; return syscall.EWOULDBLOCK })
+	if err == nil || calls != 3 || retries != 2 {
+		t.Fatal("busy attempts escaped their bound", retries, calls, err)
+	}
+	for _, alreadyCancelled := range []bool{false, true} {
+		f := installTestFixture(t)
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		calls := 0
+		if alreadyCancelled {
+			cancel()
+		}
+		_, err := runPackagedSetup(ctx, f, func(context.Context) error { calls++; cancel(); return syscall.EWOULDBLOCK })
+		if !errors.Is(err, context.Canceled) || calls > 1 || alreadyCancelled && calls != 0 {
+			t.Fatal("cancelled setup was retried", calls, err)
+		}
+	}
+}