From 4d3993b6f9bdf0d80deef34757b427809bea5ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:21:38 +0200 Subject: [PATCH 01/10] dispatch: require worker ownership and bound admission waits Reserve Workflow waiting places in the shared admission domain, refuse unowned starts and withdraw undelivered messages. Fence late dispatches, preserve recovery accounting, and harden launch observation and process cleanup evidence. Co-Authored-By: GPT-6.1 Sol --- .claude-plugin/marketplace.json | 4 +- README.md | 20 +- docs/guide.md | 20 +- package-lock.json | 4 +- package.json | 2 +- plugins/relay/.claude-plugin/plugin.json | 2 +- plugins/relay/CHANGELOG.md | 9 + plugins/relay/agents/job.md | 2 +- plugins/relay/hooks/mod/register.js | 76 ++++-- plugins/relay/hooks/mod/register.test.ts | 43 ++++ plugins/relay/scripts/cli/gate.mjs | 49 ++++ plugins/relay/scripts/cli/message.mjs | 7 +- plugins/relay/scripts/cli/start.mjs | 18 +- plugins/relay/scripts/cli/task.mjs | 14 +- plugins/relay/scripts/companion.mjs | 4 + plugins/relay/scripts/lib/jobs/lease.mjs | 14 +- plugins/relay/scripts/lib/jobs/messages.mjs | 17 ++ plugins/relay/scripts/lib/jobs/view.mjs | 12 +- plugins/relay/scripts/lib/util/process.mjs | 30 ++- .../relay/scripts/lib/worker/admission.mjs | 126 ++++++++-- plugins/relay/scripts/lib/worker/delivery.mjs | 148 ++++++++---- plugins/relay/scripts/lib/worker/recovery.mjs | 37 ++- plugins/relay/scripts/lib/worker/registry.mjs | 38 ++- plugins/relay/skills/jobs/SKILL.md | 21 +- plugins/relay/skills/jobs/internals.md | 16 +- plugins/relay/skills/jobs/orchestration.md | 12 +- tests/admission-e2e.test.mjs | 50 ++-- tests/admission-surfaces.test.mjs | 6 +- tests/admission-worker.test.mjs | 11 +- tests/admission.test.mjs | 35 +++ tests/commands.test.mjs | 6 +- tests/execution-groups.test.mjs | 113 +++++++-- tests/gate.test.mjs | 221 ++++++++++++++++++ tests/harness-workflow.test.mjs | 58 +++++ tests/process.test.mjs | 12 + tests/queued.test.mjs | 9 +- tests/recovery.test.mjs | 8 +- tests/relay-delivery.test.mjs | 12 +- 38 files changed, 1050 insertions(+), 236 deletions(-) create mode 100644 plugins/relay/scripts/cli/gate.mjs create mode 100644 tests/gate.test.mjs diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6b3c05c9..cfc7b7cf 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,13 +5,13 @@ }, "metadata": { "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", - "version": "3.0.0-dev.24" + "version": "3.0.0-dev.25" }, "plugins": [ { "name": "relay", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "author": { "name": "Libar" }, diff --git a/README.md b/README.md index 8bc495d9..fbc6f0ea 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ The plugin's hooks module answers `/relay:status`, `/relay:result`, `/relay:canc tokens spent. The answer starts with the command as you typed it, then the companion's lines in a code block, as it prints them, then a job's report as Markdown, or the job list as a table. `claude -p "/relay:result "` exits with the companion's exit code (0 completed with a report; -3 failed, dead, completed without a report, or messages not yet run; 4 queued or running; 5 cancelled), with +3 failed, dead, completed without a report, or messages not yet run; 4 queued or running; 5 cancelled; 6 not started), with `--output-format text` or `json`; under `stream-json` Claude Code exits 0. These four run only when you type them. When you ask in plain words, Claude reads jobs with the companion in Bash, which a session in `default` permission mode asks you about, and cancels a job by sending its agent the @@ -350,14 +350,12 @@ Defaults use physical total memory `T` in bytes, `GiB = 2^30` bytes and `max`/`ultra` on other models, never enters this pool. Opaque defaults remain unknown; only identified selections are covered. - At most `Q = max(8, 2F)` resident waiting workers, including workers being spawned, are kept. - Worker creation holds `64 MiB` for 30 seconds against the same host/scope budget. A full - worker budget or insufficient memory leaves durable work without a worker and returns its - original job ID. Keep calling `wait`: the submitter/adopter's wait, message or relay-hook - loop retries worker creation at most once per admission poll (3 seconds by default). - A bare `start` can remain dormant until someone waits; there is no background queue service. - `status` only reads jobs: use `wait ` to adopt deferred work. A workerless job with no - spawn retry for 15 minutes stops deferring the stop-time review gate; it remains adoptable. - The limit bounds workers, not all submissions, native transcripts or waiting clients. + Worker creation holds `64 MiB` for 30 seconds against the same host/scope budget. + A job is accepted only once its own worker owns it; a job that cannot get a worker within + a minute is reported as not started (exit 6), and nothing runs later. There is no durable queue. + Workflow relays wait at spawn as queued until a waiting place frees, without a model turn + or a job waiter. Plain `relay:job` calls wait up to 30 seconds, then report capacity. + Use a Workflow for large batches. Accepted jobs keep running after the caller ends. Linux reads `MemAvailable` (with a labelled Node >=22 fallback). macOS uses the `kern.memorystatus_level` percentage of total memory, with a labelled `vm_stat` free+inactive @@ -436,8 +434,8 @@ the relay's final text against the job's result instead. The PostToolUse hook ca relay that `TaskStop` stopped. The hooks module (`hooks/mod/register.js`) adds an `agent.spawn` hook, which checks a `relay:job` prompt's header and refuses the spawn with the parser's message when the header is bad, and a `session.send` hook: when a message is sent to a relay, it ends the relay's -current wait, and the guard delivers the message at the relay's next call. The module only checks -headers and wakes a waiting relay: delivery runs on command hooks and the transcript. Its +current wait, and the guard delivers the message at the relay's next call. The module checks +headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript. Its `command.run` hooks answer `/relay:status`, `/relay:result`, `/relay:cancel` and `/relay:transfer`, and its status line counts the repository's running and dead jobs and owed messages ([design.md](docs/design.md#the-status-line)). diff --git a/docs/guide.md b/docs/guide.md index 672a8026..e9ad8788 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -165,14 +165,16 @@ A Stop during the launch handoff may prevent dispatch; a later waiter can retry ## When a job waits -A job's block can show `state: queued` and a `queued:` line explaining why it is waiting. This is pending work, -before it starts or between turns. While its relay agent or the companion's `wait` command is waiting, the job -starts by itself when the reason clears. Leave that call running; do not launch the job again. If you started a -job directly without an agent or a `wait` call, ask Claude to wait for that job by its id. `/relay:status` shows -progress but does not start waiting jobs. You can still cancel the job or send it a message by its id. +A job's block can show `state: queued` and a `queued:` line explaining why its live worker is +waiting, before a backend starts or between turns. Its worker starts the work when the reason clears, +even if you close the waiting call. Leave the job running; collect its result by id. -In Claude Code's Workflow panel a relay agent can look idle for minutes: it sits in relay's `wait` -call, which returns within seconds of the job ending. +When relay's start queue is full, Workflow agents wait in the panel as queued, using no model tokens +and creating no job waiter until a place frees. A plain `relay:job` call waits up to 30 seconds, then +reports that relay is at capacity. Use a Workflow for large batches. A direct start that cannot get +its own worker within a minute reports `state: not started` (exit 6); nothing runs later. Submit it +again when capacity is available. A launcher that ends before handing off ownership shows as dead. +A legacy workerless queue is also dead; `wait --auto-resume` can recover it explicitly. The load guard shares capacity across your sessions. New backend starts and replacements are spaced at least 5 seconds apart; turns on a running backend are not paced. @@ -193,7 +195,7 @@ On macOS, tracking and cleanup use process groups plus sampled descendants; memo running tools are not paused or limited. A detached child that orphans between samples can escape tracking on systems without scopes. -A queued job still occupies its Claude agent and, in a Workflow, one of that run's agent slots. A Workflow can +After admission, a queued job occupies its Claude agent and one Workflow agent slot. A row held at spawn has no running relay model or waiter. A Workflow can run up to 16 agents at once, fewer with fewer CPUs; `CLAUDE_CODE_WORKFLOW_MAX_CONCURRENT_AGENTS` changes that limit. Additional agents wait. At Claude's usage limit, Agent-tool relays fail; Workflows pause when auto-continue is on. Their backend jobs keep running. Ask Claude to collect those jobs or resume the paused @@ -328,7 +330,7 @@ counts them: ⚠ relay: 2 running · 1 queued (memory low) · 1 dead · 3 owed ``` -- `queued` counts pending jobs and shows a queue reason. A job waiting to start is queued, even before its +- `queued` counts pending jobs with a live worker and shows a queue reason. A job waiting to start is queued while its background process starts. - `running` counts the jobs that are starting or running. - `dead` counts jobs whose worker died or never came up; a message or the companion's `resume` continues a dead job diff --git a/package-lock.json b/package-lock.json index a54569b0..7e4a3cf2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", diff --git a/package.json b/package.json index b14d6920..08375913 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "private": true, "type": "module", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", diff --git a/plugins/relay/.claude-plugin/plugin.json b/plugins/relay/.claude-plugin/plugin.json index 9f9a6327..5d7c02c2 100644 --- a/plugins/relay/.claude-plugin/plugin.json +++ b/plugins/relay/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "relay", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", "author": { "name": "Libar" diff --git a/plugins/relay/CHANGELOG.md b/plugins/relay/CHANGELOG.md index 5d3edc40..e7a859a9 100644 --- a/plugins/relay/CHANGELOG.md +++ b/plugins/relay/CHANGELOG.md @@ -8,6 +8,9 @@ its entries here. Changed: +- Job acceptance requires a live worker lease within one minute; refused starts exit 6 and never run later. + Workflow relays wait at spawn for a waiting place, including stall retries at their first step; + plain relay calls report capacity after 30 seconds. Large batches should use Workflows. - Foreground `task` workers run independently of the caller; Ctrl-C leaves a dispatched job running. - Jobs own their execution groups and stop their tools at backend loss, cancellation and completion. Linux workers use per-job scopes whenever a user manager is available, and set OOM score 500. @@ -21,6 +24,12 @@ Changed: Fixed: +- Continuations and recovery use the same bounded launch path. Undelivered pending messages are + withdrawn rather than acknowledged without a worker, and interrupted recovery keeps its message id. +- Repeated launch failures stop after three attempts and show the error and count. Handoff retries + preserve owed continuations, and messages racing worker exit retry without a five-second delay. +- Nested workers publish their identity independently of admission storage, including with the guard off. + Admission retains time per wait reason and reports the longest hold after admission. - Repeated cleanup preserves a predecessor's first observed death, keeping the recorded ordering of predecessor exit and replacement startup intact. - Worker heartbeats keep process accounting active with the load guard off without creating diff --git a/plugins/relay/agents/job.md b/plugins/relay/agents/job.md index 6a5d7658..17e6e95e 100644 --- a/plugins/relay/agents/job.md +++ b/plugins/relay/agents/job.md @@ -23,7 +23,7 @@ Run with Bash: node "${CLAUDE_PLUGIN_ROOT}/scripts/companion.mjs" start --from-relay ``` -If the output has a line `job: `, remember the id and go to 2. Otherwise the start failed: hand back the start output at once, every line, nothing before it and nothing after it, and run nothing else, unless a hook's reason tells you to. +If the output starts with `state: not started`, hand back that output at once, every line, and run nothing else. Otherwise, if the output has a line `job: `, remember the id and go to 2. Otherwise the start failed: hand back the start output at once, every line, nothing before it and nothing after it, and run nothing else, unless a hook's reason tells you to. ## 2. Wait diff --git a/plugins/relay/hooks/mod/register.js b/plugins/relay/hooks/mod/register.js index 4800417a..c5ace920 100644 --- a/plugins/relay/hooks/mod/register.js +++ b/plugins/relay/hooks/mod/register.js @@ -1040,6 +1040,30 @@ async function drawJob($, { Box, Text, Button, Input, Markdown }, job, board) { return h(Box, { flexDirection: "column", marginTop: 1 }, ...nodes); } +// Host process waits do not spend the hook's ten-second JavaScript budget. +// Cancellation must reach the registered catch: a swallowed abort could spawn work +// after the user stopped it. Ordinary gate failures still fall back to start's bound. +async function waitForPlace($, workflow, agentId, transcript) { + for (;;) { + const argv = ["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--wait-seconds", workflow ? "540" : "30"]; + if (agentId) argv.push("--agent", agentId, "--retry"); + if (transcript) argv.push("--transcript", transcript); + let result; + try { result = await $.process.run(argv, { timeoutMs: workflow ? 600000 : 40000 }); } + catch (error) { + if (["abort", "cancelled", "timeout"].includes(error?.kind) || error?.name === "AbortError" || /abort|cancel|timeout/i.test(String(error?.message ?? error))) throw error; + return {}; + } + if (result.exitCode !== 0) return {}; + const output = String(result.stdout ?? "").trim(); + const pass = /^pass(?: ([a-f0-9-]{36}))?$/.exec(output); + if (pass) return { reservation: pass[1] ?? null }; + const hold = /^hold (.+)$/.exec(output); + if (!hold) return {}; + if (!workflow) return { deny: "relay is at capacity: " + hold[1] + ". Try again in a minute, or start these jobs from a Workflow, which waits for a place." }; + } +} + export function register(on) { // One timer and one read in flight per load of the module. `fastUntil` is the last tick of the // every-tick reads a relay event (a relay:job spawn, a wake, one of the commands below) asks for. @@ -1089,26 +1113,48 @@ export function register(on) { return next(e); }); + // In-memory identities only avoid duplicate hook work; all places and their + // expiry live under the coordinator's admission lock, never in mod counters. + const spawned = new Set(); + const relayAgents = new Map(); + const binding = new Map(); on("agent.spawn", async ($, e, next) => { - if (e?.subagentType === "relay:job") { - poller.fastUntil = poller.ticks + CLEARED_TICKS; - } - if (!isCheckedSpawn(e)) { - return next(e); - } - let refusal = null; - try { - refusal = await headerRefusal($, e); - } catch { - // A check that could not run says nothing about the header; the relay's start checks it. - refusal = null; + if (e?.subagentType !== "relay:job") return next(e); + poller.fastUntil = poller.ticks + CLEARED_TICKS; + if (e.resume != null) return next(e); + if (isCheckedSpawn(e)) { + let refusal = null; + try { refusal = await headerRefusal($, e); } catch {} + if (refusal !== null) return { deny: refusal }; } - if (refusal !== null) { - // Claude Code puts its own "spawn denied by a plugin" before the reason. - return { deny: refusal }; + const gate = await waitForPlace($, e.workflow != null); + if (gate.deny) return gate; + const result = await next(e); + if (result?.agentId) { + spawned.add(result.agentId); + if (gate.reservation) { + const pending = $.process.run(["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--bind", gate.reservation, "--agent", result.agentId], { timeoutMs: 10000 }); + binding.set(result.agentId, pending); + try { await pending; } finally { binding.delete(result.agentId); } + } } + return result; + }).catch(($, e) => ({ deny: "relay admission was interrupted. Try again, or resume the Workflow." })); + + // Workflow stall retries bypass agent.spawn. Track each actual attempt and + // hold its first model step using the same host-process gate. + on("classic.SubagentStart", async ($, e, next) => { + if (e.agent_type === "relay:job") relayAgents.set(e.agent_id, e.transcript_path); return next(e); }); + on("turn.step", async function* ($, e, next) { + if (e.index !== 0 || !relayAgents.has(e.agentId)) return yield* next(e); + if (binding.has(e.agentId)) await binding.get(e.agentId); + if (spawned.delete(e.agentId)) return yield* next(e); + await waitForPlace($, true, e.agentId, relayAgents.get(e.agentId)); + return yield* next(e); + }).catch(async function* ($, e) { return { turnId: e.turnId, index: e.index, answer: "state: not started\nerror: relay admission was interrupted; resume the Workflow.", + toolUses: [], stopReason: "end_turn", usage: { input_tokens: 0, output_tokens: 0 } }; }); on("session.send", async ($, e, next) => { const result = await next(e); diff --git a/plugins/relay/hooks/mod/register.test.ts b/plugins/relay/hooks/mod/register.test.ts index e867751e..3924d418 100644 --- a/plugins/relay/hooks/mod/register.test.ts +++ b/plugins/relay/hooks/mod/register.test.ts @@ -165,6 +165,7 @@ function spawnEngine(on: any, check: () => unknown = () => ran(0, 'ok\n')) { const runs: any[] = [] const spawns: unknown[] = [] on('process.run', (_: unknown, e: any) => { + if (e.argv[2] === 'gate') return ran(0, 'pass\n') runs.push(e) return check() }) @@ -2891,3 +2892,45 @@ test('the board shows a closed wait with a fixed duration after admission', asyn expect(texts).not.toContain('queued:') await ui.unmount() }) + +function capacityEngine(on: any, answers: string[]) { + const gates: any[] = [], spawns: any[] = [] + on('process.run', (_: unknown, e: any) => { + if (e.argv[2] === 'check-header') return ran(0, 'ok\n') + gates.push(e) + if (e.argv.includes('--bind')) return ran(0, 'bound\n') + return ran(0, answers.shift() ?? 'pass\n') + }) + on('agent.spawn', (_: unknown, e: unknown) => { spawns.push(e); return STARTED }) + return { gates, spawns } +} + +test('Workflow admission waits on hold then binds the waiting place to the actual agent', async ($, on) => { + const id = '00000000-0000-4000-8000-000000000001' + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n', `pass ${id}\n`]) + expect(await $.agent.spawn(relay(GOOD, { workflow: { runId: 'wf-capacity', agentIndex: 1 } }) as any)).toEqual(STARTED) + expect(seen.spawns).toHaveLength(1) + expect(seen.gates).toHaveLength(3) + expect(seen.gates[0].argv.slice(2)).toEqual(['gate', '--wait-seconds', '540']) + expect(seen.gates[0].init).toEqual({ timeoutMs: 600000 }) + expect(seen.gates[2].argv.slice(2)).toEqual(['gate', '--bind', id, '--agent', 'a-job']) +}) + +test('a plain relay refuses after its thirty-second gate with the capacity explanation', async ($, on) => { + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n']) + expect(await $.agent.spawn(relay(GOOD))).toEqual({ deny: 'relay is at capacity: waiting worker limit: 2 of 2. Try again in a minute, or start these jobs from a Workflow, which waits for a place.' }) + expect(seen.spawns).toHaveLength(0) + expect(seen.gates[0].argv.slice(2)).toEqual(['gate', '--wait-seconds', '30']) +}) + +test('an invalid gate answer falls back to the bounded start', async ($, on) => { + const seen = capacityEngine(on, ['not a verdict\n']) + expect(await $.agent.spawn(relay(GOOD))).toEqual(STARTED) + expect(seen.spawns).toHaveLength(1) +}) + +test('a resumed relay bypasses capacity admission', async ($, on) => { + const seen = capacityEngine(on, ['hold no place\n']) + expect(await $.agent.spawn(relay('Continue.', { resume: { agentId: 'a-job' } }) as any)).toEqual(STARTED) + expect(seen.gates).toHaveLength(0) +}) diff --git a/plugins/relay/scripts/cli/gate.mjs b/plugins/relay/scripts/cli/gate.mjs new file mode 100644 index 00000000..381c428b --- /dev/null +++ b/plugins/relay/scripts/cli/gate.mjs @@ -0,0 +1,49 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { parseCommandInput, sleep } from "./shared.mjs"; +import { bindReservation, readReservation, reserveWaitingPlace } from "../lib/worker/admission.mjs"; +import { findAdoptableRelayJob, findRelayJob, promptSha256, readJournalKey, readRelayTranscript, transcriptCandidates } from "../lib/harness/relay.mjs"; + +// A stalled Workflow attempt gets a fresh agent id without agent.spawn. At its +// first step, reuse an existing job or reservation before asking for another place. +function hasAdoptableJob(agentId, transcriptPath, env) { + if (findRelayJob(env, agentId)) return true; + for (const file of transcriptCandidates({ agent_id: agentId, transcript_path: transcriptPath })) { + if (!fs.existsSync(file)) continue; + const run = /[\\/]workflows[\\/]([^\\/]+)[\\/]agent-[^\\/]+\.jsonl$/.exec(file)?.[1]; + if (!run) continue; + const transcript = readRelayTranscript(file); + let meta; + try { meta = JSON.parse(fs.readFileSync(file.replace(/\.jsonl$/, ".meta.json"), "utf8")); } catch {} + if (findAdoptableRelayJob(env, { agentId, workflowRunId: run, journalKey: readJournalKey(file, agentId), + promptSha256: promptSha256(transcript.prompt?.text), label: meta?.description })) return true; + } + return false; +} + +export async function handleGate(argv) { + const { options } = parseCommandInput(argv, { valueOptions: ["wait-seconds", "bind", "agent", "transcript"], booleanOptions: ["retry"] }); + if (options.bind) { + if (!options.agent) throw new Error("gate --bind needs --agent"); + process.stdout.write(bindReservation(options.bind, options.agent) ? "bound\n" : "expired\n"); + return; + } + if (options.agent) { + const reservation = readReservation(options.agent); + if (reservation) { process.stdout.write(`pass ${reservation}\n`); return; } + if (options.retry && hasAdoptableJob(options.agent, options.transcript, process.env)) { process.stdout.write("pass\n"); return; } + } + const seconds = Number(options["wait-seconds"] ?? 0); + if (!Number.isFinite(seconds) || seconds < 0) throw new Error("--wait-seconds must be non-negative"); + const deadline = Date.now() + seconds * 1000; + for (;;) { + const result = reserveWaitingPlace(); + if (result.pass) { + if (result.reservation && options.agent) bindReservation(result.reservation, options.agent); + process.stdout.write(`pass${result.reservation ? ` ${result.reservation}` : ""}\n`); return; + } + if (Date.now() >= deadline) { process.stdout.write(`hold ${result.reason}\n`); return; } + await sleep(Math.min(2000 + Math.floor(Math.random() * 1000), Math.max(1, deadline - Date.now()))); + } +} diff --git a/plugins/relay/scripts/cli/message.mjs b/plugins/relay/scripts/cli/message.mjs index 487323d9..f5af486d 100644 --- a/plugins/relay/scripts/cli/message.mjs +++ b/plugins/relay/scripts/cli/message.mjs @@ -21,6 +21,7 @@ function renderDelivery(job, kind, delivery, state) { `message: ${kind} ${delivery.message.id}`, `outcome: ${delivery.outcome?.action ?? "pending (no worker has taken it yet)"}` ]; + if (delivery.outcome?.action === "not-started") lines.push(`not delivered: ${delivery.outcome.reason}; send it again later`); if (delivery.outcome?.turnId) { lines.push(`turn: ${delivery.outcome.turnId}`); } @@ -53,7 +54,8 @@ export async function handleSteerOrContinue(kind, argv) { const state = stateAfterDelivery(jobDir); if (options.json) printJson(kind, { job: job.id, ...delivery, state }); else process.stdout.write(renderDelivery(job, kind, delivery, state)); - if (!delivery.outcome) { + if (delivery.outcome?.action === "not-started") process.exitCode = EXIT.notStarted; + else if (!delivery.outcome) { process.exitCode = EXIT.running; } } @@ -76,7 +78,8 @@ export async function handleResume(argv) { const state = stateAfterDelivery(jobDir); if (options.json) printJson("resume", { job: job.id, ...delivery, state }); else process.stdout.write(renderDelivery(job, "resume", delivery, state)); - if (!delivery.outcome) { + if (delivery.outcome?.action === "not-started") process.exitCode = EXIT.notStarted; + else if (!delivery.outcome) { process.exitCode = EXIT.running; } } diff --git a/plugins/relay/scripts/cli/start.mjs b/plugins/relay/scripts/cli/start.mjs index 139a8ba8..e73d1e31 100644 --- a/plugins/relay/scripts/cli/start.mjs +++ b/plugins/relay/scripts/cli/start.mjs @@ -3,10 +3,11 @@ import { readInputFile, createTaskJob, parseCommandInput, printJson } from "./sh import { default as fs } from "node:fs"; import { default as path } from "node:path"; import { default as process } from "node:process"; -import { spawnWorker } from "../lib/worker/delivery.mjs"; +import { startWithinWindow } from "../lib/worker/delivery.mjs"; import { appendRelayViolation } from "../lib/jobs/messages.mjs"; import { nowIso, readJob } from "../lib/jobs/store.mjs"; import { buildLaunchRequest, readLaunchPrompt, UsageError } from "../lib/jobs/request.mjs"; +import { readReservation } from "../lib/worker/admission.mjs"; import { SCHEMA_SENTENCE } from "../lib/harness/workflow.mjs"; import { backendJson, backendLines, backendSettingLines, EXIT, loadJobViewSettled as loadJobView, relayAdoptionLines } from "../lib/jobs/view.mjs"; import { adoptRelayJob, deliverRelayMessages, findAdoptableRelayJob, promptSha256, readJournalKey, readRelayPointer, readRelayTranscript, writeRelayPointer } from "../lib/harness/relay.mjs"; @@ -165,12 +166,15 @@ async function handleStartFromRelay(options) { }); let job = readJob(jobDir); writeRelayPointer(process.env, { agentId, transcriptPath, jobId: job.id, jobDir, startError: null }); - spawnWorker(jobDir, request.cwd); + await startWithinWindow(jobDir, request.cwd, process.env, { reservation: agentId ? readReservation(agentId) : null }); job = readJob(jobDir); for (const problem of transcript.problems) { appendRelayViolation(jobDir, { kind: "transcript", reason: `messages could not be checked: ${problem}`, agentId }); } - await deliverRelayMessages(jobDir, transcript.messages, { wait: false }); + if (job.admission?.state !== "not-started") await deliverRelayMessages(jobDir, transcript.messages, { wait: false }); + if (job.admission?.state === "not-started") { + fail(job.errorMessage, "not-started"); process.exitCode = EXIT.notStarted; return; + } if (options.json) printJson("start", { jobId: job.id, state: job.state, jobDir, notes: request.notes, ...backendJson(request) }); else process.stdout.write(startBlock(job, jobDir, request, job.state)); } @@ -184,8 +188,14 @@ export async function handleStart(argv) { } const { request, brief } = readLaunchFromOptions(options); const jobDir = createTaskJob(request, brief); - spawnWorker(jobDir, request.cwd); + await startWithinWindow(jobDir, request.cwd); const job = readJob(jobDir); + if (job.admission?.state === "not-started") { + process.exitCode = EXIT.notStarted; + if (options.json) printJson("start", { jobId: job.id, state: "not started", jobDir, error: { kind: "not-started", message: job.errorMessage } }); + else process.stdout.write(`state: not started\nerror: ${job.errorMessage}\n`); + return; + } if (options.json) { printJson("start", { jobId: job.id, state: job.state, jobDir, notes: request.notes, ...backendJson(request) }); return; diff --git a/plugins/relay/scripts/cli/task.mjs b/plugins/relay/scripts/cli/task.mjs index 9fc5e5c3..fe44ce18 100644 --- a/plugins/relay/scripts/cli/task.mjs +++ b/plugins/relay/scripts/cli/task.mjs @@ -4,10 +4,10 @@ import { default as fs } from "node:fs"; import { default as path } from "node:path"; import { default as process } from "node:process"; import { readStdinIfPiped } from "../lib/util/fs.mjs"; -import { spawnWorker, retryQueuedWorker, hasLiveWorker } from "../lib/worker/delivery.mjs"; +import { startWithinWindow, hasLiveWorker } from "../lib/worker/delivery.mjs"; import { requireJobDirInHome, jobPaths, readJob } from "../lib/jobs/store.mjs"; import { buildLaunchRequest, UsageError } from "../lib/jobs/request.mjs"; -import { loadJobViewSettled as loadJobView, renderJobWithReport, isTerminalView } from "../lib/jobs/view.mjs"; +import { EXIT, loadJobViewSettled as loadJobView, renderJobWithReport, isTerminalView } from "../lib/jobs/view.mjs"; import { runWorker } from "../lib/worker/worker.mjs"; const STOP_REVIEW_TASK_MARKER = "Run a stop-gate review of the previous Claude turn."; @@ -49,16 +49,20 @@ export async function handleTask(argv) { prefix: "task", summary: prompt.includes(STOP_REVIEW_TASK_MARKER) ? `${backendWith("stopGate").label} Stop Gate Review` : null }); + await startWithinWindow(jobDir, request.cwd); + const started = readJob(jobDir); + if (started.admission?.state === "not-started") { + if (options.json) printJson("task", { jobId: started.id, state: "not started", jobDir, errorMessage: started.errorMessage }); + else process.stdout.write(`state: not started\nerror: ${started.errorMessage}\n`); + process.exitCode = EXIT.notStarted; return; + } if (options.background) { - spawnWorker(jobDir, request.cwd); const job = readJob(jobDir); if (options.json) printJson("task", { jobId: job.id, state: job.state, jobDir, backend: backendFor(request).name }); else process.stdout.write(`job: ${job.id}\nstate: ${job.state}\ndir: ${jobDir}\n`); return; } - spawnWorker(jobDir, request.cwd); for (;;) { - retryQueuedWorker(jobDir); const current = loadJobView(jobDir); if (isTerminalView(current) && !hasLiveWorker(jobDir)) break; await sleep(50); diff --git a/plugins/relay/scripts/companion.mjs b/plugins/relay/scripts/companion.mjs index 7d7f24be..a2e5a166 100644 --- a/plugins/relay/scripts/companion.mjs +++ b/plugins/relay/scripts/companion.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { handleGate } from "./cli/gate.mjs"; import { handleStart } from "./cli/start.mjs"; import { handleCheckHeader } from "./cli/check-header.mjs"; import { handleWait } from "./cli/wait.mjs"; @@ -70,6 +71,9 @@ async function main() { } switch (subcommand) { + case "gate": + await handleGate(argv); + break; case "check-header": handleCheckHeader(argv); break; diff --git a/plugins/relay/scripts/lib/jobs/lease.mjs b/plugins/relay/scripts/lib/jobs/lease.mjs index 8321fb65..0652d67b 100644 --- a/plugins/relay/scripts/lib/jobs/lease.mjs +++ b/plugins/relay/scripts/lib/jobs/lease.mjs @@ -1,5 +1,6 @@ -import { jobPaths } from "./store.mjs"; +import { jobPaths, readJob } from "./store.mjs"; import { acquireOwnership, readOwnership, releaseOwnership } from "../util/ownership.mjs"; +import { pendingInboxMessages, owedMessages } from "./messages.mjs"; import { isProcessIdentityAlive } from "../util/process.mjs"; @@ -13,7 +14,16 @@ export const isWorkerAlive = isProcessIdentityAlive; // holder that consumes inbox messages, from a cancel or a recovery that takes the lease to // write job.json. `worker` in the release is kept for the callers' symmetry. export function acquireWorkerLease(jobDir, worker, role = "worker") { - return Boolean(acquireOwnership(jobPaths(jobDir).lease, { ...worker, role })); + const file = jobPaths(jobDir).lease; + if (!acquireOwnership(file, { ...worker, role })) return false; + // A dispatch already in flight may arrive after the submitting window closed. + // Under the same fence as refusal, prevent it from reviving an unstarted brief. + // A later explicit message is work the user has asked to submit again. + if (role === "worker" && readJob(jobDir)?.admission?.state === "not-started" && + pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0) { + releaseOwnership(file); return false; + } + return true; } diff --git a/plugins/relay/scripts/lib/jobs/messages.mjs b/plugins/relay/scripts/lib/jobs/messages.mjs index a93d2771..cb9abf9d 100644 --- a/plugins/relay/scripts/lib/jobs/messages.mjs +++ b/plugins/relay/scripts/lib/jobs/messages.mjs @@ -250,3 +250,20 @@ export function messageAccount(jobDir) { .map(({ message, outcome }) => ({ id: message.id, reason: outcome.reason ?? "dropped" })); return { undelivered, dropped }; } + +// Renaming races the worker's claim atomically. A refused message is retained for +// inspection but never appears in pending/owed accounting or a future worker's input. +export function withdrawMessage(jobDir, messageId) { + const paths = jobPaths(jobDir), directory = path.join(jobDir, "withdrawn"); + fs.mkdirSync(directory, { recursive: true }); + try { fs.renameSync(path.join(paths.inbox, `${messageId}.json`), path.join(directory, `${messageId}.json`)); } + catch (error) { if (error.code === "ENOENT") return fs.existsSync(path.join(directory, `${messageId}.json`)); throw error; } + const relay = path.join(paths.inbox, "relay"); + let names; + try { names = fs.readdirSync(relay); } catch (error) { if (error.code !== "ENOENT") throw error; names = []; } + for (const name of names) { + const file = path.join(relay, name); + if (readJsonOrNull(file)?.messageId === messageId) fs.rmSync(file, { force: true }); + } + return true; +} diff --git a/plugins/relay/scripts/lib/jobs/view.mjs b/plugins/relay/scripts/lib/jobs/view.mjs index 8e1921e8..9618790e 100644 --- a/plugins/relay/scripts/lib/jobs/view.mjs +++ b/plugins/relay/scripts/lib/jobs/view.mjs @@ -15,7 +15,8 @@ export const EXIT = { usage: 1, noReport: 3, running: 4, - cancelled: 5 + cancelled: 5, + notStarted: 6 }; // How many times a view reads the job and its parts before it gives up on this call. @@ -127,6 +128,7 @@ export function exitCodeFor(view) { if (view.state === "cancelled") { return EXIT.cancelled; } + if (view.job?.admission?.state === "not-started") return EXIT.notStarted; if (view.undelivered?.length > 0) { return EXIT.noReport; } @@ -254,6 +256,7 @@ export function renderJobBlock(view) { } else if (job.errorMessage && view.state !== "completed") { lines.push(`error: ${String(job.errorMessage).replace(/\s+/g, " ").trim()}`); } + if (job.admission?.launchFailures) lines.push(`worker launch: ${job.admission.launchFailures} failed launch${job.admission.launchFailures === 1 ? "" : "es"} of 3${job.admission.launchError ? ` (${job.admission.launchError})` : ""}`); if (job.launch?.note) lines.push(`note: ${job.launch.note}`); // Cleanup still changes live accounting after a turn ends. A terminal result is copied // verbatim by relay agents, so do not include that changing measurement in its text. @@ -489,7 +492,12 @@ export function computeJobView(job, options = {}) { state = "dead"; deadReason = `worker ${job.worker.pid} is gone`; } - } else if (!(stored === "queued" && job?.admission?.waitKind === "worker")) { + } else if (stored === "queued" && job?.admission?.waitKind === "worker") { + if (!isWorkerAlive(job.admission.launcher)) { + state = "dead"; + deadReason = job.admission.launcher ? "not started: the process starting it ended" : "queued by an older relay with nobody to start it"; + } + } else { const created = Date.parse(job?.launchPendingAt ?? job?.createdAt ?? ""); if (Number.isFinite(created) && now - created > (options.workerStartGraceMs ?? WORKER_START_GRACE_MS)) { state = "dead"; diff --git a/plugins/relay/scripts/lib/util/process.mjs b/plugins/relay/scripts/lib/util/process.mjs index 7174c01b..7de832f7 100644 --- a/plugins/relay/scripts/lib/util/process.mjs +++ b/plugins/relay/scripts/lib/util/process.mjs @@ -433,7 +433,10 @@ export function readProcessTableCached({ home = null, maxAgeMs = 5000, now = Dat const cached = readJsonOrNull(file); if (cached && now >= cached.at && now - cached.at < maxAgeMs) return cached.rows; const lock = `${file}.lock`; - if (!acquireOwnership(lock)) return cached?.rows ?? null; + if (!acquireOwnership(lock)) { + if (maxAgeMs > 0 && cached && now >= cached.at && now - cached.at < 2 * maxAgeMs) return cached.rows; + return list(); + } try { const rows = list(); if (rows) writeJsonAtomic(file, { at: now, rows }); @@ -485,16 +488,19 @@ function scopeMemberStarted(row, execution) { return !row.startTime || !Number.isFinite(start) || row.startedAtMs >= Math.floor(start / 1000) * 1000; } +// Independent of admission storage, including when the guard is explicitly off. +export const independentWorkerRegistry = () => path.join("/tmp", `relay-workers-${process.getuid?.() ?? "user"}`); + // Published workers form ownership boundaries, including during a nested launch handoff. function foreignWorkerPids(job, table) { const excluded = new Set(); - for (const directory of new Set(executions(job).map(e => e.workerRegistry).filter(Boolean))) { - let names; - try { names = fs.readdirSync(directory); } catch { continue; } - for (const name of names) { - const worker = readJsonOrNull(path.join(directory, name)); - if (worker?.jobDir !== job.jobDir && worker?.startTime && table.some(row => - row.pid === worker.pid && row.startTime === worker.startTime && !row.zombie)) excluded.add(worker.pid); + for (const directory of new Set([independentWorkerRegistry(), ...executions(job).map(e => e.workerRegistry).filter(Boolean)])) { + // Lookup only current process ids. Historical crash records cannot increase + // sampling work, and a newly published nested worker is visible immediately. + for (const row of table) { + const worker = readJsonOrNull(path.join(directory, `${row.pid}.json`)); + if (worker?.jobDir !== job.jobDir && worker?.startTime && + row.startTime === worker.startTime && !row.zombie) excluded.add(row.pid); } } // Never walk through another job's worker to capture its backend or tools either. @@ -551,6 +557,12 @@ export function executionMembers(job, table, { jobDir = job?.jobDir, excludeSelf const pids = new Set(fs.readFileSync(file, "utf8").trim().split(/\s+/).map(Number)); entries.push(...table.filter(row => pids.has(row.pid) && !row.zombie && scopeMemberStarted(row, execution))); + // Scope membership disproves emptiness even when the process table missed + // a concurrently forked pid. Never signal it without a verified identity. + for (const pid of pids) if (pid > 0 && !table.some(row => row.pid === pid)) { + entries.push({ pid, startTime: null, identityUnknown: true }); + cause = "an execution scope member has no process identity"; + } } catch (error) { if (error.code !== "ENOENT") cause = "the execution scope could not be read"; } } const leader = table.find(row => row.pid === execution.leader?.pid && row.startTime === execution.leader.startTime && !row.zombie); @@ -562,7 +574,7 @@ export function executionMembers(job, table, { jobDir = job?.jobDir, excludeSelf const recorded = jobDir ? readJsonOrNull(path.join(jobDir, "processes.json"))?.members ?? [] : []; const knownRoots = verifiedRoots([...roots(job), ...recorded], table, excluded); entries.push(...roots(job), ...recorded, ...snapshotProcessTree(knownRoots.map(root => root.pid), { table, excludePids: excluded })); - const live = liveSnapshotMembers(entries, { table }); + const live = [...liveSnapshotMembers(entries, { table }), ...entries.filter(entry => entry.identityUnknown && !entry.startTime && !table.some(row => row.pid === entry.pid))]; const result = [...new Map(live.filter(entry => !excluded.has(entry.pid) && (!excludeSelf || entry.pid !== process.pid)).map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; if (cause) Object.assign(result, { cause }); return result; diff --git a/plugins/relay/scripts/lib/worker/admission.mjs b/plugins/relay/scripts/lib/worker/admission.mjs index 46c78c51..ecd1a2ca 100644 --- a/plugins/relay/scripts/lib/worker/admission.mjs +++ b/plugins/relay/scripts/lib/worker/admission.mjs @@ -7,7 +7,7 @@ import process from "node:process"; import { randomUUID, createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; import { listJobDirs, readJob, readJsonOrNull, writeJsonAtomic, writeJob, TERMINAL_STATES } from "../jobs/store.mjs"; -import { pendingInboxMessages } from "../jobs/messages.mjs"; +import { pendingInboxMessages, owedMessages } from "../jobs/messages.mjs"; import { acquireOwnership, releaseOwnership, readOwnership } from "../util/ownership.mjs"; import { executionMembers, readProcessTableCached, isProcessAlive, isProcessIdentityAlive, readProcessStartTime } from "../util/process.mjs"; @@ -24,7 +24,7 @@ export const DEFAULTS = Object.freeze({ weights: Object.freeze({ light: 1, medium: 2, "very-heavy": 3 }), heavyModel: /astra/i, kimiHeavyModel: /(^|[^a-z0-9])k3([^a-z0-9]|$)/i, lightEfforts: ["low", "minimal", "none"], lightModel: /luna|flash|highspeed|-fast$/i, - pollMs: 3000, claimantPolls: 4, probeMs: 120000, sensorCacheMs: 2000, discoveryMs: 60000, + reservationMs: 45000, maxLaunchFailures: 3, pollMs: 3000, claimantPolls: 4, probeMs: 120000, sensorCacheMs: 2000, discoveryMs: 60000, sensorStaleMs: 10000, processCacheMs: 60000, spawnGraceMs: 10000, stuckLockMs: 60000, pauseExpiryMs: 6 * 3600000, clockSkewMs: 300000, usagePauseMs: 30 * 60000, ratePauseMs: 60000, maxRatePauseMs: 15 * 60000, @@ -69,7 +69,7 @@ export function guardSettings(config = {}, machine = { totalBytes: os.totalmem() return { maxActive: DEFAULTS.maxActive, busySeats, guaranteedVeryHeavy: DEFAULTS.guaranteedVeryHeavy, guaranteedOther: DEFAULTS.guaranteedOther, headroomBytes: Math.max(DEFAULTS.headroomGiB * GiB, machine.totalBytes * DEFAULTS.headroomFraction), - maxWaitingWorkers: Math.max(DEFAULTS.minWaitingWorkers, DEFAULTS.waitingPerBusySeat * busySeats) }; + maxWaitingWorkers: config.maxWaitingWorkers ?? Math.max(DEFAULTS.minWaitingWorkers, DEFAULTS.waitingPerBusySeat * busySeats) }; } export function validModelClasses(entries) { return Array.isArray(entries) ? entries.filter(e => e && typeof e.pattern === "string" && /^[0-9A-Za-z._:/*-]{1,100}$/.test(e.pattern) && Object.hasOwn(WEIGHTS, e.class)) : []; @@ -317,7 +317,7 @@ function slotFile(home, id) { if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error( function writeSlot(home, slot) { fs.mkdirSync(path.join(home, "slots"), { recursive: true }); writeJsonAtomic(slotFile(home, slot.jobId), slot); } export function executionAlive(job, deps) { if (job?.execution || job?.worker) { - const members = executionMembers(job, readProcessTableCached({ home: admissionHome(deps.env), maxAgeMs: deps.freshExecution ? 0 : 5000 }), { jobDir: job.jobDir }); + const members = executionMembers(job, (deps.listProcesses ? deps.listProcesses() : readProcessTableCached({ home: admissionHome(deps.env), maxAgeMs: deps.freshExecution ? 0 : 5000 })), { jobDir: job.jobDir }); if (members.cause || members.some(p => p.pid !== job.worker?.pid)) return true; } return [job?.server, ...(job?.attempts ?? []).flatMap(a => a.servers ?? [])].some(s => s?.pid && !s.endedAt && deps.isAlive(s)); @@ -341,6 +341,12 @@ function slots(home, deps, clean = true) { if (name.endsWith(".tmp")) { if (clean && deps.now() - fs.statSync(file).mtimeMs > 60000) fs.rmSync(file, { force: true }); continue; } if (!/^[A-Za-z0-9_-]+\.json$/.test(name)) continue; const s = readJsonOrNull(file); + if (s?.v === 1 && s.state === "reserved" && s.jobId === name.slice(0, -5) && + /^r-[a-f0-9-]{36}$/.test(s.jobId) && Number.isFinite(Date.parse(s.expiresAt))) { + if (deps.now() < Date.parse(s.expiresAt) && Date.parse(s.expiresAt) - deps.now() <= DEFAULTS.reservationMs + DEFAULTS.clockSkewMs) result.push(s); + else if (clean) fs.rmSync(file, { force: true }); + continue; + } if (!s || s.v !== 1 || typeof s.jobDir !== "string" || s.jobId !== name.slice(0, -5) || !Object.hasOwn(WEIGHTS, s.class) || !["waiting", "running", "spawning"].includes(s.state) || typeof s.backend !== "string" || !Number.isFinite(Date.parse(s.since)) || (s.state !== "spawning" && (!Number.isInteger(s.pid) || s.pid <= 0)) || @@ -364,7 +370,7 @@ function slots(home, deps, clean = true) { const job = readJob(s.jobDir); s.quotaHeavy = isVeryHeavy({ backend: s.backend, model: s.model, effort: s.effort }) || isVeryHeavy(job?.request); s.weight = WEIGHTS[s.class]; - const backendLive = executionAlive({ ...job, jobDir: s.jobDir }, deps); + const backendLive = executionAlive({ ...job, jobDir: s.jobDir }, { ...deps, freshExecution: clean }); if (live || backendLive) result.push(!live && backendLive ? { ...s, state: "running" } : s); else if (clean) fs.rmSync(file, { force: true }); } @@ -460,8 +466,20 @@ function jobAccount(home, jobId, deps) { } const kindLabel = kind => ({ usage: "usage limit", rate: "rate limit", concurrency: "concurrent request limit" })[kind] ?? kind; function refusalReason(p) { if (p.reason) return p.reason; return `${p.backend}${p.account ? ` (account ${p.account})` : ""} paused: ${kindLabel(p.kind)}${p.code ? ` (${p.code})` : ""} until ${p.until}`; } +function waitHistory(admission, now) { + const totals = { ...(admission?.waitTotals ?? {}) }; + if (admission?.state === "waiting" && admission.reason) { + const from = Date.parse(admission.reasonSince ?? admission.since); + const elapsed = Number.isFinite(from) ? Math.max(0, now - from) : 0; + const kind = admission.causeKind ?? admission.waitKind ?? "unknown"; + const key = `${kind}: ${admission.reason}`; + totals[key] = { kind, reason: admission.reason, ms: (totals[key]?.ms ?? 0) + elapsed }; + } + return totals; +} +function mainWait(totals) { return Object.values(totals).sort((a, b) => b.ms - a.ms)[0]; } function recordFor(slot, notes, reason = null, waitKind = null) { - return { state: reason ? "waiting" : "admitted", reason: reason ?? slot.waitReason ?? null, waitKind: waitKind ?? slot.waitKind ?? null, since: slot.waitingSince ?? null, admittedAt: reason ? null : slot.admittedAt ?? null, class: slot.class, weight: slot.weight, probe: Boolean(slot.probe), notes }; + return { waitTotals: slot.waitTotals ?? {}, reasonSince: slot.reasonSince ?? null, state: reason ? "waiting" : "admitted", reason: reason ?? slot.waitReason ?? null, waitKind: waitKind ?? slot.waitKind ?? null, since: slot.waitingSince ?? null, admittedAt: reason ? null : slot.admittedAt ?? null, class: slot.class, weight: slot.weight, probe: Boolean(slot.probe), notes }; } function bypass(ctx, platform = process.platform) { const notes = [["linux", "darwin"].includes(platform) ? "load guard off" : `load guard bypassed: admission unsupported on ${platform}`]; @@ -543,6 +561,7 @@ export function admit(ctx, injected = {}) { mine.waitingSince ??= iso(now); const old = readJsonOrNull(slotFile(home, ctx.jobId)); const slot = existing?.state === "running" ? { ...existing, checkedAt: iso(now) } : { ...mine, since: old?.since ?? mine.since, spawnAt: old?.spawnAt ?? null, holdAt: old?.holdAt ?? old?.spawnAt ?? null, holdBytes: old?.holdBytes ?? 0, state: "waiting", waitKind }; + slot.waitTotals = waitHistory(ctx.admission, now); slot.reasonSince = iso(now); slot.waitReason = reason; slot.waitKind = waitKind; deps.writeSlot(home, slot); return { admit: false, reason, waitKind, retryMs: Number(deps.env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs * (0.67 + 0.67 * deps.random()), slot, record: recordFor({ ...slot, waitingSince: mine.waitingSince }, [...notes, ...deps.notes], reason, waitKind) }; @@ -605,8 +624,9 @@ export function admit(ctx, injected = {}) { mine.probe = true; mine.probeGeneration = pause.probe.generation; mine.probeAt = pause.probe.at; } const previousWait = ctx.admission?.state === "admitted" ? ctx.admission : null; - const slot = { ...mine, waitingSince: ctx.waitingSince ?? previousWait?.since ?? null, - waitReason: ctx.admission?.reason ?? own?.waitReason ?? null, waitKind: (["worker", "spawned"].includes(ctx.admission?.waitKind) ? ctx.admission.causeKind : ctx.admission?.waitKind) ?? own?.waitKind ?? null, + const waitTotals = waitHistory(ctx.admission, now), longest = mainWait(waitTotals); + const slot = { ...mine, waitTotals, waitingSince: ctx.waitingSince ?? previousWait?.since ?? null, + waitReason: longest?.reason ?? ctx.admission?.reason ?? own?.waitReason ?? null, waitKind: longest?.kind ?? (["worker", "spawned"].includes(ctx.admission?.waitKind) ? ctx.admission.causeKind : ctx.admission?.waitKind) ?? own?.waitKind ?? null, spawnAt: own?.spawnAt ?? null, state: "running", admittedAt: ctx.waitingSince ? iso(now) : previousWait?.admittedAt ?? iso(now), holdAt: retainedTurn ? existing.holdAt : iso(now), holdBytes: retainedTurn ? existing.holdBytes : startupBytes(mine.backend) }; deps.writeSlot(home, slot); @@ -808,10 +828,15 @@ export function renderGuard(snapshot) { // Reserve a waiting-worker place and spawn while holding the admission lock. The // caller owns retries; completing workers never launch another session's jobs. -export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { +export function spawnWithAdmission(jobDir, env, spawn, injected = {}, { reservation = null } = {}) { const deps = dependencies({ ...injected, env }); let spawnAttempted = false; - const launch = () => { spawnAttempted = true; return spawn(); }; + const launch = () => { + const job = readJob(jobDir); + if (!job || (TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0)) return null; + spawnAttempted = true; return spawn(); + }; + const launcher = self(deps); try { const platform = deps.platform ?? process.platform; if (!["linux", "darwin"].includes(platform)) { process.stderr.write(`relay load guard: admission unsupported on ${platform}; worker bypassed\n`); return launch(); } @@ -824,11 +849,15 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { deps.lockSleep(25 + Math.floor(50 * deps.random())); machine = machineReading(home, deps); } - const settings = guardSettings(config, machine), job = readJob(jobDir), candidates = legacyCandidates(deps); + const settings = spawnSettings(config, machine, env), job = readJob(jobDir), candidates = legacyCandidates(deps); const decision = retryLock(home, deps, () => { const job = readJob(jobDir); - if (TERMINAL_STATES.has(job?.state) && pendingInboxMessages(jobDir).length === 0) return null; - const all = slots(home, deps), existing = all.find(s => s.jobId === job.id); + if (TERMINAL_STATES.has(job?.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0) return null; + let all = slots(home, deps); + const reserved = all.find(s => s.state === "reserved" && s.jobId === `r-${reservation}`); + // Conversion is atomic with the spawning slot: nobody can take the place in between. + if (reserved) all = all.filter(s => s !== reserved); + const existing = all.find(s => s.jobId === job.id); if (existing && ((existing.state === "spawning" && recent(deps.now(), existing.since, DEFAULTS.spawnGraceMs)) || deps.isAlive({ pid: existing.pid, startTime: existing.processStart }))) return null; const waiting = all.filter(s => s.state !== "running").length; const cls = modelClass(job.request); @@ -837,21 +866,22 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { const check = memoryCheck(machine, settings, holds(all, job.id, deps.now()), DEFAULTS.workerBytes, guaranteed); const reason = check?.reason ?? (waiting >= settings.maxWaitingWorkers ? `waiting worker limit: ${waiting} of ${settings.maxWaitingWorkers}` : null); if (reason) { - writeJob(jobDir, { ...job, state: check?.permanent ? "failed" : "queued", ...(check?.permanent ? { completedAt: iso(deps.now()), errorMessage: reason } : {}), worker: null, - admission: { state: check?.permanent ? "failed" : "waiting", reason, waitKind: "worker", causeKind: check?.kind ?? "workers", since: job.admission?.state === "waiting" ? job.admission.since : iso(deps.now()), class: cls, weight: WEIGHTS[cls], notes: machine.notes } }); + writeJob(jobDir, { ...job, state: check?.permanent ? "failed" : TERMINAL_STATES.has(job.state) ? job.state : "queued", ...(check?.permanent ? { completedAt: iso(deps.now()), errorMessage: reason } : {}), worker: null, + admission: { ...job.admission, waitTotals: waitHistory(job.admission, deps.now()), reasonSince: iso(deps.now()), launcher, state: check?.permanent ? "failed" : job.admission?.state === "not-started" ? "not-started" : "waiting", reason, waitKind: "worker", causeKind: check?.kind ?? "workers", since: job.admission?.state === "waiting" ? job.admission.since : iso(deps.now()), class: cls, weight: WEIGHTS[cls], notes: machine.notes } }); return null; } const slot = { v: 1, jobId: job.id, jobDir, pid: null, processStart: null, backend: job.request.backend ?? "codex", account: job.request.account ?? null, class: cls, weight: WEIGHTS[cls], state: "spawning", spawnAt: iso(deps.now()), since: iso(deps.now()), holdAt: iso(deps.now()), holdBytes: DEFAULTS.workerBytes, holdGroup: machine.group, scopes: machine.scopes }; + if (reserved) fs.rmSync(slotFile(home, reserved.jobId), { force: true }); deps.writeSlot(home, slot); if (job.state === "queued" && job.admission?.waitKind === "worker") writeJob(jobDir, { ...job, state: "starting", worker: null, launchPendingAt: iso(deps.now()), - admission: { ...job.admission, waitKind: "spawned" } }); + admission: { ...job.admission, launcher, waitKind: "spawned" } }); return slot; }); if (!decision.locked) { const previousLock = job.admission?.waitKind === "worker" && /admission lock (busy|stuck)/.test(job.admission.reason); const lockSince = previousLock ? job.admission.lockSince ?? job.admission.since : iso(deps.now()); - writeJob(jobDir, { ...job, state: "queued", worker: null, admission: { state: "waiting", waitKind: "worker", causeKind: "lock", + writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, launcher, state: "waiting", waitKind: "worker", causeKind: "lock", reason: lockReason(home, job.id, decision.holder, deps, lockSince), lockSince, since: job.admission?.since ?? iso(deps.now()), notes: [] } }); return null; @@ -871,10 +901,68 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { } catch (e) { if (spawnAttempted) throw e; const job = readJob(jobDir); - if (job && !(TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0)) { + if (job && !(TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0)) { const held = guardError({ jobId: job.id, home: admissionHome(env), waitingSince: job.admission?.since, admission: job.admission }, e, deps); - try { writeJob(jobDir, { ...job, state: "queued", worker: null, admission: { ...held.record, waitKind: "worker", causeKind: "guard-error" } }); } catch {} + try { writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, ...held.record, launcher, waitKind: "worker", causeKind: "guard-error" } }); } catch {} } return null; } } + +// Waiting places live in the same lock domain as workers. An environment override is +// useful for deliberately smaller queues; absent it, the generic CPU formula applies. +function spawnSettings(config, machine, env) { + const value = env.RELAY_MAX_WAITING_WORKERS; + return guardSettings({ ...config, ...(value != null && /^\d+$/.test(value) ? { maxWaitingWorkers: Number(value) } : {}) }, machine); +} +const validReservation = id => typeof id === "string" && /^[a-f0-9-]{36}$/.test(id); +const bindingFile = (env, agentId) => { + if (typeof agentId !== "string" || !/^[A-Za-z0-9_-]+$/.test(agentId)) throw new Error("invalid agent id"); + return path.join(admissionHome(env), "bindings", `${agentId}.json`); +}; +export function readReservation(agentId, env = process.env) { + const binding = readJsonOrNull(bindingFile(env, agentId)); + return binding && Date.parse(binding.expiresAt) > Date.now() && validReservation(binding.reservation) ? binding.reservation : null; +} +export function bindReservation(reservation, agentId, env = process.env, injected = {}) { + if (!validReservation(reservation)) throw new Error("invalid reservation id"); + const deps = dependencies({ ...injected, env }), home = admissionHome(env); + const result = retryLock(home, deps, () => { + const slot = slots(home, deps).find(s => s.jobId === `r-${reservation}` && s.state === "reserved"); + if (!slot || (slot.agentId && slot.agentId !== agentId)) return false; + deps.writeSlot(home, { ...slot, agentId }); + const file = bindingFile(env, agentId); + fs.mkdirSync(path.dirname(file), { recursive: true }); + writeJsonAtomic(file, { reservation, expiresAt: slot.expiresAt }); + return true; + }); + return result.value ?? false; +} +export function reserveWaitingPlace(env = process.env, injected = {}) { + const deps = dependencies({ ...injected, env }); + try { + if (!admissionEnabled(env, undefined, deps.platform)) return { pass: true, reservation: null }; + const home = admissionHome(env), machine = machineReading(home, deps); + const settings = spawnSettings({}, machine, env); + const result = retryLock(home, deps, () => { + const waiting = slots(home, deps).filter(s => s.state !== "running").length; + if (waiting >= settings.maxWaitingWorkers) return { pass: false, reason: `waiting worker limit: ${waiting} of ${settings.maxWaitingWorkers}` }; + const reservation = randomUUID(), now = deps.now(); + deps.writeSlot(home, { v: 1, jobId: `r-${reservation}`, state: "reserved", since: iso(now), expiresAt: iso(now + DEFAULTS.reservationMs) }); + return { pass: true, reservation }; + }); + return result.locked ? result.value : { pass: false, reason: "admission lock busy" }; + } catch (error) { return { pass: false, reason: `guard unavailable (${error.message})` }; } +} + +// Called only under the job lease after its refusal has become terminal. +export function discardSpawnReservation(jobDir, env = process.env) { + const deps = dependencies({ env }), home = admissionHome(env); + try { + if (!admissionEnabled(env)) return; + retryLock(home, deps, () => { + const job = readJob(jobDir), file = slotFile(home, job.id), slot = readJsonOrNull(file); + if (job.admission?.state === "not-started" && slot?.state === "spawning") fs.rmSync(file, { force: true }); + }); + } catch {} +} diff --git a/plugins/relay/scripts/lib/worker/delivery.mjs b/plugins/relay/scripts/lib/worker/delivery.mjs index f472e0b9..49a3900b 100644 --- a/plugins/relay/scripts/lib/worker/delivery.mjs +++ b/plugins/relay/scripts/lib/worker/delivery.mjs @@ -10,9 +10,9 @@ import process from "node:process"; import { fileURLToPath } from "node:url"; import { absoluteJobsHomeEnv, jobPaths, readJob, writeJob, readJsonOrNull, writeJsonAtomic, TERMINAL_STATES } from "../jobs/store.mjs"; -import { isMessagePending, messageAccount, pendingInboxMessages, postInboxMessage, readOutcome, writeOutcome } from "../jobs/messages.mjs"; +import { isMessagePending, messageAccount, pendingInboxMessages, postInboxMessage, readOutcome, withdrawMessage } from "../jobs/messages.mjs"; import { isWorkerLease, readLease, releaseWorkerLease } from "../jobs/lease.mjs"; -import { spawnWithAdmission, DEFAULTS } from "./admission.mjs"; +import { spawnWithAdmission, discardSpawnReservation, DEFAULTS } from "./admission.mjs"; import { acquireOwnership, releaseOwnership } from "../util/ownership.mjs"; import { holderAlive } from "./admission.mjs"; import { readProcessStartTime, isProcessIdentityAlive } from "../util/process.mjs"; @@ -28,30 +28,45 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -export function spawnWorker(jobDir, cwd, env = process.env) { +export function spawnWorker(jobDir, cwd, env = process.env, { reservation = null } = {}) { const lease = jobPaths(jobDir).lease; if (!acquireOwnership(lease, { pid: process.pid, startTime: null, role: "spawn" }, { isAlive: holderAlive })) return null; try { let job = readJob(jobDir); + pruneLaunches(jobDir); const latest = latestLaunch(jobDir); if (latest?.worker && ["pending", "acknowledged"].includes(latest.state) && isProcessIdentityAlive(latest.worker)) return null; - if (job && !TERMINAL_STATES.has(job.state) && launchInterrupted(job, latest) && job.admission?.causeKind !== "launch") { - job = { ...job, state: "queued", worker: null, admission: { ...job.admission, state: "waiting", + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && launchInterrupted(job, latest) && job.admission?.causeKind !== "launch") { + job = { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, state: "waiting", waitKind: "worker", causeKind: "launch", reason: "worker launch interrupted; retrying", since: new Date().toISOString() } }; writeJob(jobDir, job); } + const pendingId = pendingInboxMessages(jobDir).at(0) ?? null; + if (pendingId && job?.admission?.launchMessageId !== pendingId && job?.admission?.launchFailures) { + job = { ...job, admission: { ...job.admission, launchFailures: 0, launchError: null, launchMessageId: pendingId } }; + writeJob(jobDir, job); + } + if ((job?.admission?.launchFailures ?? 0) >= DEFAULTS.maxLaunchFailures) return null; const retryAt = Date.parse(job?.admission?.workerRetryAt); const poll = envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs); - if (job?.state === "queued" && job.admission?.waitKind === "worker" && Date.now() >= retryAt && Date.now() - retryAt < poll) return null; - const result = spawnWithAdmission(jobDir, env, () => spawnWorkerProcess(jobDir, cwd, env)); + if (job?.state === "queued" && job.admission?.waitKind === "worker" && Date.now() - retryAt >= 0 && Date.now() - retryAt < poll) return null; + const result = spawnWithAdmission(jobDir, env, () => spawnWorkerProcess(jobDir, cwd, env), {}, { reservation }); const deferred = readJob(jobDir); if (deferred?.state === "queued" && deferred.admission?.waitKind === "worker") writeJob(jobDir, { ...deferred, launchPendingAt: new Date(Date.now()).toISOString(), admission: { ...deferred.admission, workerRetryAt: new Date(Date.now()).toISOString() } }); return result; } catch (error) { + // Dispatch acknowledgement is an observation, not worker ownership. An + // observation timeout cannot revoke a worker that may already have started. + const uncertain = latestLaunch(jobDir); + if (hasLiveWorker(jobDir) || isProcessIdentityAlive(uncertain?.worker)) { + process.stderr.write(`relay: worker dispatch observation uncertain (${error.message}); waiting for its owner\n`); + return uncertain?.state === "acknowledged" ? uncertain.worker.pid : null; + } const job = readJob(jobDir); - if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length > 0)) writeJob(jobDir, { ...job, state: "queued", worker: null, + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length > 0)) writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, state: "waiting", waitKind: "worker", causeKind: "launch", - reason: "worker launch interrupted; retrying", launchError: error.message, + reason: `worker launch failed (${(job.admission?.launchFailures ?? 0) + 1} of ${DEFAULTS.maxLaunchFailures}): ${error.message}`, launchError: error.message, + launchFailures: (job.admission?.launchFailures ?? 0) + 1, launchMessageId: pendingInboxMessages(jobDir).at(0) ?? null, since: new Date().toISOString(), workerRetryAt: new Date().toISOString() } }); process.stderr.write(`relay load guard: deferred worker spawn failed (${error.message}); work remains queued for retry\n`); return null; @@ -60,6 +75,7 @@ export function spawnWorker(jobDir, cwd, env = process.env) { } function spawnWorkerProcess(jobDir, cwd, env) { + pruneLaunches(jobDir, 19); const file = path.join(jobDir, `launch-${randomUUID()}.json`); writeJsonAtomic(file, { jobId: path.basename(jobDir), trigger: env.RELAY_WORKER_TRIGGER ?? process.argv[2] ?? "dispatch", caller: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, requestedAt: new Date().toISOString(), state: "pending" }); @@ -82,6 +98,8 @@ function spawnWorkerProcess(jobDir, cwd, env) { if (launch?.state?.startsWith("handoff-")) throw new Error("Worker launch handoff failed; no backend was started."); Atomics.wait(pause, 0, 0, 10); } + const uncertain = readJsonOrNull(file); + if (isProcessIdentityAlive(uncertain?.worker)) return uncertain.state === "acknowledged" ? uncertain.worker.pid : null; throw new Error("Worker launch acknowledgement timed out; no dispatch was confirmed."); } @@ -106,15 +124,6 @@ export function releaseLeaseToConsumer(jobDir, holder, env = process.env, { spaw return true; } -// A worker was spawned for this job a moment ago and has not taken its lease yet. -function workerJustSpawned(job, now = Date.now()) { - if (job?.worker?.pid || ["completed", "failed", "cancelled"].includes(job?.state)) { - return false; - } - const created = Date.parse(job?.createdAt ?? ""); - return Number.isFinite(created) && now - created < 10000; -} - // Whether the job ended after an attempt that ran this message. function attemptEnded(jobDir, messageId) { const job = readJob(jobDir); @@ -125,44 +134,84 @@ function attemptEnded(jobDir, messageId) { // Make sure a worker will take a message already in the inbox, and wait up to timeoutMs // for its outcome. With wait:false, wait only until a live consumer exists. // A timeout leaves consumer:false so a relay keeps the original message pending. +// A submission is acknowledged only after a live worker takes the lease. Refusal +// is fenced by the same lease as dispatch; a later reader cannot resurrect the brief. +export async function startWithinWindow(jobDir, cwd, env = process.env, options = {}) { + const now = options.now ?? Date.now, pause = options.sleep ?? sleep; + const windowMs = options.windowMs ?? envNumber(env, "RELAY_JOBS_ACK_TIMEOUT_MS", 60000); + const poll = Math.max(10, envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs)); + const deadline = now() + windowMs; + let nextSpawn = -Infinity, spawned = 0; + for (;;) { + if (hasLiveWorker(jobDir)) return { accepted: true, spawned }; + const job = readJob(jobDir); + if (!job || (TERMINAL_STATES.has(job.state) && !(options.messageId && messageAccount(jobDir).undelivered.includes(options.messageId)))) return { accepted: false, terminal: true, spawned }; + if (now() >= nextSpawn && now() <= deadline) { + if ((options.spawn ?? spawnWorker)(jobDir, cwd, env, { reservation: options.reservation })) spawned++; + nextSpawn = now() + poll; + if (hasLiveWorker(jobDir)) return { accepted: true, spawned }; + } + const current = readJob(jobDir); + if (TERMINAL_STATES.has(current?.state) && !(options.messageId && messageAccount(jobDir).undelivered.includes(options.messageId))) return { accepted: false, terminal: true, spawned }; + if (now() >= deadline || (current?.admission?.launchFailures ?? 0) >= DEFAULTS.maxLaunchFailures) { + const reason = current?.admission?.reason ?? "no live worker took ownership"; + if (options.refuse === false) return { accepted: false, reason, spawned }; + const lease = jobPaths(jobDir).lease; + if (acquireOwnership(lease, { pid: process.pid, startTime: readProcessStartTime(process.pid), role: "refusal" }, { isAlive: holderAlive })) { + try { + const final = readJob(jobDir); + if (final && !TERMINAL_STATES.has(final.state)) writeJob(jobDir, { ...final, state: "failed", worker: null, + completedAt: new Date().toISOString(), errorMessage: `Not started: ${reason}. Nothing ran; start it again later.`, + admission: { ...final.admission, state: "not-started", reason } }); + discardSpawnReservation(jobDir, env); + } finally { releaseOwnership(lease); } + return { accepted: false, reason, spawned }; + } + // An in-flight dispatch owns the fence. Wait for its live lease or recheck + // the terminal record before refusing; never race its backend start. + } + await pause(Math.max(10, Math.min(50, deadline - now()))); + } +} + export async function awaitOutcome(jobDir, messageId, options = {}) { const env = options.env ?? process.env; const timeoutMs = options.timeoutMs ?? envNumber(env, "RELAY_JOBS_ACK_TIMEOUT_MS", 60000); - const pollMs = envNumber(env, "RELAY_JOBS_POLL_MS", 200); + const pollMs = Math.max(10, envNumber(env, "RELAY_JOBS_POLL_MS", 200)); + const spawnPoll = Math.max(10, envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs)); const deadline = Date.now() + timeoutMs; - let lastSpawnAt = 0; let spawned = 0; + if (!hasLiveWorker(jobDir) && messageAccount(jobDir).undelivered.includes(messageId) && timeoutMs > 0) { + const job = readJob(jobDir); + spawned += (await startWithinWindow(jobDir, job?.request?.cwd ?? process.cwd(), env, + { windowMs: timeoutMs, messageId, refuse: false })).spawned; + } + let nextSpawn = -Infinity, wasLive = false; for (;;) { - retryQueuedWorker(jobDir, env); const outcome = readOutcome(jobDir, messageId); const live = hasLiveWorker(jobDir); - // A worker that ran this message in an attempt and then ended the job (a failed resume, - // say) leaves it owed for `resume`, which the state block names. Only a worker that died - // before it ended such an attempt gets a successor here; otherwise the attempt count would - // depend on whether this loop read the outcome before or after that worker exited. if (outcome && (live || !messageAccount(jobDir).undelivered.includes(messageId) || attemptEnded(jobDir, messageId))) { return { outcome, spawned, consumer: true }; } - const deferred = readJob(jobDir); - if (!live && deferred?.state === "queued" && deferred.admission?.waitKind === "worker") { - const queued = outcome ?? { action: "queued-next-turn", reason: deferred.admission.reason }; - if (!outcome) writeOutcome(jobDir, messageId, queued); - return { outcome: queued, spawned, consumer: true }; - } - if (options.wait === false && live) { - return { outcome: null, spawned, consumer: true }; - } + if (options.wait === false && live) return { outcome: null, spawned, consumer: true }; if (Date.now() >= deadline) { + if (live) return { outcome: null, spawned, consumer: false }; + const reason = readJob(jobDir)?.admission?.reason ?? "no live worker took the message"; + if (withdrawMessage(jobDir, messageId)) return { outcome: { action: "not-started", reason }, spawned, consumer: false }; + // A claim beat the rename. Re-read its outcome before deciding: the worker + // owns delivery now, and the caller must not invite a duplicate send. + const claimedOutcome = readOutcome(jobDir, messageId); + if (claimedOutcome && (hasLiveWorker(jobDir) || !messageAccount(jobDir).undelivered.includes(messageId) || attemptEnded(jobDir, messageId))) return { outcome: claimedOutcome, spawned, consumer: true }; + if (hasLiveWorker(jobDir)) { await sleep(pollMs); continue; } return { outcome: null, spawned, consumer: false }; } - if ((isMessagePending(jobDir, messageId) || messageAccount(jobDir).undelivered.includes(messageId)) && !live && Date.now() - lastSpawnAt > 5000) { + if (!live && (wasLive || Date.now() >= nextSpawn) && + (isMessagePending(jobDir, messageId) || messageAccount(jobDir).undelivered.includes(messageId))) { const job = readJob(jobDir); - if (job?.admission?.waitKind === "worker" || !workerJustSpawned(job)) { - const pid = spawnWorker(jobDir, job?.request?.cwd ?? process.cwd(), env); - lastSpawnAt = Date.now(); - if (pid) spawned += 1; - } + if (spawnWorker(jobDir, job?.request?.cwd ?? process.cwd(), env)) spawned++; + nextSpawn = Date.now() + (isProcessIdentityAlive(latestLaunch(jobDir)?.worker) ? pollMs : spawnPoll); } + wasLive = live; await sleep(Math.max(10, Math.min(pollMs, deadline - Date.now()))); } } @@ -170,14 +219,25 @@ export async function awaitOutcome(jobDir, messageId, options = {}) { // Drop a message into the job's inbox and wait until a worker records its outcome. export async function deliverMessage(jobDir, kind, text, extra = {}, options = {}) { const message = postInboxMessage(jobDir, { kind, text, ...extra }); - const { outcome, spawned } = await awaitOutcome(jobDir, message.id, options); - return { message, outcome, spawned }; + const { outcome, spawned, consumer } = await awaitOutcome(jobDir, message.id, options); + return { message, outcome, spawned, consumer }; } +export function pruneLaunches(jobDir, keep = 20) { + const files = fs.readdirSync(jobDir).filter(name => /^launch-.*\.json$/.test(name)); + const records = files.map(name => ({ name, record: readJsonOrNull(path.join(jobDir, name)) })) + .sort((a, b) => String(b.record?.requestedAt ?? "").localeCompare(String(a.record?.requestedAt ?? "")) || b.name.localeCompare(a.name)); + for (const entry of records.slice(keep)) { + // An unresolved live handoff is never discarded, regardless of history age. + if (entry.record?.state === "pending" && isProcessIdentityAlive(entry.record.worker)) continue; + fs.rmSync(path.join(jobDir, entry.name), { force: true }); + } + return records.slice(0, keep).map(entry => entry.record).filter(Boolean); +} function latestLaunch(jobDir) { return fs.readdirSync(jobDir).filter(name => /^launch-.*\.json$/.test(name)) .map(name => readJsonOrNull(path.join(jobDir, name))).filter(Boolean) - .sort((a, b) => String(a.requestedAt).localeCompare(String(b.requestedAt))).at(-1); + .sort((a, b) => String(b.requestedAt).localeCompare(String(a.requestedAt)))[0] ?? null; } function launchInterrupted(job, latest) { @@ -187,7 +247,7 @@ function launchInterrupted(job, latest) { export function retryQueuedWorker(jobDir, env = process.env) { const job = readJob(jobDir); - if (job && !TERMINAL_STATES.has(job.state) && !hasLiveWorker(jobDir) && !isProcessIdentityAlive(job.worker)) { + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && !hasLiveWorker(jobDir) && !isProcessIdentityAlive(job.worker)) { const latest = latestLaunch(jobDir); if (launchInterrupted(job, latest)) { // Re-read and reserve under the spawn lease; concurrent waiters share that fence. diff --git a/plugins/relay/scripts/lib/worker/recovery.mjs b/plugins/relay/scripts/lib/worker/recovery.mjs index 42ce91e1..5cbc736c 100644 --- a/plugins/relay/scripts/lib/worker/recovery.mjs +++ b/plugins/relay/scripts/lib/worker/recovery.mjs @@ -3,7 +3,8 @@ import process from "node:process"; import fs from "node:fs"; import path from "node:path"; -import { releaseLeaseToConsumer, spawnWorker } from "./delivery.mjs"; +import { DEFAULTS } from "./admission.mjs"; +import { releaseLeaseToConsumer, startWithinWindow } from "./delivery.mjs"; import { acquireWorkerLease, isWorkerAlive } from "../jobs/lease.mjs"; import { computeJobView } from "../jobs/view.mjs"; import { messageExists, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; @@ -51,9 +52,11 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { let launch = false; let cancelling = false; let cwd; + let original; try { let job = readJob(jobDir); if (computeJobView(job).state !== "dead" || (job.kind && job.kind !== "task")) return false; + original = job; const epoch = job.recoveryEpoch ?? "start"; const previous = recoveryForEpoch(job); // A reservation interrupted before posting is repaired with the same message id. @@ -64,26 +67,35 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { cancelling = cancelIsRunning(jobDir); if (cancelling) return "busy"; if (previous && previous.state !== "reserved") { - writeJob(jobDir, { ...job, server: null, serverPid: null }); + writeJob(jobDir, { ...job, server: null, serverPid: null, + automaticResumes: job.automaticResumes.map(entry => entry.id === previous.id ? previous : entry) }); return false; } - const record = previous ?? { id: newMessageId(), epoch, at: nowIso(), fromAttempt: job.attempts?.length ?? 0, reason: "worker", state: "reserved" }; + const record = previous ?? { id: newMessageId(), epoch, at: nowIso(), fromAttempt: job.attempts?.length ?? 0, fromWorkerCount: job.workers?.length ?? 0, reason: "worker", state: "reserved" }; const records = previous ? job.automaticResumes : [...(job.automaticResumes ?? []), record]; writeJob(jobDir, { ...job, automaticResumes: records }); if (!messageExists(jobDir, record.id)) { postInboxMessage(jobDir, { id: record.id, kind: "resume", text: DEFAULT_RESUME_MESSAGE, isDefault: true, automatic: true }); } - writeJob(jobDir, { - ...job, automaticResumes: records.map(entry => entry.id === record.id ? { ...entry, state: "launched" } : entry), - state: "starting", worker: null, server: null, serverPid: null, launchPendingAt: nowIso() - }); cwd = job.request.cwd; launch = true; } finally { releaseLeaseToConsumer(jobDir, me, env, { spawn: !launch && !cancelling }); } - if (launch) spawnWorker(jobDir, cwd, { ...env, RELAY_WORKER_TRIGGER: "recovery" }); - return launch; + if (!launch) return false; + const result = await startWithinWindow(jobDir, cwd, { ...env, RELAY_WORKER_TRIGGER: "recovery" }, + { windowMs: Number(env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs, refuse: false }); + // Workers update the job concurrently. Take its lease before changing the + // recovery record, or let the worker's own attempt bookkeeping be authoritative. + if (acquireWorkerLease(jobDir, me, "recovery")) { + try { + const job = readJob(jobDir), record = recoveryForEpoch(job); + if (!result.accepted && record?.state === "reserved") writeJob(jobDir, { ...job, state: original.state, worker: original.worker }); + if (record?.state === "reserved" && result.accepted) writeJob(jobDir, { ...job, + automaticResumes: job.automaticResumes.map(entry => entry.id === record.id ? { ...entry, state: "launched" } : entry) }); + } finally { releaseLeaseToConsumer(jobDir, me, env, { spawn: false }); } + } + return result.accepted; } // One automatic recovery per user-work epoch, shared by worker and app-server deaths. @@ -91,5 +103,10 @@ export const DEFAULT_RESUME_MESSAGE = "Your session was cut off. Continue the ta export function recoveryForEpoch(job) { const epoch = job.recoveryEpoch ?? "start"; - return (job.automaticResumes ?? []).find(entry => entry.epoch === epoch); + const record = (job.automaticResumes ?? []).find(entry => entry.epoch === epoch); + // Worker-owned history proves that a reservation was taken without a launcher + // overwriting job.json while the worker held it. Preserve the single retry epoch. + if (record?.state === "reserved" && ((job.attempts ?? []).some(a => a.messageId === record.id) || + (record.fromWorkerCount != null && (job.workers?.length ?? 0) > record.fromWorkerCount))) return { ...record, state: "launched" }; + return record; } diff --git a/plugins/relay/scripts/lib/worker/registry.mjs b/plugins/relay/scripts/lib/worker/registry.mjs index 740e309a..74bceafd 100644 --- a/plugins/relay/scripts/lib/worker/registry.mjs +++ b/plugins/relay/scripts/lib/worker/registry.mjs @@ -1,20 +1,34 @@ import fs from "node:fs"; import path from "node:path"; -import { admissionEnabled, admissionHome } from "./admission.mjs"; -import { writeJsonAtomic } from "../jobs/store.mjs"; +import { independentWorkerRegistry, isProcessIdentityAlive } from "../util/process.mjs"; +import { readJsonOrNull, writeJsonAtomic } from "../jobs/store.mjs"; export function registerWorker(worker, { env = process.env, platform = process.platform } = {}) { if (!["linux", "darwin"].includes(platform)) return null; - const directory = `${admissionHome(env)}.workers`; + // Cleanup boundaries must survive an unwritable admission directory, even with + // the guard off. Publish before handoff; inability to publish prevents launch. + const directory = independentWorkerRegistry(); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + pruneWorkerRegistry(directory); const file = path.join(directory, `${worker.pid}.json`); - try { - fs.mkdirSync(directory, { recursive: true }); - writeJsonAtomic(file, worker); - return file; - } catch (error) { - if (admissionEnabled(env, undefined, platform)) throw error; - // Guard-off launches cannot require admission storage. The launch intent and its - // reparenting acknowledgement remain mandatory writes in the job's own directory. - return null; + writeJsonAtomic(file, worker); + return file; +} + +// Registration performs a small rotating maintenance pass, independently of +// heartbeat sampling. Identity checks retain live or unreadable identities. +export function pruneWorkerRegistry(directory = independentWorkerRegistry(), { limit = 128, isAlive = isProcessIdentityAlive } = {}) { + let names; + try { names = fs.readdirSync(directory).filter(name => /^\d+\.json$/.test(name)).sort(); } + catch (error) { if (error.code === "ENOENT") return 0; throw error; } + const cursorFile = path.join(directory, ".sweep.json"); + const cursor = readJsonOrNull(cursorFile)?.after ?? ""; + const ordered = [...names.filter(name => name > cursor), ...names.filter(name => name <= cursor)].slice(0, limit); + let removed = 0; + for (const name of ordered) { + const file = path.join(directory, name), worker = readJsonOrNull(file); + if (worker?.startTime && !isAlive(worker)) { fs.rmSync(file, { force: true }); removed++; } } + if (ordered.length) writeJsonAtomic(cursorFile, { after: ordered.at(-1) }); + return removed; } diff --git a/plugins/relay/skills/jobs/SKILL.md b/plugins/relay/skills/jobs/SKILL.md index 76f6192f..0f82f1f0 100644 --- a/plugins/relay/skills/jobs/SKILL.md +++ b/plugins/relay/skills/jobs/SKILL.md @@ -171,7 +171,7 @@ dir: ~/.relay-jobs// ``` -- `state` is `completed`, `failed` (with `error:`), `cancelled`, `dead` (the job has not ended, but its worker is gone, or none started within 60 s of the launch, except a workerless job queued for admission; whether a recovery was tried is another matter, shown by `automatic-resume:`), or `queued` (waiting for the load guard; `queued:` says why), `starting`/`running`. A failed start is `state: not started` with `error:`. A block with no `---` has no report. +- `state` is `completed`, `failed` (with `error:`), `cancelled`, `dead` (the job has not ended, but its worker is gone, or its launch process ended before ownership, whether a recovery was tried is another matter, shown by `automatic-resume:`), or `queued` (waiting for the load guard; `queued:` says why), `starting`/`running`. A failed start is `state: not started` with `error:`. A block with no `---` has no report. - `completed` means the job finished its answer, not that its commands passed. Read the report. - Before you act on a result or show it, load the `relay:result-handling` skill: it reads the state before the report. - Lines that may appear: `queued:`, `refusal:`, `backend:`, `grok-home:` and `kimi-home:` (see Grok jobs, GLM jobs and Kimi jobs), `kind:` (a review), `tier: fast`, `mcp-off:` (the MCP servers this job turned off), `relay:`, `note:`, `stalled:` (no event for ten minutes; information only), `capacity-retries:`, `automatic-resume:` (a recovery and its cause), `relay-adopted:`, `undelivered:` (messages to the job, acknowledged and not run yet; `resume` runs them; it says nothing about the report), `not-delivered:` (dropped, for example by a cancel), and `relay-violation:` (the hook let the relay go, or a delivery or a cancel failed; read the job yourself with `C result `). @@ -256,10 +256,12 @@ A person stopping a relay (`x`, Esc or the app's Stop) leaves its job running. S A wide fan-out that needs no steering can start its jobs with `C start` a few seconds apart and collect each with `C wait `; `C status` only reads jobs and does not start deferred work; no Claude agent is held per job. Jobs started within the same second can hit a provider's request-rate limit. For Codex and Grok routes, the worker retries at 30, 60 and 120 s. ZCode and Kimi retry model calls themselves. -A job can wait as `queued` before it starts or between turns. Keep calling `wait`: queued work -remains pending and adoptable, including in a Workflow. Cancel, steer and continue still work. -Jobs without a worker need a submitter/adopter's wait, message or relay hook to retry spawning; -a bare `start` can stay dormant. `status` alone does not start deferred work: use `C wait `. +A job can wait as `queued` before its backend starts or between turns; its live worker owns that wait. +When the start queue is full, Workflow agents wait at spawn as queued without model tokens or job +waiters. A plain `relay:job` call waits up to 30 seconds then reports capacity. Use a Workflow for +large batches. A direct start that cannot get a worker within a minute reports not started (exit 6); +nothing runs later. A dead launch process or legacy workerless queue is dead and can be recovered +with `C wait --auto-resume`. Cancel, steer and continue still work for accepted jobs. The guard paces new backend starts and replacements 5 seconds apart across this user's sessions. Turns on a running backend are not paced and wait only for critical memory/pressure or provider pauses/probes. The first two identified very-heavy and first three other active jobs skip the host reserve and the busy-seat allowance, subject to critical checks, startup allowance, scope/fit limits, provider protection, ordering and start pacing. There is no very-heavy maximum. @@ -269,20 +271,21 @@ A macOS pressure warning is shown but holds nothing. Memory and load can delay n healthy machines have no fixed active ceiling. Turn the guard off only when the user asks. Sensor details, small-host fit limits and formulas are in [internals.md](internals.md). -`--json` prints one object whose first fields are `schema` (`relay./1`), `relayVersion` and `relayBuild`. An error under `--json` is that header with `error: { kind, message }` on stdout, `kind` one of `busy` (exit 4: try again), `not-found`, `usage`, `attempt-changed` or `error`. `C status --json` lists slim rows, newest first, 50 a page (`--limit N`, up to 200), with `nextCursor` for `--cursor`; `--active` lists only jobs that are queued, run, died or owe messages. One job (`C status --json`, `result`, `wait`, `cancel`) has every field, among them `attemptRecords` and `messages`, each message with how the job took it. +`--json` prints one object whose first fields are `schema` (`relay./1`), `relayVersion` and `relayBuild`. An error under `--json` is that header with `error: { kind, message }` on stdout, `kind` one of `busy` (exit 4: try again), `not-found`, `usage`, `attempt-changed`, `not-started` (exit 6) or `error`. `C status --json` lists slim rows, newest first, 50 a page (`--limit N`, up to 200), with `nextCursor` for `--cursor`; `--active` lists only jobs that are queued, run, died or owe messages. One job (`C status --json`, `result`, `wait`, `cancel`) has every field, among them `attemptRecords` and `messages`, each message with how the job took it. -Exit codes of `wait` and `result`: `0` completed with a report; `3` ended with no answer to take, which is `failed` or `dead` (even with a report), `completed` with no report, or messages not yet run (`undelivered:`), so read the state, then any report; `4` non-terminal (`queued`, `starting` or `running`); `5` cancelled, including with a pending continuation because cancellation takes precedence; `1` usage error. +Exit codes of `wait` and `result`: `0` completed with a report; `3` ended with no answer to take, which is `failed` or `dead` (even with a report), `completed` with no report, or messages not yet run (`undelivered:`), so read the state, then any report; `4` non-terminal (`queued`, `starting` or `running`); `5` cancelled, including with a pending continuation because cancellation takes precedence; `6` not started; `1` usage error. ### Continue a job by its id `C steer`, `C continue` and `C resume` post a new message each time they run, and print a receipt: `job:`, `message: `, `outcome:` and `state:`. - Exit `0`: the job recorded how it took the message, such as `steered`, `queued-next-turn`, `accepted-next-turn` or `turn-started`; `dropped` and `skipped` did not run, and `reason:` says why. -- Exit `4`: the message was posted, but no outcome came within the acknowledgement window (60 s), so `outcome:` is `pending`. It stays in the job's inbox for a worker to take. That is neither a refusal nor a failure. (`resume` also exits 4 when the job is busy, before it posts anything.) +- Exit `6`: no worker took the message within a minute. `not delivered: …; send it again later` means it was withdrawn before claim and will not run later. +- Exit `4`: the message has a live worker or remains claimed, but no outcome came within the acknowledgement window (60 s), so `outcome:` is `pending`. That is neither a refusal nor a failure. (`resume` also exits 4 when the job is busy, before it posts anything.) - Exit `1` can also follow posting: delivery or the state read can fail before a receipt is printed. Absence of a receipt alone proves nothing. Only a known pre-post validation refusal (such as an empty message or unknown job) proves nothing was posted. - Whether the turn then failed is the job's state, which `wait` reports. Before you send again after any nonzero exit, read the job's `messages` in `C status --json`: your message may have run since. -For a workerless queued job, `queued-next-turn` with a queue `reason:` acknowledges the message; it does not mean it ran. +Only a live worker writes `queued-next-turn` or `accepted-next-turn`. These receipts acknowledge ownership; they do not prove the turn ran. A continue, in full: diff --git a/plugins/relay/skills/jobs/internals.md b/plugins/relay/skills/jobs/internals.md index 8cf3846c..2bc79706 100644 --- a/plugins/relay/skills/jobs/internals.md +++ b/plugins/relay/skills/jobs/internals.md @@ -29,7 +29,7 @@ The harness marks each `agent()` call by the key it writes to the run's `journal The relay reads two things Claude Code does not publish as an interface: the agent transcript and the Workflow tool's run journal (`journal.jsonl`). Its markers were read from Claude Code 2.1.287 and 2.1.288 (`scripts/lib/harness/workflow.mjs`), and its hooks need `SubagentHandback`. Its hooks module needs the `session.send`, `agent.spawn` and `command.run` events of the Mod API, so the floor is Claude Code 2.1.307. A Claude Code update can break it. The check after an update is one workflow that runs a `relay:job` agent: the job's `job.json` must hold the call's key as `relay.journalKey` (once another relay has adopted the job, its `relay.json` holds the binding), and the script must get the state block back. -The module only checks headers and wakes a waiting relay: delivery runs on command hooks and the transcript. When a message is sent to a relay, the module's `session.send` hook runs `wake --agent `, which writes `relay-wake.json` in the job directory if that agent is the job's current relay, also when the job has ended, since a message can continue it. The guard passes each relay wait `--wake-after `, the time of the last transcript entry before the assistant message that holds the call, where Claude Code last put queued messages into the relay's conversation. The wait returns within a second of a marker newer than that, and the hook delivers the message at the relay's next call. Nothing removes a marker; a marker covered by the next boundary ends no more waits. +The module checks headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript. When a message is sent to a relay, the module's `session.send` hook runs `wake --agent `, which writes `relay-wake.json` in the job directory if that agent is the job's current relay, also when the job has ended, since a message can continue it. The guard passes each relay wait `--wake-after `, the time of the last transcript entry before the assistant message that holds the call, where Claude Code last put queued messages into the relay's conversation. The wait returns within a second of a marker newer than that, and the hook delivers the message at the relay's next call. Nothing removes a marker; a marker covered by the next boundary ends no more waits. Before a `relay:job` agent starts, the module's `agent.spawn` hook runs `check-header` on its prompt, the parser and request builder `start` uses, and denies the spawn with the parser's message when the header is bad. It leaves relative paths to `start`, since the relay's directory is not known at the spawn, and lets the spawn through when it cannot run the check, for a resumed agent, and for a Workflow agent whose prompt has no `---` line (a run that keeps its script from hooks puts a placeholder there); `start` checks again. A `PostToolUse` hook on `TaskStop` (`scripts/hooks/relay-stop.mjs`) takes the stopped agent's id from the tool's result and, when that agent is its job's current relay and the job has not ended, runs `cancel` and puts the result beside the TaskStop's; a failed cancel is a `relay-violation:`. A TaskStop on a whole Workflow cancels nothing, so a resume adopts the running jobs; the hook lists the run's jobs that keep running. @@ -99,10 +99,16 @@ Grok home. A job keeps its tier even when the `fast-tier` setting changes while The guard shares one admission home across this user's sessions, independently of jobs homes. For `C` CPU cores, the busy allowance is `F = min(16, max(2, C−2))`; the resident waiting-worker -limit is `Q = max(8, 2F)`. A workerless queued job remains valid; a wait, message or relay hook -retries its spawn at most once per admission poll, with a 64 MiB startup hold per waiting worker. -A workerless job with no spawn retry for 15 minutes stops deferring the stop-time review gate; -that changes gate deferral, not queue validity. +limit is `Q = max(8, 2F)`. Reservations, spawning workers and resident waiting workers share Q. +A reservation expires after 45 seconds and turns into a spawning slot atomically under the +admission lock. `RELAY_MAX_WAITING_WORKERS` explicitly overrides Q; the default follows CPU count. +Starts wait up to 60 seconds for a live worker lease or report not started (exit 6), with no later +execution. Workflow relays wait at `agent.spawn` through host processes, plain relays refuse after +30 seconds, and Workflow stall retries are gated at their first step. Every dispatch uses the +short-lived launcher. Failed launches stop after three failures and show their error and count. +A dead launcher or older workerless queue is dead, with explicit recovery through `wait --auto-resume`. +Undelivered messages refused before claim move to `withdrawn/`; a claim racing that rename retains +worker ownership. Admission keeps a tally of time per wait reason and reports the longest hold. Outside the minimum, new backend start admission requires `available − otherHolds − J ≥ R`, where `R = max(2 GiB, 0.15 × total)`. `J = 0.5 GiB` for Codex/Grok and `1 GiB` for Kimi/ZCode/unknown, held for 30 seconds without an diff --git a/plugins/relay/skills/jobs/orchestration.md b/plugins/relay/skills/jobs/orchestration.md index c61e5eb5..5d77a04c 100644 --- a/plugins/relay/skills/jobs/orchestration.md +++ b/plugins/relay/skills/jobs/orchestration.md @@ -28,7 +28,7 @@ outgrows the budget or finding counts keep rising. More rounds alone may not con | Claude-only side task | native `Agent` | one native agent | | Codex, Grok, ZCode or Kimi job needing messages | `relay:job` | one Haiku relay for the whole job | | Waves, branches, loops or recovery after caller loss | Workflow with `relay:job` | deterministic control flow and `resumeFromRunId` adoption | -| Wide fan-out with no relay steering needed | `C start` then `C wait ` | no Claude agent held per job; keep a waiter for deferred work | +| Wide fan-out with no relay steering needed | `C start` then `C wait ` | no Claude agent held per job; accepted jobs own their workers | A relay holds its Claude agent for the job's whole run, including queued waits, and spends some Claude usage. A Workflow slot stays occupied too. The documented per-run cap is up to 16 agents, @@ -62,9 +62,13 @@ probe goes first; a live probe is never replaced because of its age. These are b That job may finish `failed`; inspect its report and use same-job recovery where appropriate. Newly queued work keeps waiting. Acknowledgement of a message is not evidence its next turn ran. -`queued` is pending, never a result or a reason to submit the job again. Keep a wait/adopter: -`status` only reads; a wait, message or relay hook retries a workerless queue. Control Workflow jobs -by job id. Send a continue once, reconcile message records after uncertain errors, wait, then collect. +`queued` is pending under a live worker, never a result or a reason to submit the job again. +When the start queue is full, Workflow agents wait at spawn as queued, using no model tokens and +creating no job waiter until a place frees. A plain `relay:job` call waits up to 30 seconds then +reports capacity. Use a Workflow for large batches. A direct start without a worker after a minute +reports not started (exit 6); nothing runs later. Submit it again when capacity frees. +A dead launcher or an older workerless queue reads as dead; `wait --auto-resume` can recover it. +Control Workflow jobs by job id. Send a continue once, reconcile message records after uncertain errors, wait, then collect. At Claude's usage limit plain relays fail; Workflow runs pause where auto-continue is on. Backend jobs run on. Collect with `C wait ` or resume the Workflow with `resumeFromRunId`, never restart those jobs. Provider subscription windows and API dollar costs are different constraints; check diff --git a/tests/admission-e2e.test.mjs b/tests/admission-e2e.test.mjs index 034a725d..a2b3c400 100644 --- a/tests/admission-e2e.test.mjs +++ b/tests/admission-e2e.test.mjs @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import assert from "node:assert/strict"; import test from "node:test"; -import { setup, companion, startJob } from "./support/job-helpers.mjs"; +import { setup, companion, startJob, waitFor } from "./support/job-helpers.mjs"; import { DEFAULTS as D } from "../plugins/relay/scripts/lib/worker/admission.mjs"; import { relayContext, relayStart, jobDirOf, pre, stop, decision } from "./support/relay-fixture.mjs"; const GiB = 2 ** 30; @@ -12,45 +12,45 @@ function readings(env, root) { env.RELAY_ADMISSION_HOME = path.join(root, "domain"); env.RELAY_TEST_ADMISSION_READINGS = path.join(root, "readings.json"); env.RELAY_ADMISSION_POLL_MS = "25"; + env.RELAY_JOBS_ACK_TIMEOUT_MS = "200"; const write = availableBytes => fs.writeFileSync(env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify({ availableBytes, totalBytes: 128 * GiB, cores: 64, load5: 0 })); write(0); return write; } -test("start defers a worker at critical pressure; continue is durable, wait owns retry and cancel works", t => { +test("workerless starts and continuations refuse; an explicit resume after pressure clears runs the work", t => { const ctx = setup(t), write = readings(ctx.env, ctx.root); - const { id, start } = startJob(ctx); - assert.match(start, /state: queued/); + const file = path.join(ctx.root, "prompt.md"); + fs.writeFileSync(file, "model: synthetic-model\neffort: low\n---\n[quick] Initial brief."); + const start = companion(ctx, ["start", "--prompt-file", file, "--json"]); + assert.equal(start.status, 6, start.stdout + start.stderr); + const id = JSON.parse(start.stdout).jobId; const initial = JSON.parse(companion(ctx, ["status", id, "--json"]).stdout); - assert.equal(initial.state, "queued"); assert.equal(initial.attempts, 0); + assert.equal(initial.state, "failed"); assert.equal(initial.attempts, 0); assert.equal(fs.existsSync(path.join(ctx.bin, "fake-codex-state.json")), false); - const next = companion(ctx, ["continue", id, "--message", "[quick] Queued follow-up.", "--json"]); - assert.equal(next.status, 0, next.stderr); - assert.equal(JSON.parse(next.stdout).outcome.action, "queued-next-turn"); - const pending = companion(ctx, ["wait", id, "--timeout-seconds", "0", "--exit-zero-while-running"]); - assert.equal(pending.status, 0); assert.match(pending.stdout, /still running: not an error/); + const next = companion(ctx, ["continue", id, "--message", "[quick] Withdraw this.", "--json"]); + assert.equal(next.status, 6, next.stdout + next.stderr); + assert.equal(JSON.parse(next.stdout).outcome.action, "not-started"); write(100 * GiB); - const done = companion(ctx, ["wait", id, "--json", "--timeout-seconds", String((D.spacingMs * 3) / 1000 + 10)]); + assert.equal(companion(ctx, ["wait", id, "--timeout-seconds", "1"]).status, 6); + assert.equal(fs.existsSync(path.join(ctx.bin, "fake-codex-state.json")), false); + ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = "5000"; + assert.equal(companion(ctx, ["resume", id]).status, 0); + const done = companion(ctx, ["wait", id, "--timeout-seconds", "15", "--json"]); assert.equal(done.status, 0, done.stdout + done.stderr); - assert.equal(JSON.parse(done.stdout).attempts, 2); - write(0); - const second = startJob(ctx); - const cancelled = companion(ctx, ["cancel", second.id, "--json"]); - assert.equal(cancelled.status, 0, cancelled.stdout + cancelled.stderr); - assert.equal(JSON.parse(cancelled.stdout).state, "cancelled"); - const ended = JSON.parse(companion(ctx, ["status", second.id, "--json"]).stdout); - assert.equal(ended.admission.state, "cancelled"); - assert.ok(ended.admission.since); assert.match(ended.admission.reason, /memory/); + assert.equal(JSON.parse(done.stdout).state, "completed"); }); for (const workflow of [false, true]) test(`relay ${workflow ? "Workflow" : "Agent"} queued result never finishes before admission`, async () => { const ctx = relayContext("job", { workflow: workflow ? "synthetic-run" : null, turnMs: 50 }); // Fixture paths are beneath a synthetic temporary home. const write = readings(ctx.env, ctx.env.RELAY_JOBS_HOME); + write(128 * GiB * D.criticalFraction + 2 * D.workerBytes); + ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = "5000"; if (workflow) ctx.append(ctx.userRequest(), ctx.environment(), ctx.workflowPrompt("computedTask", "[quick] Queued relay."), ctx.snapshot()); else ctx.append(ctx.prompt(`model: synthetic-model\neffort: ${D.lightEfforts[0]}\n---\n[quick] Queued relay.`)); const { jobId: id, started } = await relayStart(ctx); assert.ok(id, started.stdout); - assert.equal(JSON.parse(fs.readFileSync(path.join(jobDirOf(ctx, id), "job.json"))).state, "queued"); + await waitFor(() => JSON.parse(fs.readFileSync(path.join(jobDirOf(ctx, id), "job.json"))).state === "queued"); assert.equal(decision(await pre(ctx, "SubagentHandback", { message: "done" })), "deny"); assert.equal((await stop(ctx, "done"))?.decision, "block"); write(100 * GiB); @@ -59,10 +59,10 @@ for (const workflow of [false, true]) test(`relay ${workflow ? "Workflow" : "Age assert.equal(JSON.parse(done.stdout).state, "completed"); }); -for (const json of [false, true]) test(`background task reports deferred state (${json ? "JSON" : "text"})`, { skip: !["linux", "darwin"].includes(process.platform) }, t => { +for (const json of [false, true]) test(`background task refuses without a worker (${json ? "JSON" : "text"})`, { skip: !["linux", "darwin"].includes(process.platform) }, t => { const ctx = setup(t); readings(ctx.env, ctx.root); const result = companion(ctx, ["task", "--background", ...(json ? ["--json"] : []), "[quick] Deferred task."]); - assert.equal(result.status, 0, result.stderr); - if (json) assert.equal(JSON.parse(result.stdout).state, "queued"); - else assert.match(result.stdout, /state: queued/); + assert.equal(result.status, 6, result.stderr); + if (json) assert.equal(JSON.parse(result.stdout).state, "not started"); + else assert.match(result.stdout, /state: not started/); }); diff --git a/tests/admission-surfaces.test.mjs b/tests/admission-surfaces.test.mjs index 8bb3eb52..e3a3ff88 100644 --- a/tests/admission-surfaces.test.mjs +++ b/tests/admission-surfaces.test.mjs @@ -60,8 +60,12 @@ test("corrupt stored settings normalize; JSON listing reads no sensors or lock", test("a worker-spawn failure is never retried as a guard failure", async t => { const { spawnWithAdmission } = await import("../plugins/relay/scripts/lib/worker/admission.mjs"); + const ctx = setup(t); + const { createJobDir, writeJob } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { jobDir } = createJobDir(ctx.work, "spawn-failure", ctx.env); + writeJob(jobDir, { id: "spawn-failure", state: "starting", request: {} }); let calls = 0; - assert.throws(() => spawnWithAdmission("/synthetic/absent", { RELAY_ADMISSION: "off" }, () => { calls++; throw new Error("synthetic spawn failure"); }), /synthetic spawn failure/); + assert.throws(() => spawnWithAdmission(jobDir, { RELAY_ADMISSION: "off" }, () => { calls++; throw new Error("synthetic spawn failure"); }), /synthetic spawn failure/); assert.equal(calls, 1); }); diff --git a/tests/admission-worker.test.mjs b/tests/admission-worker.test.mjs index c25c92b9..cfcdf37d 100644 --- a/tests/admission-worker.test.mjs +++ b/tests/admission-worker.test.mjs @@ -254,18 +254,13 @@ test("first dispatch consumes its start grant once and preserves the actual queu }); -test("dormant workerless jobs stop deferring the repository review gate after fifteen minutes", async t => { +test("workerless queues without a live launcher do not defer the review gate", async t => { const { findActiveJob } = await import("../plugins/relay/scripts/hooks/stop-review-gate.mjs"); const s = seed(t), oldHome = process.env.RELAY_JOBS_HOME; process.env.RELAY_JOBS_HOME = s.ctx.env.RELAY_JOBS_HOME; t.after(() => { if (oldHome === undefined) delete process.env.RELAY_JOBS_HOME; else process.env.RELAY_JOBS_HOME = oldHome; }); - const now = Date.now(), at = new Date(now).toISOString(); - writeJob(s.dir, { ...readJob(s.dir), state: "queued", worker: null, admission: { waitKind: "worker", since: at, workerRetryAt: at } }); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000).id, readJob(s.dir).id); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000 + 1), null); - const job = readJob(s.dir); - writeJob(s.dir, { ...job, admission: { ...job.admission, workerRetryAt: new Date(now + 15 * 60000 + 1).toISOString() } }); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000 + 1).id, job.id, "an adopter makes it active again"); + writeJob(s.dir, { ...readJob(s.dir), state: "queued", worker: null, admission: { waitKind: "worker" } }); + assert.equal(findActiveJob(s.ctx.work), null); }); test("a heartbeat during slow probe startup releases the claim without provider success", async t => { diff --git a/tests/admission.test.mjs b/tests/admission.test.mjs index 547ca8df..6f2ef305 100644 --- a/tests/admission.test.mjs +++ b/tests/admission.test.mjs @@ -280,6 +280,7 @@ test("domain ignores custom jobs homes; spawn reservations bound waiting workers test("Windows admission is a visible no-op even with stale files and failed sensors", t => { const s = setup(t), c = s.ctx(); + writeJob(c.jobDir, { id: c.jobId, state: "starting", request: c.request }); const before = fs.readdirSync(s.root); const deps = { ...s.deps, platform: "win32", readAdmissionConfig() { throw new Error("no config read"); }, readMachine() { throw new Error("no sensor read"); } }; assert.match(admit(c, deps).record.notes.join(" "), /unsupported on win32/); @@ -1059,6 +1060,7 @@ test("the environment off switch survives configuration read errors", t => { const s = setup(t), c = s.ctx(), env = { ...s.env, RELAY_ADMISSION: "off" }; const deps = { ...s.deps, env, readAdmissionConfig: () => { throw new Error("unreadable configuration"); } }; assert.equal(admit(c, deps).admit, true); + writeJob(c.jobDir, { id: c.jobId, state: "starting", request: c.request }); assert.equal(spawnWithAdmission(c.jobDir, env, () => 42, deps), 42); }); @@ -1132,3 +1134,36 @@ test("a burst can reserve other seats while the first launch handshake is still assert.equal(sleeps, 0); assert.ok(jobs.every(c => readJob(c.jobDir).admission?.causeKind !== "lock")); }); + +test('admitted records retain the longest hold instead of the last short pacing hold', t => { + const s = setup(t), c = s.ctx({ model: 'synthetic-model' }); + s.setMachine({ availableBytes: 0 }); + let result = admit(c, s.deps); + c.admission = result.record; c.waitingSince = result.record.since; + assert.equal(result.waitKind, 'critical'); + s.tick(120000); s.setMachine({ availableBytes: 100 * GiB }); + fs.writeFileSync(path.join(s.home, 'timing.json'), JSON.stringify({ lastStart: new Date(s.deps.now()).toISOString() })); + result = admit(c, s.deps); assert.equal(result.waitKind, 'stagger'); c.admission = result.record; + s.tick(D.spacingMs); result = admit(c, s.deps); + assert.equal(result.admit, true); assert.equal(result.record.waitKind, 'critical'); + assert.match(result.record.reason, /memory/); + assert.equal(Object.values(result.record.waitTotals).find(x => x.kind === 'critical').ms, 120000); + assert.equal(Object.values(result.record.waitTotals).find(x => x.kind === 'stagger').ms, D.spacingMs); +}); + +test('slot cleaning and release require fresh live execution evidence even with a busy process cache', async t => { + const { executionAlive } = await import('../plugins/relay/scripts/lib/worker/admission.mjs'); + const s = setup(t), c = s.ctx(), dir = c.jobDir, file = path.join(s.home, 'slots', `${c.jobId}.json`); + const leader = { pid: 600000000, startTime: 'linux:100' }; + const tool = { pid: 600000001, ppid: 1, pgid: leader.pid, startTime: 'linux:200' }; + writeJob(dir, { id: c.jobId, request: c.request, worker: leader, server: tool, state: 'running', execution: { leader, pgid: leader.pid } }); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ v: 1, jobId: c.jobId, jobDir: dir, pid: leader.pid, processStart: leader.startTime, + backend: 'codex', class: 'light', state: 'running', since: new Date(s.deps.now()).toISOString() })); + fs.writeFileSync(path.join(s.home, '.ps-table.json'), JSON.stringify({ at: Date.now() - 60000, rows: [] })); + const lock = path.join(s.home, '.ps-table.json.lock'); assert.ok(acquireOwnership(lock)); t.after(() => releaseOwnership(lock)); + const deps = { ...s.deps, listProcesses: () => [tool] }; + assert.equal(executionAlive({ ...readJob(dir), jobDir: dir }, deps), true); + assert.equal(describeGuard(s.env, {}, deps).activeSeats, 1); assert.equal(fs.existsSync(file), true); + assert.equal(releaseSlot(s.home, c.jobId, leader, deps), false); assert.equal(fs.existsSync(file), true); +}); diff --git a/tests/commands.test.mjs b/tests/commands.test.mjs index dbc9f1a5..c61140c1 100644 --- a/tests/commands.test.mjs +++ b/tests/commands.test.mjs @@ -253,7 +253,7 @@ test("the jobs skill and design.md say a message that arrives while a turn is st test("dead and exit 3 are defined as view.mjs computes them, and the continue recipe waits within the Bash timeout", () => { const skill = flat(read("skills/jobs/SKILL.md")); - for (const fact of ["none started within 60 s of the launch", "except a workerless job queued for admission", "(even with a report)", "`completed` with no report", + for (const fact of ["its launch process ended before ownership", "not started (exit 6)", "(even with a report)", "`completed` with no report", "`completed` is only the first gate"]) { assert.ok(skill.includes(fact), fact); } @@ -265,7 +265,7 @@ test("dead and exit 3 are defined as view.mjs computes them, and the continue re const recipe = section(skill, "### Continue a job by its id", "How the hook carries the relay"); for (const fact of ["Exit `4`", "acknowledgement window (60 s)", "`skipped`", "--timeout-seconds 100", "the Bash tool's default timeout of 120 s", "the `undelivered` list", "After a `resume`, go back to step 2.", - "`report: none` is no new report", "Absence of a receipt alone proves nothing", "after any nonzero exit", "`queued-next-turn` with a queue `reason:`"]) { + "`report: none` is no new report", "Absence of a receipt alone proves nothing", "after any nonzero exit", "Only a live worker writes `queued-next-turn` or `accepted-next-turn`"]) { assert.ok(recipe.includes(fact), fact); } assert.doesNotMatch(recipe, /`no-op`/); @@ -510,7 +510,7 @@ test("README carries the first hour, stop gate, Claude Code dependency and devel assert.match(source, /A Claude Code update can break it\./); assert.ok(source.includes("the job's `job.json` must hold the call's key as `relay.journalKey`")); assert.ok(source.includes("once another relay has adopted the job, its `relay.json` holds the binding")); - assert.match(source, /The module only checks headers and wakes a waiting relay: delivery runs on command hooks and the transcript/); + assert.match(source, /The module checks headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript/); } assert.ok(internals.includes("stores that key as `relay.journalKey` in the job's `job.json`")); assert.doesNotMatch(internals, /binds the job to that key in the job's `relay\.json`/); diff --git a/tests/execution-groups.test.mjs b/tests/execution-groups.test.mjs index abcef172..189f7d79 100644 --- a/tests/execution-groups.test.mjs +++ b/tests/execution-groups.test.mjs @@ -5,7 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { spawn, spawnSync } from "node:child_process"; import { once } from "node:events"; -import { setup, startJob, readJob, jobDir, companion, waitFor, finishJob, ROOT } from "./support/job-helpers.mjs"; +import { setup, startJob, promptFile, jobDirs, readJob, jobDir, companion, waitFor, finishJob, ROOT } from "./support/job-helpers.mjs"; import { listProcesses, executionMembers, sampleExecution, stopExecution, readProcessStartTime, isProcessIdentityAlive, readProcessTableCached } from "../plugins/relay/scripts/lib/util/process.mjs"; import { describeGuard, renderGuard } from "../plugins/relay/scripts/lib/worker/admission.mjs"; import { writeJob, readJob as readRecord } from "../plugins/relay/scripts/lib/jobs/store.mjs"; @@ -176,7 +176,19 @@ for (const phase of ["running", "admission", "recovery"]) test(`a long-lived wai ctx.env.RELAY_TEST_ADMISSION_READINGS = fixture; fs.writeFileSync(fixture, JSON.stringify({ totalBytes: 16 * 2 ** 30, availableBytes: 0, load5: 0, cores: 8 })); } - const { id } = startJob(ctx, undefined, "[long] Work"); + let id; + if (phase === "recovery") ({ id } = startJob(ctx, undefined, "[long] Work")); + else { + // Model a launcher interrupted during its bounded start window, before receipt. + const { createJobDir, jobPaths } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { buildLaunchRequest } = await import("../plugins/relay/scripts/lib/jobs/request.mjs"); + id = `job-deferred-${phase}`; + const { jobDir: dir } = createJobDir(ctx.work, id, ctx.env); + writeJob(dir, { id, kind: "task", state: "starting", createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: buildLaunchRequest({ model: "gpt-test", effort: "low", cd: ctx.work }, { baseDir: ctx.work, env: ctx.env }) }); + fs.writeFileSync(jobPaths(dir).brief, "[long] Work"); + assert.equal(spawnWorker(dir, ctx.work, ctx.env), null); + } const dir = jobDir(ctx, id); let old = readJob(ctx, id); if (phase === "recovery") { @@ -254,7 +266,8 @@ if(process.argv[1]?.endsWith('/launcher.mjs')) { // A pending handoff remains retryable. Concurrent waiters use the same lease fence. const waiters = Array.from({ length: 3 }, () => spawn(process.execPath, [script, "wait", id, "--timeout-seconds", "30"], { env: ctx.env, cwd: ctx.work, stdio: "ignore" })); t.after(() => { for (const waiter of waiters) { try { waiter.kill("SIGKILL"); } catch {} } }); - await waitFor(() => readJob(ctx, id).worker?.pid && readJob(ctx, id).launch?.state === "acknowledged", 20000); + try { await waitFor(() => readJob(ctx, id).worker?.pid && readJob(ctx, id).launch?.state === "acknowledged", 20000); } + catch (error) { const job = readJob(ctx, id); throw new Error(`${error.message}: ${JSON.stringify({ state: job.state, worker: job.worker, admission: job.admission, launch: job.launch, error: job.errorMessage, intents: fs.readdirSync(dir).filter(name => name.startsWith("launch-")).map(name => JSON.parse(fs.readFileSync(path.join(dir, name)))) })}`); } assert.equal(readJob(ctx, id).workers.filter(w => w.pid !== old.worker.pid).length, 1); assert.ok(readJob(ctx, id).launches.some(launch => launch.state === "death-observed" && launch.deathObservedAt && !launch.endedAt)); }); @@ -414,20 +427,19 @@ test("a nested worker and its backend are excluded even if sampled during handof assert.deepEqual(result.survivors, []); }); -test("a failed launch stays queued and plain wait dispatches it without auto-resume", async t => { - const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: "off" } }); +for (const command of ["start", "task"]) test(`a permanently failing ${command} launcher stops after three failures and exposes its error`, async t => { + const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: "off", RELAY_ADMISSION_POLL_MS: "20" } }); const preload = path.join(ctx.root, "fail-launch.mjs"); - fs.writeFileSync(preload, `if(process.argv[1]?.endsWith('/launcher.mjs')) process.exit(1);`); - const { id } = startJob(ctx, undefined, "[quick] Retry the launch.", { NODE_OPTIONS: `--import=${preload}` }); - const queued = readJob(ctx, id); - assert.equal(queued.state, "queued"); - assert.equal(queued.admission.causeKind, "launch"); - assert.equal(queued.admission.waitKind, "worker"); - assert.equal(companion(ctx, ["wait", id, "--timeout-seconds", "15"]).status, 0); - await finishJob(ctx, id); - assert.equal(readJob(ctx, id).attempts.length, 1); - assert.equal(readJob(ctx, id).workers.length, 1); - assert.equal(readJob(ctx, id).autoRecoveries?.length ?? 0, 0); + fs.writeFileSync(preload, `if(process.argv[1]?.endsWith('/launcher.mjs')) { console.error('synthetic launch failure'); process.exit(1); }`); + const result = companion(ctx, command === "start" ? ["start", "--prompt-file", promptFile(ctx)] : ["task", "[quick] Dummy task."], { NODE_OPTIONS: `--import=${preload}` }); + assert.equal(result.status, 6, result.stdout + result.stderr); + const dir = jobDirs(ctx)[0], failed = JSON.parse(fs.readFileSync(path.join(dir, "job.json"))); + assert.equal(failed.admission.launchFailures, 3); assert.match(failed.admission.launchError, /synthetic launch failure/); + const waited = companion(ctx, ["wait", failed.id, "--timeout-seconds", "1"]); + assert.equal(waited.status, 6); assert.match(waited.stdout, /3 failed launches of 3.*synthetic launch failure/); + assert.equal(fs.readdirSync(dir).filter(name => name.startsWith("launch-")).length, 3); + assert.equal(companion(ctx, ["resume", failed.id, "--message", "[quick] Retry explicitly."]).status, 0); + await finishJob(ctx, failed.id); }); for (const cancel of [false, true]) test(`an actual nested task survives outer cleanup after sampling its launcher handoff (cancel=${cancel})`, async t => { @@ -435,7 +447,7 @@ for (const cancel of [false, true]) test(`an actual nested task survives outer c const inner = setup(t, { turnMs: 30000, env: { RELAY_WORKER_SCOPE: "off" } }); const shared = path.join(outer.root, "admission"); outer.env.RELAY_ADMISSION_HOME = shared; - inner.env.RELAY_ADMISSION_HOME = shared; + inner.env.RELAY_ADMISSION_HOME = cancel ? "/dev/null/unwritable" : shared; const marker = path.join(outer.root, "nested-started"), blocked = path.join(inner.root, "blocked"), release = path.join(inner.root, "release"); const preload = path.join(inner.root, "hold-launcher.mjs"); fs.writeFileSync(preload, `import fs from 'node:fs'; @@ -486,7 +498,7 @@ test("scope cleanup excludes older members and preserves nested workers sharing const scopeKills = [], signals = []; t.mock.method(fs, "readFileSync", (file, ...args) => { if (file === "/proc/sys/kernel/random/boot_id") return "current-boot\n"; - if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return `${older.pid}\n${worker.pid}\n${tool.pid}\n`; + if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return table.map(row => row.pid).join("\n"); return read(file, ...args); }); t.mock.method(fs, "statSync", (file, ...args) => file === `/sys/fs/cgroup${group}` ? { ino: 37 } : stat(file, ...args)); @@ -502,3 +514,68 @@ test("scope cleanup excludes older members and preserves nested workers sharing assert.deepEqual(scopeKills, []); assert.deepEqual(result.survivors, []); }); + +test('scope-only pids absent from the process table remain unresolved and keep their seat', async t => { + const { executionAlive } = await import('../plugins/relay/scripts/lib/worker/admission.mjs'); + const group = '/synthetic/relay-jobs.slice/run-unresolved.scope', leader = { pid: 600000000, startTime: 'linux:100' }, pid = 600000001; + const read = fs.readFileSync, stat = fs.statSync; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === '/proc/sys/kernel/random/boot_id') return 'synthetic-boot\n'; + if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return `${pid}\n`; + return read(file, ...args); + }); + t.mock.method(fs, 'statSync', (file, ...args) => file === `/sys/fs/cgroup${group}` ? { ino: 37 } : stat(file, ...args)); + const job = { worker: leader, execution: { leader, pgid: leader.pid, cgroup: group, cgroupId: 37, bootId: 'synthetic-boot' } }; + const members = executionMembers(job, []); + assert.deepEqual([...members], [{ pid, startTime: null, identityUnknown: true }]); + assert.match(members.cause, /no process identity/); + assert.equal(executionAlive(job, { env: {}, listProcesses: () => [], isAlive: () => false }), true); +}); + +test('launch history and crash registry maintenance stay bounded without scanning history on sample', async t => { + const { pruneLaunches } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const { pruneWorkerRegistry } = await import('../plugins/relay/scripts/lib/worker/registry.mjs'); + const ctx = setup(t), directory = path.join(ctx.root, 'launch-history'), registry = path.join(ctx.root, 'registry-history'); + fs.mkdirSync(directory); fs.mkdirSync(registry); + for (let i = 0; i < 100; i++) { + fs.writeFileSync(path.join(directory, `launch-${i}.json`), JSON.stringify({ state: 'acknowledged', requestedAt: new Date(i * 1000).toISOString() })); + fs.writeFileSync(path.join(registry, `${600000000 + i}.json`), JSON.stringify({ pid: 600000000 + i, startTime: 'gone', jobDir: '/synthetic/other' })); + } + pruneLaunches(directory); assert.equal(fs.readdirSync(directory).length, 20); + let checks = 0; + while (pruneWorkerRegistry(registry, { limit: 16, isAlive: () => { checks++; return false; } })) {} + assert.equal(checks, 100); assert.equal(fs.readdirSync(registry).filter(name => /^\d/.test(name)).length, 0); + const readdir = fs.readdirSync; + t.mock.method(fs, 'readdirSync', (dir, ...args) => { assert.notEqual(dir, registry, 'sampling cannot enumerate historical workers'); return readdir(dir, ...args); }); + const worker = { pid: 600000001, startTime: 'linux:200', ppid: 1, pgid: 600000001 }; + fs.writeFileSync(path.join(registry, `${worker.pid}.json`), JSON.stringify({ ...worker, jobDir: '/synthetic/other' })); + assert.deepEqual(executionMembers({ jobDir: directory, server: worker, execution: { leader: worker, pgid: worker.pid, workerRegistry: registry } }, [worker]), []); +}); + +test('caller acknowledgement observation timeout never erases a live dispatched worker', async t => { + const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: 'off' } }); + const { createJobDir, jobPaths } = await import('../plugins/relay/scripts/lib/jobs/store.mjs'); + const { buildLaunchRequest } = await import('../plugins/relay/scripts/lib/jobs/request.mjs'); + const { startWithinWindow } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const { jobDir: dir } = createJobDir(ctx.work, 'job-delayed-observation', ctx.env); + writeJob(dir, { id: 'job-delayed-observation', kind: 'task', state: 'starting', createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: buildLaunchRequest({ model: 'gpt-test', effort: 'low', cd: ctx.work }, { baseDir: ctx.work, env: ctx.env }) }); + fs.writeFileSync(jobPaths(dir).brief, '[held] Wait for observation.'); + const read = fs.readFileSync, now = Date.now; let offset = 0, delayed = false; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + const result = read(file, ...args); + if (!delayed && String(file).startsWith(path.join(dir, 'launch-')) && String(result).includes('"acknowledged"')) { + delayed = true; offset = 11000; + return JSON.stringify({ ...JSON.parse(String(result)), state: 'pending' }); + } + return result; + }); + t.mock.method(Date, 'now', () => now() + offset); + const result = await startWithinWindow(dir, ctx.work, ctx.env, { windowMs: 20000 }); + assert.equal(delayed, true); assert.equal(result.accepted, true); + const job = JSON.parse(fs.readFileSync(path.join(dir, 'job.json'))); + assert.ok(job.worker?.pid); assert.notEqual(job.state, 'queued'); + await waitFor(() => ctx.fakeState().appServerStarts === 1); + assert.equal(job.workers.length, 1); assert.equal(ctx.fakeState().appServerStarts, 1); + assert.equal(fs.readdirSync(dir).filter(name => /^launch-/.test(name)).length, 1); +}); diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs new file mode 100644 index 00000000..aec805f6 --- /dev/null +++ b/tests/gate.test.mjs @@ -0,0 +1,221 @@ +import './support/isolate-env.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { setup, companion, promptFile, jobDirs, waitFor, ROOT } from './support/job-helpers.mjs'; +import { reserveWaitingPlace, bindReservation, readReservation, describeGuard, spawnWithAdmission, admit, DEFAULTS as D } from '../plugins/relay/scripts/lib/worker/admission.mjs'; +import { readJob, writeJob, jobPaths } from '../plugins/relay/scripts/lib/jobs/store.mjs'; +import { computeJobView, loadJobView, exitCodeFor } from '../plugins/relay/scripts/lib/jobs/view.mjs'; +import { hasLiveWorker, startWithinWindow, awaitOutcome } from '../plugins/relay/scripts/lib/worker/delivery.mjs'; +import { acquireWorkerLease, releaseWorkerLease } from '../plugins/relay/scripts/lib/jobs/lease.mjs'; +import { claimInboxMessages, messageAccount, postInboxMessage, writeOutcome } from '../plugins/relay/scripts/lib/jobs/messages.mjs'; +import { readProcessStartTime } from '../plugins/relay/scripts/lib/util/process.mjs'; +import { resumeDeadJobOnce } from '../plugins/relay/scripts/lib/worker/recovery.mjs'; + +function domain(t, limit = 2) { + const ctx = setup(t, { env: { RELAY_ADMISSION: 'on', RELAY_MAX_WAITING_WORKERS: String(limit), RELAY_ADMISSION_POLL_MS: '20', RELAY_JOBS_ACK_TIMEOUT_MS: '200' } }); + ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root, 'admission'); + ctx.env.RELAY_TEST_ADMISSION_READINGS = path.join(ctx.root, 'readings.json'); + const machine = { totalBytes: 18 * 2**30, availableBytes: 16 * 2**30, cores: 12, load5: 0 }; + fs.writeFileSync(ctx.env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify(machine)); + let now = Date.now(); + const deps = { now: () => now, readMachine: () => machine, scanUnslotted: () => [], env: ctx.env }; + return { ctx, deps, tick: (ms) => now += ms, machine }; +} +function job(s, id = 'job-window', state = 'starting') { + const dir = path.join(s.ctx.jobs, 'repo', id); fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(jobPaths(dir).brief, '[quick] Dummy job.'); + writeJob(dir, { id, kind: 'task', state, createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: { backend: 'codex', cwd: s.ctx.work, model: 'synthetic-model', effort: 'low', notes: [] } }); + return dir; +} + +test('gate reserves Q places, binds once, converts without double counting, and expires after 45 seconds', t => { + const s = domain(t), a = reserveWaitingPlace(s.ctx.env, s.deps), b = reserveWaitingPlace(s.ctx.env, s.deps); + assert.ok(a.pass && b.pass); assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, false); + assert.equal(bindReservation(a.reservation, 'a-first', s.ctx.env, s.deps), true); + assert.equal(readReservation('a-first', s.ctx.env), a.reservation); + assert.equal(bindReservation(a.reservation, 'a-other', s.ctx.env, s.deps), false); + const dir = job(s); + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => 123, s.deps, { reservation: a.reservation }), 123); + const files = fs.readdirSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots')); + assert.equal(files.length, 2); assert.ok(files.includes('job-window.json')); + writeJob(dir, { ...readJob(dir), state: 'completed' }); + s.tick(D.reservationMs + 1); + assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, true); +}); + +test('guard-off gate passes without reading or writing admission storage', t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_ADMISSION_HOME = '/dev/null/unwritable'; + assert.deepEqual(reserveWaitingPlace(s.ctx.env, s.deps), { pass: true, reservation: null }); + assert.equal(companion(s.ctx, ['gate']).stdout, 'pass\n'); +}); + +test('window refusal exits 6 and readers never resurrect its brief after capacity frees', t => { + const s = domain(t, 0); + const result = companion(s.ctx, ['start', '--prompt-file', promptFile(s.ctx)]); + assert.equal(result.status, 6, result.stderr + result.stdout); + assert.match(result.stdout, /state: not started.*\nerror: Not started: waiting worker limit: 0 of 0/); + const dir = jobDirs(s.ctx)[0], refused = readJob(dir); + assert.equal(refused.state, 'failed'); assert.equal(refused.admission.state, 'not-started'); assert.equal(hasLiveWorker(dir), false); + assert.equal(fs.existsSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `${refused.id}.json`)), false); + s.ctx.env.RELAY_MAX_WAITING_WORKERS = '4'; + for (const argv of [['status', refused.id], ['result', refused.id], ['wait', refused.id, '--timeout-seconds', '1'], ['status', '--json', '--all']]) { + const read = companion(s.ctx, argv); assert.doesNotMatch(read.stderr, /spawn|launch failed/); + } + assert.equal(readJob(dir).attempts.length, 0); +}); + +test('pressure clearing inside the start window acknowledges a real live worker lease', async t => { + const s = domain(t), dir = job(s); s.machine.availableBytes = 0; + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + let attempts = 0; + const result = await startWithinWindow(dir, s.ctx.work, s.ctx.env, { windowMs: 200, spawn: () => { + attempts++; + if (attempts === 1) return spawnWithAdmission(dir, s.ctx.env, () => assert.fail('pressure'), s.deps); + s.machine.availableBytes = 16 * 2**30; + assert.equal(acquireWorkerLease(dir, me), true); + } }); + assert.equal(result.accepted, true); assert.equal(hasLiveWorker(dir), true); releaseWorkerLease(dir, me); +}); + +test('a killed launcher is dead rather than an ownerless durable queue', async t => { + const s = domain(t, 0); s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '60000'; + const child = spawn(process.execPath, [path.join(ROOT, 'plugins/relay/scripts/companion.mjs'), 'start', '--prompt-file', promptFile(s.ctx)], { cwd: s.ctx.work, env: s.ctx.env, stdio: 'ignore' }); + t.after(() => { try { child.kill('SIGKILL'); } catch {} }); + const dir = await waitFor(() => jobDirs(s.ctx).find(dir => readJob(dir).admission?.launcher)); + const ended = once(child, 'exit'); child.kill('SIGKILL'); await ended; + const view = computeJobView(readJob(dir)); assert.equal(view.state, 'dead'); assert.match(view.deadReason, /process starting it ended/); +}); + +test('a completed job continuation without a place is withdrawn and never runs later', async t => { + const s = domain(t, 0), dir = job(s, 'job-completed', 'completed'); + const message = postInboxMessage(dir, { kind: 'continue', text: 'Never run this.' }); + const relay = path.join(jobPaths(dir).inbox, 'relay'); fs.mkdirSync(relay); + fs.writeFileSync(path.join(relay, 'entry.json'), JSON.stringify({ messageId: message.id })); + const result = await awaitOutcome(dir, message.id, { env: s.ctx.env, timeoutMs: 100 }); + assert.equal(result.consumer, false); assert.equal(result.outcome.action, 'not-started'); + assert.ok(fs.existsSync(path.join(dir, 'withdrawn', `${message.id}.json`))); + assert.equal(fs.existsSync(path.join(relay, 'entry.json')), false); + assert.deepEqual(messageAccount(dir).undelivered, []); + s.ctx.env.RELAY_MAX_WAITING_WORKERS = '4'; + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => assert.fail('withdrawn message ran'), s.deps), null); +}); + +test('a worker claim beating withdrawal delivers once rather than refusing the claimed message', async t => { + const s = domain(t, 0), dir = job(s, 'job-race', 'completed'); + const msg = postInboxMessage(dir, { kind: 'continue', text: 'Once.' }); + const rename = fs.renameSync; let claims = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === path.join(dir, 'withdrawn', `${msg.id}.json`)) { + claims += claimInboxMessages(dir).length; + writeOutcome(dir, msg.id, { action: 'turn-started', turnId: 'synthetic-turn' }); + } + return rename(from, to); + }); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, timeoutMs: 0 }); + assert.equal(result.consumer, true); assert.equal(result.outcome.action, 'turn-started'); assert.equal(claims, 1); +}); + +test('recovery with no place preserves dead state and retries the same reserved message', async t => { + const s = domain(t, 0), dir = job(s, 'job-recovery', 'running'); + writeJob(dir, { ...readJob(dir), worker: { pid: 2147483647, startTime: 'gone' } }); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), false); + const first = readJob(dir); assert.equal(computeJobView(first).state, 'dead'); assert.equal(first.automaticResumes[0].state, 'reserved'); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), false); + assert.equal(readJob(dir).automaticResumes[0].id, first.automaticResumes[0].id); + assert.equal(fs.readdirSync(jobPaths(dir).inbox).filter(n => n.endsWith('.json')).length, 1); +}); + +test('33 gated submissions reach backend admission with no refused or unowned work', t => { + const s = domain(t, 20), waiting = [], running = [], me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + let submitted = 0, admitted = 0, held = 0; + while (admitted < 33) { + while (submitted < 33) { + const place = reserveWaitingPlace(s.ctx.env, s.deps); + if (!place.pass) { held++; break; } + const dir = job(s, `job-floor-${++submitted}`); + const request = { ...readJob(dir).request, model: submitted <= 30 ? 'synthetic-sol' : 'synthetic-astra', effort: 'high' }; + writeJob(dir, { ...readJob(dir), request }); + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => 1, s.deps, { reservation: place.reservation }), 1); + // A synthetic worker owns each waiting slot while the fake backend is gated. + const file = path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `${readJob(dir).id}.json`); + const slot = JSON.parse(fs.readFileSync(file)); fs.writeFileSync(file, JSON.stringify({ ...slot, pid: me.pid, processStart: me.startTime, state: 'waiting' })); + waiting.push({ home: s.ctx.env.RELAY_ADMISSION_HOME, jobDir: dir, jobId: readJob(dir).id, me, request }); + } + for (const ctx of [...waiting]) { + const result = admit(ctx, s.deps); ctx.admission = result.record; ctx.waitingSince = result.record.since; + if (result.admit) { waiting.splice(waiting.indexOf(ctx), 1); running.push(ctx); admitted++; } + } + s.tick(7000); + // Fake remote work uses little resident memory; release startup holds naturally. + assert.ok(admitted + waiting.length === submitted); + } + assert.equal(submitted, 33); assert.ok(held > 0); assert.equal(running.length, 33); + for (const ctx of running) writeJob(ctx.jobDir, { ...readJob(ctx.jobDir), state: 'completed' }); +}); + +test('foreground task uses a detached worker rather than the waiting caller', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; + const script = path.join(ROOT, 'plugins/relay/scripts/companion.mjs'); + const caller = spawn(process.execPath, [script, 'task', '[quick] Dummy.'], { cwd: s.ctx.work, env: s.ctx.env, stdio: 'ignore' }); + const exited = once(caller, 'exit'); + t.after(() => { try { caller.kill('SIGKILL'); } catch {} }); + const dir = await waitFor(() => jobDirs(s.ctx).find(dir => readJob(dir).worker?.pid)); + assert.notEqual(readJob(dir).worker.pid, caller.pid); assert.equal(hasLiveWorker(dir), true); + await exited; +}); + +test('a continuation interrupted at handoff is named as owed on the finished job and can retry', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; + const started = companion(s.ctx, ['start', '--prompt-file', promptFile(s.ctx), '--json']); + assert.equal(started.status, 0, started.stdout + started.stderr); + const id = JSON.parse(started.stdout).jobId; + assert.equal(companion(s.ctx, ['wait', id, '--timeout-seconds', '15']).status, 0); + const dir = jobDirs(s.ctx)[0]; + await waitFor(() => !hasLiveWorker(dir)); + const msg = postInboxMessage(dir, { kind: 'continue', text: '[quick] Interrupted handoff.' }); + fs.writeFileSync(path.join(dir, 'launch-interrupted.json'), JSON.stringify({ state: 'handoff-failed', requestedAt: new Date(Date.now() + 1).toISOString(), worker: { pid: 2147483647, startTime: 'gone' } })); + const view = loadJobView(dir); assert.ok(view.undelivered.includes(msg.id)); + const block = companion(s.ctx, ['result', id, '--state-only']); assert.match(block.stdout, new RegExp(msg.id)); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, wait: false, timeoutMs: 5000 }); + assert.equal(result.consumer, true); + assert.equal(companion(s.ctx, ['wait', id, '--timeout-seconds', '15']).status, 0); + assert.equal(readJob(dir).attempts.filter(a => a.messageId === msg.id).length, 1); +}); + +test('a message racing warm-worker exit redispatches promptly after the lease disappears', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_ADMISSION_POLL_MS = '3000'; + const dir = job(s, 'job-warm-exit', 'completed'); + const leaseModule = new URL('../plugins/relay/scripts/lib/jobs/lease.mjs', import.meta.url).href; + const processModule = new URL('../plugins/relay/scripts/lib/util/process.mjs', import.meta.url).href; + const warm = spawn(process.execPath, ['--input-type=module', '-e', `import {acquireWorkerLease,releaseWorkerLease} from ${JSON.stringify(leaseModule)}; +import {readProcessStartTime} from ${JSON.stringify(processModule)}; +const dir=${JSON.stringify(dir)}, me={pid:process.pid,startTime:readProcessStartTime(process.pid)}; +acquireWorkerLease(dir,me); console.log(JSON.stringify(me)); +setTimeout(()=>releaseWorkerLease(dir,me),100); setTimeout(()=>process.exit(0),300);`], { cwd: s.ctx.work, env: s.ctx.env, stdio: ['ignore', 'pipe', 'inherit'] }); + t.after(() => { try { warm.kill('SIGKILL'); } catch {} }); + const identity = JSON.parse(String((await once(warm.stdout, 'data'))[0])); + fs.writeFileSync(path.join(dir, 'launch-warm.json'), JSON.stringify({ state: 'acknowledged', worker: identity, requestedAt: new Date().toISOString() })); + writeJob(dir, { ...readJob(dir), worker: identity }); + const msg = postInboxMessage(dir, { kind: 'continue', text: '[quick] Warm-exit message.' }); + const begin = Date.now(); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, timeoutMs: 5000 }); + assert.equal(result.consumer, true); assert.match(result.outcome?.action, /^(queued-next-turn|turn-started)$/); assert.notEqual(readJob(dir).worker?.pid, identity.pid); assert.ok(Date.now() - begin < 2800, 'redispatch must not wait for the old five-second timer'); +}); + +test('a dispatch arriving after refusal cannot acquire the worker lease or run the brief', async t => { + const s = domain(t, 0), dir = job(s); + await startWithinWindow(dir, s.ctx.work, s.ctx.env, { windowMs: 0 }); + assert.equal(readJob(dir).admission.state, 'not-started'); + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + assert.equal(acquireWorkerLease(dir, me), false); + assert.equal(hasLiveWorker(dir), false); + const explicit = postInboxMessage(dir, { kind: 'resume', text: '[quick] Submit again explicitly.' }); + assert.equal(acquireWorkerLease(dir, me), true); releaseWorkerLease(dir, me); + // The assertion fixture has no actual worker; close the explicit message for teardown. + claimInboxMessages(dir); writeOutcome(dir, explicit.id, { action: 'skipped' }); +}); diff --git a/tests/harness-workflow.test.mjs b/tests/harness-workflow.test.mjs index 2f34d3ab..baa06ce7 100644 --- a/tests/harness-workflow.test.mjs +++ b/tests/harness-workflow.test.mjs @@ -84,3 +84,61 @@ test("readWorkflowRunId returns null for absent or unreadable transcripts and ab } assert.equal(readWorkflowRunId(transcript([WORKFLOW_RESULT]), undefined), null); }); + +// Exercise the module's admission hooks with the host API faked, including retries +// that enter the first step without another agent.spawn event. +import { register as registerHooks } from 'node:module'; +const modStub = 'data:text/javascript,' + encodeURIComponent('export const atom = (key, initial) => ({key, initial}); export const read = async () => null; export const update = async () => {};'); +registerHooks('data:text/javascript,' + encodeURIComponent(`export async function resolve(s, c, next) { return s === 'claude-code' ? {url: ${JSON.stringify(modStub)}, shortCircuit: true} : next(s, c); }`)); +const { register } = await import('../plugins/relay/hooks/mod/register.js'); +function hooks(answers) { + const handlers = {}, catches = {}, runs = [], calls = []; + register((event, filter, handler) => { handlers[event] = handler ?? filter; return { catch(fn) { catches[event] = fn; return this; } }; }); + const $ = { plugin: { root: '/synthetic/plugin' }, session: { cwd: async () => '/synthetic/work' }, + process: { run: async (argv, init) => { + runs.push({ argv, init }); + if (argv[2] === 'check-header') return { exitCode: 0, stdout: 'ok\n' }; + if (argv.includes('--bind')) return { exitCode: 0, stdout: 'bound\n' }; + const answer = answers.shift(); if (answer instanceof Error) throw answer; + return { exitCode: 0, stdout: answer ?? 'pass\n' }; + } } }; + const next = async e => { calls.push(e); return { agentId: 'a-new' }; }; + return { handlers, catches, runs, calls, $, next }; +} +const spawnEvent = { subagentType: 'relay:job', prompt: 'model: synthetic\neffort: low\n---\nDummy.' }; + +test('Workflow spawn holds through host waits and binds its reservation before the wrapper runs', async () => { + const id = '00000000-0000-4000-8000-000000000001'; + const h = hooks(['hold waiting worker limit: 2 of 2', 'hold waiting worker limit: 2 of 2', `pass ${id}`]); + assert.deepEqual(await h.handlers['agent.spawn'](h.$, { ...spawnEvent, workflow: { runId: 'wf-one', agentIndex: 1 } }, h.next), { agentId: 'a-new' }); + assert.equal(h.calls.length, 1); + const gate = h.runs.filter(r => r.argv[2] === 'gate'); assert.equal(gate.length, 4); + assert.equal(gate[0].init.timeoutMs, 600000); assert.ok(gate[0].argv.includes('540')); + assert.deepEqual(gate.at(-1).argv.slice(3), ['--bind', id, '--agent', 'a-new']); + assert.equal(typeof h.catches['agent.spawn'], 'function'); +}); +test('plain spawns wait at most 30 seconds then deny with the capacity reason', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2']); + const result = await h.handlers['agent.spawn'](h.$, spawnEvent, h.next); + assert.match(result.deny, /relay is at capacity: waiting worker limit: 2 of 2/); assert.match(result.deny, /Workflow/); + assert.equal(h.calls.length, 0); assert.ok(h.runs.at(-1).argv.includes('30')); +}); +test('a gate crash falls back to the bounded start, while cancellation is denied by catch', async () => { + const h = hooks([new Error('synthetic gate crash')]); + await h.handlers['agent.spawn'](h.$, spawnEvent, h.next); assert.equal(h.calls.length, 1); + const cancelled = hooks([new Error('process.run aborted')]); + await assert.rejects(cancelled.handlers['agent.spawn'](cancelled.$, spawnEvent, cancelled.next), /aborted/); + assert.ok(cancelled.catches['agent.spawn'](cancelled.$, spawnEvent).deny); assert.equal(cancelled.calls.length, 0); +}); +test('resume spawns do not take another waiting place', async () => { + const h = hooks([]); await h.handlers['agent.spawn'](h.$, { ...spawnEvent, resume: { agentId: 'a-old' } }, h.next); + assert.equal(h.runs.length, 0); assert.equal(h.calls.length, 1); +}); +test('a Workflow stall retry is gated at its first step even without agent.spawn', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2', 'pass']); + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-retry', transcript_path: '/synthetic/session.jsonl' }, h.next); + h.calls.length = 0; + const step = h.handlers['turn.step'](h.$, { index: 0, agentId: 'a-retry', turnId: 'turn-retry' }, async function* (e) { return await h.next(e); }); + await step.next(); + assert.equal(h.calls.length, 1); assert.equal(h.runs.length, 2); assert.ok(h.runs.every(r => r.argv.includes('--retry') && r.argv.includes('a-retry'))); +}); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index cd350f71..089c2d14 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -304,3 +304,15 @@ test("Darwin cleanup of an otherwise empty worker group finds no phantom sampler const result = await stopExecution({ execution: { leader: identity, pgid } }, { listProcesses: list, graceMs: 0 }); assert.deepEqual(result.survivors, []); }); + +test('a busy process-cache lock never supplies stale rows to a fresh execution read', async t => { + const { readProcessTableCached } = await import('../plugins/relay/scripts/lib/util/process.mjs'); + const { acquireOwnership, releaseOwnership } = await import('../plugins/relay/scripts/lib/util/ownership.mjs'); + const home = makeTempDir('relay-fresh-cache-'), file = path.join(home, '.ps-table.json'); + fs.writeFileSync(file, JSON.stringify({ at: Date.now() - 60000, rows: [] })); + assert.equal(Boolean(acquireOwnership(`${file}.lock`)), true); + t.after(() => releaseOwnership(`${file}.lock`)); + const rows = [{ pid: 123, startTime: 'synthetic-live' }]; let reads = 0; + for (const maxAgeMs of [0, 5000]) assert.deepEqual(readProcessTableCached({ home, maxAgeMs, list: () => { reads++; return rows; } }), rows); + assert.equal(reads, 2); +}); diff --git a/tests/queued.test.mjs b/tests/queued.test.mjs index 3e1bffdc..22e37522 100644 --- a/tests/queued.test.mjs +++ b/tests/queued.test.mjs @@ -4,8 +4,8 @@ import test from "node:test"; import { computeJobView, exitCodeFor, isTerminalView, jobViewJson, renderJobBlock } from "../plugins/relay/scripts/lib/jobs/view.mjs"; import { readProcessStartTime } from "../plugins/relay/scripts/lib/util/process.mjs"; -test("durable queued jobs without a worker stay pending, with reason and since", () => { - const job = { id: "job-queued", state: "queued", createdAt: "2025-01-01T00:00:00.000Z", admission: { state: "waiting", waitKind: "worker", reason: "memory low", since: "2025-01-01T00:00:00.000Z" } }; +test("workerless queued jobs stay pending only while their launcher lives", () => { + const job = { id: "job-queued", state: "queued", createdAt: "2025-01-01T00:00:00.000Z", admission: { launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, state: "waiting", waitKind: "worker", reason: "memory low", since: "2025-01-01T00:00:00.000Z" } }; const view = { ...computeJobView(job), job, hasReport: false, paths: { brief: "/synthetic/absent" }, jobDir: "/synthetic/job-queued" }; assert.equal(view.state, "queued"); assert.equal(view.stalled, false); @@ -20,3 +20,8 @@ test("resident queued jobs never stall; dead queued workers remain recoverable", assert.equal(computeJobView(job).stalled, false); assert.equal(computeJobView({ ...job, worker: { pid: 2147483647 } }).state, "dead"); }); + +test("legacy workerless queues are dead with an explicit no-dispatcher reason", () => { + const view = computeJobView({ state: "queued", admission: { waitKind: "worker" } }); + assert.equal(view.state, "dead"); assert.match(view.deadReason, /older relay with nobody to start it/); +}); diff --git a/tests/recovery.test.mjs b/tests/recovery.test.mjs index f02b8ebe..6580d74d 100644 --- a/tests/recovery.test.mjs +++ b/tests/recovery.test.mjs @@ -698,7 +698,7 @@ for (const persisted of [false, true]) { assert.equal(fs.existsSync(path.join(ctx.binDir, "fake-codex-state.json")), false); }); - test("automatic recovery's lease is no wait-false consumer", async t => { + test("automatic recovery's lease cannot accept a message before its worker owns it", async t => { const marker = path.join(makeTempDir("relay-recovery-marker-"), "term"); const { identity } = await oldServer(t, { ignoreTerm: true, marker }); const ctx = sweepJob({ server: identity, servers: [identity] }); @@ -726,11 +726,11 @@ for (const persisted of [false, true]) { const message = postInboxMessage(ctx.jobDir, { kind: "continue", text: "[quick] Arrived during recovery" }); const delivered = await awaitOutcome(ctx.jobDir, message.id, { env, wait: false, timeoutMs: 50 }); assert.equal(delivered.consumer, false); - assert.equal(delivered.outcome, null); + assert.equal(delivered.outcome.action, "not-started"); assert.equal(readOutcome(ctx.jobDir, message.id), null); await recovering; - await waitFor(() => readOutcome(ctx.jobDir, message.id)?.action === "turn-started" && !readLease(ctx.jobDir)); - assert.equal(ctx.fakeState().turnStarts.filter(turn => turn.prompt === message.text).length, 1); + await waitFor(() => !readLease(ctx.jobDir)); + assert.equal(ctx.fakeState().turnStarts.filter(turn => turn.prompt === message.text).length, 0); } finally { await recovering; } diff --git a/tests/relay-delivery.test.mjs b/tests/relay-delivery.test.mjs index 0329ed43..ba59fdd4 100644 --- a/tests/relay-delivery.test.mjs +++ b/tests/relay-delivery.test.mjs @@ -192,7 +192,7 @@ test("completed Workflow adoption consumes an early message with wait false", as assert.equal(runs(first, text), 1); }); -test("an acknowledgement timeout retries the original post exactly once and in order", async () => { +test("an acknowledgement timeout under a live worker retries the original post exactly once and in order", async () => { const ctx = relayContext("job", { turnMs: 100 }); const { jobId, dir } = await completed(ctx); const release = holdLease(dir); @@ -513,7 +513,7 @@ test("wait false reports no consumer on timeout", async () => { const message = postInboxMessage(dir, { kind: "continue", text: "[quick] No consumer yet" }); const result = await awaitOutcome(dir, message.id, { env: ctx.env, wait: false, timeoutMs: 0 }); assert.equal(result.consumer, false); - assert.equal(result.outcome, null); + assert.equal(result.outcome.action, "not-started"); assert.equal(runs(ctx, message.text), 0); }); @@ -623,12 +623,12 @@ fs.linkSync = function (from, to, ...rest) { fs.writeFileSync(go, ""); await ended; } - assert.notEqual(receipt.status, "done", "a cancel holding the lease cannot acknowledge consumption"); - // Released, the cancel read the inbox again and started a worker for the follow-up. + assert.equal(receipt, null, "a cancel lease cannot accept work; the refused mapping was removed"); + assert.equal(runs(ctx, text), 0); + // The next hook posts the withdrawn message again and gets a real consumer. + await deliverRelayMessages(dir, [{ uuid: entry.uuid, text }], { env: ctx.env, wait: false }); await waitFor(() => runs(ctx, text) === 1); await waitFor(() => readJobJson(ctx, jobId).state === "completed" && !readLease(dir)); - // The next hook closes the receipt without running the message again. - await deliverRelayMessages(dir, [{ uuid: entry.uuid, text }], { env: ctx.env, wait: false }); assert.equal(readLedger(dir, entry.uuid).status, "done"); assert.equal(runs(ctx, text), 1); }); From 222f0ce25abbdfc25d53f9fdc443a8e5f70812e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:23:40 +0200 Subject: [PATCH 02/10] dispatch: keep interrupted handoff waits visibly owned A waiter retrying a dead handoff publishes its own launcher identity before the spawn throttle. It stays queued through the spawning grace instead of ending as dead. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/scripts/lib/worker/delivery.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/relay/scripts/lib/worker/delivery.mjs b/plugins/relay/scripts/lib/worker/delivery.mjs index 49a3900b..09d70ff4 100644 --- a/plugins/relay/scripts/lib/worker/delivery.mjs +++ b/plugins/relay/scripts/lib/worker/delivery.mjs @@ -36,9 +36,9 @@ export function spawnWorker(jobDir, cwd, env = process.env, { reservation = null pruneLaunches(jobDir); const latest = latestLaunch(jobDir); if (latest?.worker && ["pending", "acknowledged"].includes(latest.state) && isProcessIdentityAlive(latest.worker)) return null; - if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && launchInterrupted(job, latest) && job.admission?.causeKind !== "launch") { + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && launchInterrupted(job, latest)) { job = { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, state: "waiting", - waitKind: "worker", causeKind: "launch", reason: "worker launch interrupted; retrying", since: new Date().toISOString() } }; + waitKind: "worker", causeKind: "launch", launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, reason: "worker launch interrupted; retrying", since: new Date().toISOString() } }; writeJob(jobDir, job); } const pendingId = pendingInboxMessages(jobDir).at(0) ?? null; From 2331d19260fd6f2e1e72b33d0befe233259666b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:28:59 +0200 Subject: [PATCH 03/10] admission: refresh process evidence only before removing a dead slot A live slot is already counted. Avoid repeated fresh process-table reads for its owner while retaining the fresh check before any destructive slot cleanup. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/scripts/lib/worker/admission.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/relay/scripts/lib/worker/admission.mjs b/plugins/relay/scripts/lib/worker/admission.mjs index ecd1a2ca..1de74df1 100644 --- a/plugins/relay/scripts/lib/worker/admission.mjs +++ b/plugins/relay/scripts/lib/worker/admission.mjs @@ -370,7 +370,7 @@ function slots(home, deps, clean = true) { const job = readJob(s.jobDir); s.quotaHeavy = isVeryHeavy({ backend: s.backend, model: s.model, effort: s.effort }) || isVeryHeavy(job?.request); s.weight = WEIGHTS[s.class]; - const backendLive = executionAlive({ ...job, jobDir: s.jobDir }, { ...deps, freshExecution: clean }); + const backendLive = !live && executionAlive({ ...job, jobDir: s.jobDir }, { ...deps, freshExecution: clean }); if (live || backendLive) result.push(!live && backendLive ? { ...s, state: "running" } : s); else if (clean) fs.rmSync(file, { force: true }); } From 6670c1d7d826494e0a48d55d3ba36064fb0c7cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:53:48 +0200 Subject: [PATCH 04/10] dispatch: preserve cancellation and reservations through handoff races Keep unresolved scope membership conservative for capacity while stopping verified processes, reuse adoption reservations, and aggregate wait causes independently of changing readings. Validate private worker identity storage and preserve delegated hook failures. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/hooks/mod/register.js | 32 +++++++---- plugins/relay/hooks/mod/register.test.ts | 23 ++++++++ plugins/relay/scripts/cli/cancel.mjs | 4 ++ plugins/relay/scripts/cli/gate.mjs | 10 +++- plugins/relay/scripts/cli/start.mjs | 9 ++- .../relay/scripts/hooks/stop-review-gate.mjs | 2 +- plugins/relay/scripts/lib/harness/relay.mjs | 3 +- plugins/relay/scripts/lib/util/process.mjs | 32 ++++++++--- .../relay/scripts/lib/worker/admission.mjs | 25 +++++++-- plugins/relay/scripts/lib/worker/delivery.mjs | 8 +-- plugins/relay/scripts/lib/worker/recovery.mjs | 6 +- plugins/relay/scripts/lib/worker/registry.mjs | 7 ++- tests/admission.test.mjs | 17 ++++++ tests/cancel-dead.test.mjs | 18 ++++++ tests/execution-groups.test.mjs | 56 ++++++++++++++++++- tests/gate.test.mjs | 26 ++++++++- tests/harness-workflow.test.mjs | 19 ++++++- tests/message-account.test.mjs | 2 +- tests/relay-guard.test.mjs | 17 ++++-- 19 files changed, 270 insertions(+), 46 deletions(-) diff --git a/plugins/relay/hooks/mod/register.js b/plugins/relay/hooks/mod/register.js index c5ace920..9edfffc8 100644 --- a/plugins/relay/hooks/mod/register.js +++ b/plugins/relay/hooks/mod/register.js @@ -1121,7 +1121,7 @@ export function register(on) { on("agent.spawn", async ($, e, next) => { if (e?.subagentType !== "relay:job") return next(e); poller.fastUntil = poller.ticks + CLEARED_TICKS; - if (e.resume != null) return next(e); + if (e.resume != null) { if (e.resume.agentId) spawned.add(e.resume.agentId); return next(e); } if (isCheckedSpawn(e)) { let refusal = null; try { refusal = await headerRefusal($, e); } catch {} @@ -1133,13 +1133,20 @@ export function register(on) { if (result?.agentId) { spawned.add(result.agentId); if (gate.reservation) { - const pending = $.process.run(["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--bind", gate.reservation, "--agent", result.agentId], { timeoutMs: 10000 }); - binding.set(result.agentId, pending); - try { await pending; } finally { binding.delete(result.agentId); } + try { + const pending = $.process.run(["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--bind", gate.reservation, "--agent", result.agentId], { timeoutMs: 10000 }); + binding.set(result.agentId, pending); + await pending; + } catch { + // The agent is already running; start's bounded admission remains authoritative. + } finally { binding.delete(result.agentId); } } } return result; - }).catch(($, e) => ({ deny: "relay admission was interrupted. Try again, or resume the Workflow." })); + }).catch(($, e, next) => { + if (next.called) return next(e); + return { deny: "relay admission was interrupted. Try again, or resume the Workflow." }; + }); // Workflow stall retries bypass agent.spawn. Track each actual attempt and // hold its first model step using the same host-process gate. @@ -1149,12 +1156,16 @@ export function register(on) { }); on("turn.step", async function* ($, e, next) { if (e.index !== 0 || !relayAgents.has(e.agentId)) return yield* next(e); - if (binding.has(e.agentId)) await binding.get(e.agentId); - if (spawned.delete(e.agentId)) return yield* next(e); - await waitForPlace($, true, e.agentId, relayAgents.get(e.agentId)); + if (binding.has(e.agentId)) { try { await binding.get(e.agentId); } catch {} } + if (!spawned.delete(e.agentId)) await waitForPlace($, true, e.agentId, relayAgents.get(e.agentId)); return yield* next(e); - }).catch(async function* ($, e) { return { turnId: e.turnId, index: e.index, answer: "state: not started\nerror: relay admission was interrupted; resume the Workflow.", - toolUses: [], stopReason: "end_turn", usage: { input_tokens: 0, output_tokens: 0 } }; }); + }).catch(async function* ($, e, next) { + // The engine reuses an in-flight next result here; it does not run the model twice. + if (next.called) return yield* next(e); + const answer = "state: not started\nerror: relay admission was interrupted; resume the Workflow."; + yield { kind: "text", index: 0, text: answer }; + return { turnId: e.turnId, index: e.index, answer, toolUses: [], stopReason: "end_turn", usage: null }; + }); on("session.send", async ($, e, next) => { const result = await next(e); @@ -1207,6 +1218,7 @@ export function register(on) { // after it has returned, and a slow or stuck listing never holds Claude Code's stop. on("turn.complete", async ($, e, next) => { const completed = await next(e); + if (e.agentId != null) { relayAgents.delete(e.agentId); spawned.delete(e.agentId); binding.delete(e.agentId); } if (e.agentId != null && e.reason === "aborted") { try { $.clock.after(0, () => { diff --git a/plugins/relay/hooks/mod/register.test.ts b/plugins/relay/hooks/mod/register.test.ts index 3924d418..4ec34d8c 100644 --- a/plugins/relay/hooks/mod/register.test.ts +++ b/plugins/relay/hooks/mod/register.test.ts @@ -2934,3 +2934,26 @@ test('a resumed relay bypasses capacity admission', async ($, on) => { expect(await $.agent.spawn(relay('Continue.', { resume: { agentId: 'a-job' } }) as any)).toEqual(STARTED) expect(seen.gates).toHaveLength(0) }) + +test('a resumed relay first step makes no capacity call even with no existing job', async ($, on) => { + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n']) + on('classic.SubagentStart', () => ({})) + on('turn.step', async function* (_: any, e: any) { + yield { kind: 'text', index: 0, text: 'ok' } + return { turnId: e.turnId, index: e.index, answer: 'ok', toolUses: [], stopReason: 'end_turn', usage: null } + }) + await $.agent.spawn(relay('Continue.', { resume: { agentId: 'a-job' } }) as any) + await $.classic.SubagentStart({ agent_type: 'relay:job', agent_id: 'a-job' } as any) + const stream = $.turn.step({ turnId: 't-resumed', index: 0, model: 'claude-test', messageCount: 1, agentId: 'a-job' } as any) + let step = await stream.next(); while (step.done !== true) step = await stream.next() + expect(step.value.answer).toBe('ok') + expect(seen.gates).toHaveLength(0) +}) + +test('a binding failure after spawn keeps the admitted agent result', async ($, on) => { + let spawned = 0 + on('process.run', (_: unknown, e: any) => e.argv.includes('--bind') ? { deny: 'synthetic bind failure' } : ran(0, e.argv[2] === 'check-header' ? 'ok\n' : 'pass 00000000-0000-4000-8000-000000000001\n')) + on('agent.spawn', () => { spawned++; return STARTED }) + expect(await $.agent.spawn(relay(GOOD))).toEqual(STARTED) + expect(spawned).toBe(1) +}) diff --git a/plugins/relay/scripts/cli/cancel.mjs b/plugins/relay/scripts/cli/cancel.mjs index f5c74e3e..afd40bd5 100644 --- a/plugins/relay/scripts/cli/cancel.mjs +++ b/plugins/relay/scripts/cli/cancel.mjs @@ -59,6 +59,10 @@ function markCancelledAfterKill(jobDir, reason, attempt = null, killed = false, const pause = () => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); const deadline = Date.now() + 2000; while (!acquireWorkerLease(jobDir, me, "cancel")) { + const current = readJob(jobDir); + // A finished worker may still be closing its backend. Its job needs one + // no-op receipt, not a delegated cancel plus a second receipt. + if (attempt === null && TERMINAL_STATES.has(current.state)) return { ended: current.state }; const holder = readLease(jobDir); if (!isWorkerLease(holder) && Date.now() < deadline) { pause(); diff --git a/plugins/relay/scripts/cli/gate.mjs b/plugins/relay/scripts/cli/gate.mjs index 381c428b..f7a0a04d 100644 --- a/plugins/relay/scripts/cli/gate.mjs +++ b/plugins/relay/scripts/cli/gate.mjs @@ -7,19 +7,22 @@ import { findAdoptableRelayJob, findRelayJob, promptSha256, readJournalKey, read // A stalled Workflow attempt gets a fresh agent id without agent.spawn. At its // first step, reuse an existing job or reservation before asking for another place. -function hasAdoptableJob(agentId, transcriptPath, env) { +function retryAlreadyCovered(agentId, transcriptPath, env) { + let workflow = false; if (findRelayJob(env, agentId)) return true; for (const file of transcriptCandidates({ agent_id: agentId, transcript_path: transcriptPath })) { if (!fs.existsSync(file)) continue; const run = /[\\/]workflows[\\/]([^\\/]+)[\\/]agent-[^\\/]+\.jsonl$/.exec(file)?.[1]; if (!run) continue; + workflow = true; const transcript = readRelayTranscript(file); let meta; try { meta = JSON.parse(fs.readFileSync(file.replace(/\.jsonl$/, ".meta.json"), "utf8")); } catch {} if (findAdoptableRelayJob(env, { agentId, workflowRunId: run, journalKey: readJournalKey(file, agentId), promptSha256: promptSha256(transcript.prompt?.text), label: meta?.description })) return true; } - return false; + // Plain resumes are never capacity-gated, including a start previously refused. + return !workflow; } export async function handleGate(argv) { @@ -32,13 +35,14 @@ export async function handleGate(argv) { if (options.agent) { const reservation = readReservation(options.agent); if (reservation) { process.stdout.write(`pass ${reservation}\n`); return; } - if (options.retry && hasAdoptableJob(options.agent, options.transcript, process.env)) { process.stdout.write("pass\n"); return; } + if (options.retry && retryAlreadyCovered(options.agent, options.transcript, process.env)) { process.stdout.write("pass\n"); return; } } const seconds = Number(options["wait-seconds"] ?? 0); if (!Number.isFinite(seconds) || seconds < 0) throw new Error("--wait-seconds must be non-negative"); const deadline = Date.now() + seconds * 1000; for (;;) { const result = reserveWaitingPlace(); + if (result.error) { process.stderr.write(`${result.reason}\n`); process.exitCode = 1; return; } if (result.pass) { if (result.reservation && options.agent) bindReservation(result.reservation, options.agent); process.stdout.write(`pass${result.reservation ? ` ${result.reservation}` : ""}\n`); return; diff --git a/plugins/relay/scripts/cli/start.mjs b/plugins/relay/scripts/cli/start.mjs index e73d1e31..c8f6a0b1 100644 --- a/plugins/relay/scripts/cli/start.mjs +++ b/plugins/relay/scripts/cli/start.mjs @@ -7,7 +7,8 @@ import { startWithinWindow } from "../lib/worker/delivery.mjs"; import { appendRelayViolation } from "../lib/jobs/messages.mjs"; import { nowIso, readJob } from "../lib/jobs/store.mjs"; import { buildLaunchRequest, readLaunchPrompt, UsageError } from "../lib/jobs/request.mjs"; -import { readReservation } from "../lib/worker/admission.mjs"; +import { readReservation, releaseReservation } from "../lib/worker/admission.mjs"; +import { resumeDeadJobOnce } from "../lib/worker/recovery.mjs"; import { SCHEMA_SENTENCE } from "../lib/harness/workflow.mjs"; import { backendJson, backendLines, backendSettingLines, EXIT, loadJobViewSettled as loadJobView, relayAdoptionLines } from "../lib/jobs/view.mjs"; import { adoptRelayJob, deliverRelayMessages, findAdoptableRelayJob, promptSha256, readJournalKey, readRelayPointer, readRelayTranscript, writeRelayPointer } from "../lib/harness/relay.mjs"; @@ -155,7 +156,11 @@ async function handleStartFromRelay(options) { for (const problem of transcript.problems) { appendRelayViolation(adoptable, { kind: "transcript", reason: `messages could not be checked: ${problem}`, agentId }); } - await deliverRelayMessages(adoptable, transcript.messages, { wait: false }); + const reservation = readReservation(agentId); + try { + if (loadJobView(adoptable).state === "dead") await resumeDeadJobOnce(adoptable); + await deliverRelayMessages(adoptable, transcript.messages, { wait: false, reservation }); + } finally { if (reservation) releaseReservation(reservation); } const adopted = readJob(adoptable); if (options.json) printJson("start", { jobId: adopted.id, state: loadJobView(adoptable).state, jobDir: adoptable, notes: adopted.request?.notes ?? [], ...backendJson(adopted.request) }); else process.stdout.write(adoptedBlock(adopted, adoptable)); diff --git a/plugins/relay/scripts/hooks/stop-review-gate.mjs b/plugins/relay/scripts/hooks/stop-review-gate.mjs index 540a7622..a4f760a4 100644 --- a/plugins/relay/scripts/hooks/stop-review-gate.mjs +++ b/plugins/relay/scripts/hooks/stop-review-gate.mjs @@ -17,7 +17,7 @@ import { readRepoConfig } from "../lib/jobs/config.mjs"; const STOP_REVIEW_TIMEOUT_MS = 10 * 60 * 1000; const STOP_PROBE_TIMEOUT_MS = 10 * 1000; // Starting the review job, the margin of its wait, and the cancel after a timeout. -export const STOP_COMMAND_TIMEOUT_MS = 60 * 1000; +export const STOP_COMMAND_TIMEOUT_MS = 90 * 1000; const REVIEW_BACKEND = backendWith("stopGate"); const MANUAL_REVIEW = "Run /relay:review --wait to review by hand."; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); diff --git a/plugins/relay/scripts/lib/harness/relay.mjs b/plugins/relay/scripts/lib/harness/relay.mjs index 725d3f06..4eb4d1c1 100644 --- a/plugins/relay/scripts/lib/harness/relay.mjs +++ b/plugins/relay/scripts/lib/harness/relay.mjs @@ -685,6 +685,7 @@ export async function deliverRelayMessages(jobDir, messages, options = {}) { async function deliverRelayMessagesUnderLock(jobDir, messages, options) { const env = options.env ?? process.env; const wait = options.wait ?? true; + const reservation = options.reservation ?? null; if (undeliveredMessages(jobDir, messages).length === 0) { return { busy: false, delivered: [], undelivered: [] }; } @@ -761,7 +762,7 @@ async function deliverRelayMessagesUnderLock(jobDir, messages, options) { postInboxMessage(jobDir, { id: record.messageId, kind: record.kind, text: message.text, relayUuid: message.uuid }); } const ackTimeoutMs = envNumber(env, "RELAY_ACK_TIMEOUT_MS", 20000); - const { outcome, consumer } = await awaitOutcome(jobDir, record.messageId, { env, timeoutMs: ackTimeoutMs, wait }); + const { outcome, consumer } = await awaitOutcome(jobDir, record.messageId, { env, timeoutMs: ackTimeoutMs, wait, reservation }); if (!consumer) { appendRelayViolation(jobDir, { kind: "delivery", reason: `message ${message.uuid} acknowledgement timed out; will retry with message id ${record.messageId}` }); break; diff --git a/plugins/relay/scripts/lib/util/process.mjs b/plugins/relay/scripts/lib/util/process.mjs index 7de832f7..4bc7fa60 100644 --- a/plugins/relay/scripts/lib/util/process.mjs +++ b/plugins/relay/scripts/lib/util/process.mjs @@ -490,11 +490,20 @@ function scopeMemberStarted(row, execution) { // Independent of admission storage, including when the guard is explicitly off. export const independentWorkerRegistry = () => path.join("/tmp", `relay-workers-${process.getuid?.() ?? "user"}`); +export function safeWorkerRegistry(directory) { + try { + const stat = fs.lstatSync(directory); + return stat.isDirectory() && !stat.isSymbolicLink() && + (process.getuid == null || stat.uid === process.getuid()) && (stat.mode & 0o077) === 0; + } catch { return false; } +} + // Published workers form ownership boundaries, including during a nested launch handoff. function foreignWorkerPids(job, table) { const excluded = new Set(); - for (const directory of new Set([independentWorkerRegistry(), ...executions(job).map(e => e.workerRegistry).filter(Boolean)])) { + const independent = independentWorkerRegistry(); + for (const directory of new Set([...(safeWorkerRegistry(independent) ? [independent] : []), ...executions(job).map(e => e.workerRegistry).filter(Boolean)])) { // Lookup only current process ids. Historical crash records cannot increase // sampling work, and a newly published nested worker is visible immediately. for (const row of table) { @@ -561,7 +570,6 @@ export function executionMembers(job, table, { jobDir = job?.jobDir, excludeSelf // a concurrently forked pid. Never signal it without a verified identity. for (const pid of pids) if (pid > 0 && !table.some(row => row.pid === pid)) { entries.push({ pid, startTime: null, identityUnknown: true }); - cause = "an execution scope member has no process identity"; } } catch (error) { if (error.code !== "ENOENT") cause = "the execution scope could not be read"; } } @@ -609,12 +617,22 @@ export async function stopExecution(job, options = {}) { } } const stopped = await stopProcessSnapshot(members, options); - // Tools may fork while TERM is being handled. Verify the execution itself again, - // rather than declaring it empty just because the original snapshot is gone. - const remaining = executionMembers(job, (options.listProcesses ?? listProcesses)(), options); + // A member may have forked after the first table read. Stop verified members + // first, then take one fresh discovery pass for survivors. Unknown scope pids + // retain capacity, but never prevent signals to processes we can identify. + let finalTable = (options.listProcesses ?? listProcesses)(); + let remaining = executionMembers(job, finalTable, options); + const discovered = remaining.filter(entry => entry.startTime && !entry.identityUnknown && + !members.some(old => old.pid === entry.pid && old.startTime === entry.startTime)); + if (!remaining.cause && discovered.length) { + await stopProcessSnapshot(discovered, options); + finalTable = (options.listProcesses ?? listProcesses)(); + remaining = executionMembers(job, finalTable, options); + } + const stillStopped = liveSnapshotMembers(stopped, { table: finalTable }); /** @type {ProcessSnapshot} */ - const survivors = [...new Map([...stopped, ...remaining].map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; - if (stopped.cause || remaining.cause) survivors.cause = stopped.cause ?? remaining.cause; + const survivors = [...new Map([...stillStopped, ...remaining].map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; + if (stillStopped.cause || remaining.cause) survivors.cause = stillStopped.cause ?? remaining.cause; for (const server of roots(job)) { if (server.pid !== process.pid && isProcessIdentityAlive(server) && !survivors.some(p => p.pid === server.pid)) survivors.push(server); } diff --git a/plugins/relay/scripts/lib/worker/admission.mjs b/plugins/relay/scripts/lib/worker/admission.mjs index 1de74df1..8673eb45 100644 --- a/plugins/relay/scripts/lib/worker/admission.mjs +++ b/plugins/relay/scripts/lib/worker/admission.mjs @@ -467,13 +467,18 @@ function jobAccount(home, jobId, deps) { const kindLabel = kind => ({ usage: "usage limit", rate: "rate limit", concurrency: "concurrent request limit" })[kind] ?? kind; function refusalReason(p) { if (p.reason) return p.reason; return `${p.backend}${p.account ? ` (account ${p.account})` : ""} paused: ${kindLabel(p.kind)}${p.code ? ` (${p.code})` : ""} until ${p.until}`; } function waitHistory(admission, now) { - const totals = { ...(admission?.waitTotals ?? {}) }; + const totals = {}; + // Reason text includes changing readings. Aggregate by cause, with bounded + // keys, rather than making a new durable entry for every sample. + for (const old of Object.values(admission?.waitTotals ?? {})) { + if (typeof old?.kind !== "string" || !Number.isFinite(old.ms) || old.ms < 0) continue; + totals[old.kind] = { ...old, ms: (totals[old.kind]?.ms ?? 0) + old.ms }; + } if (admission?.state === "waiting" && admission.reason) { const from = Date.parse(admission.reasonSince ?? admission.since); const elapsed = Number.isFinite(from) ? Math.max(0, now - from) : 0; const kind = admission.causeKind ?? admission.waitKind ?? "unknown"; - const key = `${kind}: ${admission.reason}`; - totals[key] = { kind, reason: admission.reason, ms: (totals[key]?.ms ?? 0) + elapsed }; + totals[kind] = { kind, reason: admission.reason, ms: (totals[kind]?.ms ?? 0) + elapsed }; } return totals; } @@ -952,7 +957,7 @@ export function reserveWaitingPlace(env = process.env, injected = {}) { return { pass: true, reservation }; }); return result.locked ? result.value : { pass: false, reason: "admission lock busy" }; - } catch (error) { return { pass: false, reason: `guard unavailable (${error.message})` }; } + } catch (error) { return { pass: false, error: true, reason: `guard unavailable (${error.message})` }; } } // Called only under the job lease after its refusal has become terminal. @@ -966,3 +971,15 @@ export function discardSpawnReservation(jobDir, env = process.env) { }); } catch {} } + +export function releaseReservation(reservation, env = process.env) { + if (!validReservation(reservation)) return false; + const deps = dependencies({ env }), home = admissionHome(env); + try { return retryLock(home, deps, () => { + const file = slotFile(home, `r-${reservation}`), slot = readJsonOrNull(file); + if (slot?.state !== "reserved") return false; + fs.rmSync(file, { force: true }); + if (slot.agentId) fs.rmSync(bindingFile(env, slot.agentId), { force: true }); + return true; + }).value ?? false; } catch { return false; } +} diff --git a/plugins/relay/scripts/lib/worker/delivery.mjs b/plugins/relay/scripts/lib/worker/delivery.mjs index 09d70ff4..49f754ff 100644 --- a/plugins/relay/scripts/lib/worker/delivery.mjs +++ b/plugins/relay/scripts/lib/worker/delivery.mjs @@ -52,7 +52,7 @@ export function spawnWorker(jobDir, cwd, env = process.env, { reservation = null if (job?.state === "queued" && job.admission?.waitKind === "worker" && Date.now() - retryAt >= 0 && Date.now() - retryAt < poll) return null; const result = spawnWithAdmission(jobDir, env, () => spawnWorkerProcess(jobDir, cwd, env), {}, { reservation }); const deferred = readJob(jobDir); - if (deferred?.state === "queued" && deferred.admission?.waitKind === "worker") writeJob(jobDir, { ...deferred, launchPendingAt: new Date(Date.now()).toISOString(), admission: { ...deferred.admission, workerRetryAt: new Date(Date.now()).toISOString() } }); + if (deferred?.state === "queued" && deferred.admission?.waitKind === "worker") writeJob(jobDir, { ...deferred, launchPendingAt: new Date(Date.now()).toISOString(), admission: { ...deferred.admission, launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, workerRetryAt: new Date(Date.now()).toISOString() } }); return result; } catch (error) { // Dispatch acknowledgement is an observation, not worker ownership. An @@ -184,7 +184,7 @@ export async function awaitOutcome(jobDir, messageId, options = {}) { if (!hasLiveWorker(jobDir) && messageAccount(jobDir).undelivered.includes(messageId) && timeoutMs > 0) { const job = readJob(jobDir); spawned += (await startWithinWindow(jobDir, job?.request?.cwd ?? process.cwd(), env, - { windowMs: timeoutMs, messageId, refuse: false })).spawned; + { windowMs: timeoutMs, messageId, refuse: false, reservation: options.reservation })).spawned; } let nextSpawn = -Infinity, wasLive = false; for (;;) { @@ -219,8 +219,8 @@ export async function awaitOutcome(jobDir, messageId, options = {}) { // Drop a message into the job's inbox and wait until a worker records its outcome. export async function deliverMessage(jobDir, kind, text, extra = {}, options = {}) { const message = postInboxMessage(jobDir, { kind, text, ...extra }); - const { outcome, spawned, consumer } = await awaitOutcome(jobDir, message.id, options); - return { message, outcome, spawned, consumer }; + const { outcome, spawned } = await awaitOutcome(jobDir, message.id, options); + return { message, outcome, spawned }; } export function pruneLaunches(jobDir, keep = 20) { diff --git a/plugins/relay/scripts/lib/worker/recovery.mjs b/plugins/relay/scripts/lib/worker/recovery.mjs index 5cbc736c..212d9538 100644 --- a/plugins/relay/scripts/lib/worker/recovery.mjs +++ b/plugins/relay/scripts/lib/worker/recovery.mjs @@ -3,7 +3,7 @@ import process from "node:process"; import fs from "node:fs"; import path from "node:path"; -import { DEFAULTS } from "./admission.mjs"; +import { DEFAULTS, readReservation } from "./admission.mjs"; import { releaseLeaseToConsumer, startWithinWindow } from "./delivery.mjs"; import { acquireWorkerLease, isWorkerAlive } from "../jobs/lease.mjs"; import { computeJobView } from "../jobs/view.mjs"; @@ -53,10 +53,12 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { let cancelling = false; let cwd; let original; + let reservation; try { let job = readJob(jobDir); if (computeJobView(job).state !== "dead" || (job.kind && job.kind !== "task")) return false; original = job; + reservation = job.relay?.agentId ? readReservation(job.relay.agentId, env) : null; const epoch = job.recoveryEpoch ?? "start"; const previous = recoveryForEpoch(job); // A reservation interrupted before posting is repaired with the same message id. @@ -84,7 +86,7 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { } if (!launch) return false; const result = await startWithinWindow(jobDir, cwd, { ...env, RELAY_WORKER_TRIGGER: "recovery" }, - { windowMs: Number(env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs, refuse: false }); + { windowMs: Number(env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs, refuse: false, reservation }); // Workers update the job concurrently. Take its lease before changing the // recovery record, or let the worker's own attempt bookkeeping be authoritative. if (acquireWorkerLease(jobDir, me, "recovery")) { diff --git a/plugins/relay/scripts/lib/worker/registry.mjs b/plugins/relay/scripts/lib/worker/registry.mjs index 74bceafd..591d7964 100644 --- a/plugins/relay/scripts/lib/worker/registry.mjs +++ b/plugins/relay/scripts/lib/worker/registry.mjs @@ -1,14 +1,15 @@ import fs from "node:fs"; import path from "node:path"; -import { independentWorkerRegistry, isProcessIdentityAlive } from "../util/process.mjs"; +import { independentWorkerRegistry, safeWorkerRegistry, isProcessIdentityAlive } from "../util/process.mjs"; import { readJsonOrNull, writeJsonAtomic } from "../jobs/store.mjs"; -export function registerWorker(worker, { env = process.env, platform = process.platform } = {}) { +export function registerWorker(worker, { env = process.env, platform = process.platform, directory = independentWorkerRegistry() } = {}) { if (!["linux", "darwin"].includes(platform)) return null; // Cleanup boundaries must survive an unwritable admission directory, even with // the guard off. Publish before handoff; inability to publish prevents launch. - const directory = independentWorkerRegistry(); fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + if (!safeWorkerRegistry(directory)) throw new Error("Worker identity directory must be private, owned by this user and not a symlink."); + if (!worker?.startTime) throw new Error("Worker process identity could not be read; no backend was started."); pruneWorkerRegistry(directory); const file = path.join(directory, `${worker.pid}.json`); writeJsonAtomic(file, worker); diff --git a/tests/admission.test.mjs b/tests/admission.test.mjs index 6f2ef305..f3bead95 100644 --- a/tests/admission.test.mjs +++ b/tests/admission.test.mjs @@ -1167,3 +1167,20 @@ test('slot cleaning and release require fresh live execution evidence even with assert.equal(describeGuard(s.env, {}, deps).activeSeats, 1); assert.equal(fs.existsSync(file), true); assert.equal(releaseSlot(s.home, c.jobId, leader, deps), false); assert.equal(fs.existsSync(file), true); }); + +test('changing memory readings accumulate one bounded hold kind and outweigh a short final stagger', t => { + const s = setup(t); fillOtherMinimum(s); + const c = s.ctx({ model: 'synthetic-model' }); + for (let i = 0; i < 100; i++) { + s.setMachine({ availableBytes: (10 + i / 100) * GiB }); + const held = admit(c, s.deps); assert.equal(held.waitKind, 'memory'); + c.admission = held.record; c.waitingSince = held.record.since; s.tick(1000); + } + s.setMachine({ availableBytes: 100 * GiB }); + fs.writeFileSync(path.join(s.home, 'timing.json'), JSON.stringify({ lastStart: new Date(s.deps.now()).toISOString() })); + const stagger = admit(c, s.deps); c.admission = stagger.record; + s.tick(D.spacingMs); const result = admit(c, s.deps); + assert.equal(result.admit, true); assert.equal(result.record.waitKind, 'memory'); + assert.equal(result.record.waitTotals.memory.ms, 100000); + assert.equal(Object.keys(result.record.waitTotals).length, 2); +}); diff --git a/tests/cancel-dead.test.mjs b/tests/cancel-dead.test.mjs index 3798a29b..44d9269f 100644 --- a/tests/cancel-dead.test.mjs +++ b/tests/cancel-dead.test.mjs @@ -5,6 +5,7 @@ import { once } from "node:events"; import test from "node:test"; import assert from "node:assert/strict"; import { setup, companion, waitFor } from "./support/job-helpers.mjs"; +import { acquireWorkerLease, releaseWorkerLease } from "../plugins/relay/scripts/lib/jobs/lease.mjs"; import { writeExecutable } from "./support/helpers.mjs"; import { createJobDir, readJob, writeJob } from "../plugins/relay/scripts/lib/jobs/store.mjs"; import { isProcessIdentityAlive, listProcesses, readProcessStartTime } from "../plugins/relay/scripts/lib/util/process.mjs"; @@ -235,3 +236,20 @@ fs.readFileSync = function(file, ...args) { assert.equal(isProcessIdentityAlive(identity), false); assert.equal(isProcessIdentityAlive(member), false); }); + + +test("a completed job with a closing worker records one no-op cancel without delegating another", async t => { + const ctx = setup(t); + const worker = await sleeper(t, true), server = await sleeper(t, false); + const { id, jobDir } = recorded(ctx, worker.identity, server.identity, { state: "completed", attempts: [{ attempt: 1, state: "completed" }] }); + assert.equal(acquireWorkerLease(jobDir, worker.identity), true); + t.after(() => releaseWorkerLease(jobDir, worker.identity)); + const result = companion(ctx, ["cancel", id]); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(isProcessIdentityAlive(worker.identity), true); + assert.equal(isProcessIdentityAlive(server.identity), false); + const messages = ["inbox", "inbox/claimed"].flatMap(dir => fs.readdirSync(path.join(jobDir, dir)).filter(n => n.endsWith(".json")).map(n => JSON.parse(fs.readFileSync(path.join(jobDir, dir, n))))).filter(m => m.kind === "cancel"); + assert.equal(messages.length, 1); + const outcome = JSON.parse(fs.readFileSync(path.join(jobDir, "inbox/outcomes", `${messages[0].id}.json`))); + assert.equal(outcome.action, "no-op"); +}); diff --git a/tests/execution-groups.test.mjs b/tests/execution-groups.test.mjs index 189f7d79..f71f24d6 100644 --- a/tests/execution-groups.test.mjs +++ b/tests/execution-groups.test.mjs @@ -528,7 +528,7 @@ test('scope-only pids absent from the process table remain unresolved and keep t const job = { worker: leader, execution: { leader, pgid: leader.pid, cgroup: group, cgroupId: 37, bootId: 'synthetic-boot' } }; const members = executionMembers(job, []); assert.deepEqual([...members], [{ pid, startTime: null, identityUnknown: true }]); - assert.match(members.cause, /no process identity/); + assert.equal(members.cause, undefined, "unknown scope pids retain capacity without suppressing verified signals"); assert.equal(executionAlive(job, { env: {}, listProcesses: () => [], isAlive: () => false }), true); }); @@ -579,3 +579,57 @@ test('caller acknowledgement observation timeout never erases a live dispatched assert.equal(job.workers.length, 1); assert.equal(ctx.fakeState().appServerStarts, 1); assert.equal(fs.readdirSync(dir).filter(name => /^launch-/.test(name)).length, 1); }); + +test('a fork between process-table and scope reads does not stop cancellation of verified processes', { skip: process.platform !== 'linux' }, async t => { + const probe = spawnSync('systemd-run', ['--user', '--scope', '--quiet', '--collect', '--slice=relay-jobs.slice', '--', 'true']); + if (probe.status !== 0) return t.skip('user systemd manager unavailable'); + const ctx = setup(t, { turnMs: 30000 }); + const marker = path.join(ctx.root, 'fork-now'), childFile = path.join(ctx.root, 'forked-child.json'); + const preload = path.join(ctx.root, 'fork-during-cancel.mjs'); + const processModule = new URL('../plugins/relay/scripts/lib/util/process.mjs', import.meta.url).href; + fs.writeFileSync(preload, `import fs from 'node:fs'; import {spawn} from 'node:child_process'; +import {readProcessStartTime} from ${JSON.stringify(processModule)}; +const marker=${JSON.stringify(marker)}, childFile=${JSON.stringify(childFile)}; +if(process.argv[1]?.endsWith('/bin/codex')) { + let forked=false; + const timer=setInterval(()=>{if(!forked && fs.existsSync(marker)) { + forked=true; const child=spawn('sleep',['300'],{detached:true,stdio:'ignore'}); + child.on('spawn',()=>{fs.writeFileSync(childFile,JSON.stringify({pid:child.pid,startTime:readProcessStartTime(child.pid)}));child.unref();}); + }},5); timer.unref(); +} +if(process.argv[2]==='cancel') { + const read=fs.readFileSync; + let paused=false; + fs.readFileSync=function(file,...args) { + if(!paused && String(file).endsWith('/cgroup.procs')) { + paused=true;fs.writeFileSync(marker,'fork'); const until=Date.now()+5000; + while(!fs.existsSync(childFile) && Date.now() readJob(ctx, id).turnId); + assert.ok(readJob(ctx, id).execution.cgroup); + let child; + t.after(() => { if (child && isProcessIdentityAlive(child)) process.kill(child.pid, 'SIGKILL'); }); + const cancelled = companion(ctx, ['cancel', id]); + child = JSON.parse(fs.readFileSync(childFile)); + assert.equal(cancelled.status, 0, cancelled.stdout + cancelled.stderr); + assert.equal(readJob(ctx, id).state, 'cancelled'); + await waitFor(() => !isProcessIdentityAlive(child)); +}); + +test('worker identity storage rejects symlinks, shared modes and missing identities', async t => { + const { registerWorker } = await import('../plugins/relay/scripts/lib/worker/registry.mjs'); + const ctx = setup(t), directory = path.join(ctx.root, 'identity-storage'), link = path.join(ctx.root, 'identity-link'); + fs.mkdirSync(directory, { mode: 0o700 }); fs.symlinkSync(directory, link); + const worker = { pid: process.pid, startTime: readProcessStartTime(process.pid), jobDir: ctx.root }; + assert.throws(() => registerWorker(worker, { directory: link }), /private.*symlink/); + fs.chmodSync(directory, 0o777); + assert.throws(() => registerWorker(worker, { directory }), /private.*symlink/); + fs.chmodSync(directory, 0o700); + assert.throws(() => registerWorker({ ...worker, startTime: null }, { directory }), /identity could not be read/); + const file = registerWorker(worker, { directory }); assert.equal(JSON.parse(fs.readFileSync(file)).startTime, worker.startTime); +}); diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs index aec805f6..c41e3d3a 100644 --- a/tests/gate.test.mjs +++ b/tests/gate.test.mjs @@ -159,7 +159,7 @@ test('33 gated submissions reach backend admission with no refused or unowned wo }); test('foreground task uses a detached worker rather than the waiting caller', async t => { - const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; const script = path.join(ROOT, 'plugins/relay/scripts/companion.mjs'); const caller = spawn(process.execPath, [script, 'task', '[quick] Dummy.'], { cwd: s.ctx.work, env: s.ctx.env, stdio: 'ignore' }); const exited = once(caller, 'exit'); @@ -219,3 +219,27 @@ test('a dispatch arriving after refusal cannot acquire the worker lease or run t // The assertion fixture has no actual worker; close the explicit message for teardown. claimInboxMessages(dir); writeOutcome(dir, explicit.id, { action: 'skipped' }); }); + +test('recovery converts an adopting relay reservation rather than blocking behind it', async t => { + const s = domain(t, 1); s.ctx.env.RELAY_ADMISSION_POLL_MS = '1000'; + const dir = job(s, 'job-reserved-recovery', 'running'); + writeJob(dir, { ...readJob(dir), worker: { pid: 2147483647, startTime: 'gone' }, relay: { agentId: 'a-adopted' } }); + const place = reserveWaitingPlace(s.ctx.env, s.deps); + assert.equal(bindReservation(place.reservation, 'a-adopted', s.ctx.env, s.deps), true); + assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, false); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), true); + assert.equal(fs.existsSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `r-${place.reservation}.json`)), false); + await waitFor(() => readJob(dir).state === 'completed' && !hasLiveWorker(dir)); +}); + +test('a guard error is an error from the gate so hooks can fall back to bounded start', t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION_HOME = '/dev/null/unwritable'; + const result = companion(s.ctx, ['gate', '--wait-seconds', '30']); + assert.equal(result.status, 1); assert.match(result.stderr, /guard unavailable/); assert.equal(result.stdout, ''); +}); + +test('a plain resume with no job bypasses the retry gate even if no waiting place is free', t => { + const s = domain(t, 0); + const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-refused', '--wait-seconds', '30']); + assert.equal(result.status, 0); assert.equal(result.stdout, 'pass\n'); +}); diff --git a/tests/harness-workflow.test.mjs b/tests/harness-workflow.test.mjs index baa06ce7..478ddc6f 100644 --- a/tests/harness-workflow.test.mjs +++ b/tests/harness-workflow.test.mjs @@ -128,7 +128,7 @@ test('a gate crash falls back to the bounded start, while cancellation is denied await h.handlers['agent.spawn'](h.$, spawnEvent, h.next); assert.equal(h.calls.length, 1); const cancelled = hooks([new Error('process.run aborted')]); await assert.rejects(cancelled.handlers['agent.spawn'](cancelled.$, spawnEvent, cancelled.next), /aborted/); - assert.ok(cancelled.catches['agent.spawn'](cancelled.$, spawnEvent).deny); assert.equal(cancelled.calls.length, 0); + assert.ok(cancelled.catches['agent.spawn'](cancelled.$, spawnEvent, { called: false }).deny); assert.equal(cancelled.calls.length, 0); }); test('resume spawns do not take another waiting place', async () => { const h = hooks([]); await h.handlers['agent.spawn'](h.$, { ...spawnEvent, resume: { agentId: 'a-old' } }, h.next); @@ -142,3 +142,20 @@ test('a Workflow stall retry is gated at its first step even without agent.spawn await step.next(); assert.equal(h.calls.length, 1); assert.equal(h.runs.length, 2); assert.ok(h.runs.every(r => r.argv.includes('--retry') && r.argv.includes('a-retry'))); }); + +test('admission catches preserve a delegated result and never restart an already delegated step', async () => { + const h = hooks([]), result = { agentId: 'a-delegated' }; + let calls = 0; + const next = async () => { calls++; return result; }; next.called = true; + assert.equal(await h.catches['agent.spawn'](h.$, spawnEvent, next), result); assert.equal(calls, 1); + const step = async function* () { throw new Error('synthetic model API error'); }; step.called = true; + const caught = h.catches['turn.step'](h.$, { index: 0, turnId: 't-error' }, step); + await assert.rejects(caught.next(), /synthetic model API error/); +}); +test('a SendMessage resume skips the first-step gate even when its initial start had no job', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2']); + await h.handlers['agent.spawn'](h.$, { ...spawnEvent, resume: { agentId: 'a-resumed' } }, h.next); + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-resumed' }, h.next); + const step = h.handlers['turn.step'](h.$, { index: 0, agentId: 'a-resumed', turnId: 't-resumed' }, async function* (e) { return await h.next(e); }); + await step.next(); assert.equal(h.runs.length, 0); +}); diff --git a/tests/message-account.test.mjs b/tests/message-account.test.mjs index 7e128223..7540080b 100644 --- a/tests/message-account.test.mjs +++ b/tests/message-account.test.mjs @@ -85,7 +85,7 @@ test("cancelled with a pending continuation exits five", () => { test("worker and ownership share identity decisions and keep private helpers private", async () => { const modules = makeTempDir("relay-identity-check-"); - for (const file of ["jobs/store.mjs", "jobs/lease.mjs", "util/ownership.mjs", "util/errors.mjs", "util/fs.mjs"]) { + for (const file of ["jobs/store.mjs", "jobs/lease.mjs", "jobs/messages.mjs", "util/ownership.mjs", "util/errors.mjs", "util/fs.mjs"]) { fs.mkdirSync(path.dirname(path.join(modules, file)), { recursive: true }); fs.copyFileSync(new URL(`../plugins/relay/scripts/lib/${file}`, import.meta.url), path.join(modules, file)); } diff --git a/tests/relay-guard.test.mjs b/tests/relay-guard.test.mjs index 44c0d7e2..0bee99dc 100644 --- a/tests/relay-guard.test.mjs +++ b/tests/relay-guard.test.mjs @@ -461,13 +461,20 @@ test("without SubagentHandback, SubagentStop still checks the final text", async }); test("late unresolved cleanup preserves the copied result and stays visible in status", async () => { - const preload = path.join(makeTempDir(), "fail-launch.mjs"); - fs.writeFileSync(preload, "if (process.argv[1]?.endsWith('/launcher.mjs')) process.exit(1);\n"); - const ctx = relayContext("job", { turnMs: 50, env: { NODE_OPTIONS: `--import=${preload}` } }); - const jobId = await startedRelay(ctx, QUICK); - assert.equal(readJobJson(ctx, jobId).state, "queued"); + const ctx = relayContext("job", { turnMs: 50 }); + const { createJobDir, writeJob, jobPaths } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { readLaunchPrompt } = await import("../plugins/relay/scripts/lib/jobs/request.mjs"); + const { writeRelayPointer } = await import("../plugins/relay/scripts/lib/harness/relay.mjs"); const { runWorker } = await import("../plugins/relay/scripts/lib/worker/worker.mjs"); const { listProcesses } = await import("../plugins/relay/scripts/lib/util/process.mjs"); + ctx.append(ctx.prompt(QUICK), ctx.systemReminder()); + const jobId = "job-late-cleanup", { jobDir } = createJobDir(ctx.work, jobId, ctx.env); + const launch = readLaunchPrompt(QUICK, { baseDir: ctx.work, env: ctx.env }); + const relay = { agentId: ctx.agentId, transcriptPath: ctx.transcript }; + writeJob(jobDir, { id: jobId, kind: "task", state: "starting", createdAt: new Date().toISOString(), + pendingFirstTurn: true, attempts: [], request: launch.request, relay }); + fs.writeFileSync(jobPaths(jobDir).brief, launch.brief); + writeRelayPointer(ctx.env, { ...relay, jobId, jobDir, startError: null }); let copied; await runWorker(jobDirOf(ctx, jobId), { env: ctx.env, From 1f54ff2f94200313d8c73972ee93acf26a94d65a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:57:32 +0200 Subject: [PATCH 05/10] test: allow worker startup time in handoff fixtures on macOS Keep short refusal windows in refusal tests; positive continuation and warm-exit fixtures use a five-second startup window. Co-Authored-By: GPT-6.1 Sol --- tests/gate.test.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs index c41e3d3a..120c6cc1 100644 --- a/tests/gate.test.mjs +++ b/tests/gate.test.mjs @@ -170,7 +170,7 @@ test('foreground task uses a detached worker rather than the waiting caller', as }); test('a continuation interrupted at handoff is named as owed on the finished job and can retry', async t => { - const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; const started = companion(s.ctx, ['start', '--prompt-file', promptFile(s.ctx), '--json']); assert.equal(started.status, 0, started.stdout + started.stderr); const id = JSON.parse(started.stdout).jobId; @@ -188,7 +188,7 @@ test('a continuation interrupted at handoff is named as owed on the finished job }); test('a message racing warm-worker exit redispatches promptly after the lease disappears', async t => { - const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_ADMISSION_POLL_MS = '3000'; + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; s.ctx.env.RELAY_ADMISSION_POLL_MS = '3000'; const dir = job(s, 'job-warm-exit', 'completed'); const leaseModule = new URL('../plugins/relay/scripts/lib/jobs/lease.mjs', import.meta.url).href; const processModule = new URL('../plugins/relay/scripts/lib/util/process.mjs', import.meta.url).href; From 811db23b25a34b33203e40d05485974ff9afc15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 22:59:07 +0200 Subject: [PATCH 06/10] gate: require transcript evidence before exempting plain resumes Unknown retry provenance waits for capacity; a known plain transcript still bypasses the retry gate. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/scripts/cli/gate.mjs | 6 +++--- tests/gate.test.mjs | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/relay/scripts/cli/gate.mjs b/plugins/relay/scripts/cli/gate.mjs index f7a0a04d..d8d2f3d8 100644 --- a/plugins/relay/scripts/cli/gate.mjs +++ b/plugins/relay/scripts/cli/gate.mjs @@ -8,12 +8,12 @@ import { findAdoptableRelayJob, findRelayJob, promptSha256, readJournalKey, read // A stalled Workflow attempt gets a fresh agent id without agent.spawn. At its // first step, reuse an existing job or reservation before asking for another place. function retryAlreadyCovered(agentId, transcriptPath, env) { - let workflow = false; + let workflow = false, plain = false; if (findRelayJob(env, agentId)) return true; for (const file of transcriptCandidates({ agent_id: agentId, transcript_path: transcriptPath })) { if (!fs.existsSync(file)) continue; const run = /[\\/]workflows[\\/]([^\\/]+)[\\/]agent-[^\\/]+\.jsonl$/.exec(file)?.[1]; - if (!run) continue; + if (!run) { plain = true; continue; } workflow = true; const transcript = readRelayTranscript(file); let meta; @@ -22,7 +22,7 @@ function retryAlreadyCovered(agentId, transcriptPath, env) { promptSha256: promptSha256(transcript.prompt?.text), label: meta?.description })) return true; } // Plain resumes are never capacity-gated, including a start previously refused. - return !workflow; + return plain && !workflow; } export async function handleGate(argv) { diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs index 120c6cc1..7fa54d4c 100644 --- a/tests/gate.test.mjs +++ b/tests/gate.test.mjs @@ -240,6 +240,16 @@ test('a guard error is an error from the gate so hooks can fall back to bounded test('a plain resume with no job bypasses the retry gate even if no waiting place is free', t => { const s = domain(t, 0); - const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-refused', '--wait-seconds', '30']); + const transcript = path.join(s.ctx.root, 'subagents', 'agent-a-refused.jsonl'); + fs.mkdirSync(path.dirname(transcript)); fs.writeFileSync(transcript, ''); + const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-refused', '--transcript', transcript, '--wait-seconds', '30']); assert.equal(result.status, 0); assert.equal(result.stdout, 'pass\n'); }); + + +test('an unclassified retry with no transcript waits for capacity instead of bypassing the gate', t => { + const s = domain(t, 0); + const transcript = path.join(s.ctx.root, 'subagents', 'workflows', 'wf-test', 'agent-a-unknown.jsonl'); + const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-unknown', '--transcript', transcript]); + assert.equal(result.status, 0); assert.match(result.stdout, /^hold waiting worker limit: 0 of 0/); +}); From 99706d936d0f09eb37562ece57ba7fddca8ae7b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 23:11:30 +0200 Subject: [PATCH 07/10] lifecycle: fail explicitly when message storage cannot be verified Treat only absent message directories as empty. Check message storage before terminal publication and correct an uncertain final handoff under the job lease without overwriting a successor. Cover finalization and both admission hooks with storage-failure regressions. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/CHANGELOG.md | 1 + plugins/relay/scripts/lib/jobs/messages.mjs | 38 +++++------ plugins/relay/scripts/lib/worker/worker.mjs | 23 ++++++- tests/cancel-dead.test.mjs | 4 +- tests/grok-delivery.test.mjs | 3 +- tests/harness-workflow.test.mjs | 37 ++++++++++ tests/message-storage.test.mjs | 76 +++++++++++++++++++++ 7 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 tests/message-storage.test.mjs diff --git a/plugins/relay/CHANGELOG.md b/plugins/relay/CHANGELOG.md index e7a859a9..789f299c 100644 --- a/plugins/relay/CHANGELOG.md +++ b/plugins/relay/CHANGELOG.md @@ -24,6 +24,7 @@ Changed: Fixed: +- Unreadable message storage fails explicitly and preserves owed work; it cannot be mistaken for an empty inbox at completion. - Continuations and recovery use the same bounded launch path. Undelivered pending messages are withdrawn rather than acknowledged without a worker, and interrupted recovery keeps its message id. - Repeated launch failures stop after three attempts and show the error and count. Handoff retries diff --git a/plugins/relay/scripts/lib/jobs/messages.mjs b/plugins/relay/scripts/lib/jobs/messages.mjs index cb9abf9d..fc5f21d8 100644 --- a/plugins/relay/scripts/lib/jobs/messages.mjs +++ b/plugins/relay/scripts/lib/jobs/messages.mjs @@ -27,18 +27,26 @@ export function postInboxMessage(jobDir, message) { } -export function pendingInboxMessages(jobDir) { - const paths = jobPaths(jobDir); - try { - return fs - .readdirSync(paths.inbox) - .filter((name) => name.endsWith(".json") && !name.startsWith(".")) - .sort(); - } catch { - return []; +export class InboxStorageError extends Error { + constructor(directory, cause) { + super(`Message storage uncertain: cannot list ${directory} (${cause.code ?? cause.message}); completion cannot be confirmed.`, { cause }); + this.name = "InboxStorageError"; } } +function messageNames(directory) { + try { return fs.readdirSync(directory); } + catch (error) { + if (error.code === "ENOENT") return []; + throw new InboxStorageError(directory, error); + } +} + +export function pendingInboxMessages(jobDir) { + return messageNames(jobPaths(jobDir).inbox) + .filter(name => name.endsWith(".json") && !name.startsWith(".")).sort(); +} + export function claimInboxMessages(jobDir) { const paths = jobPaths(jobDir); @@ -175,10 +183,7 @@ const WORK_KINDS = new Set(["steer", "continue", "resume"]); export function readJobMessages(jobDir) { const paths = jobPaths(jobDir); const relays = new Map(); - const namesIn = (dir) => { - try { return fs.readdirSync(dir).filter(name => name.endsWith(".json") && !name.startsWith(".")); } - catch { return []; } - }; + const namesIn = dir => messageNames(dir).filter(name => name.endsWith(".json") && !name.startsWith(".")); const relayDir = path.join(paths.inbox, "relay"); for (const name of namesIn(relayDir).sort()) { const record = readJsonOrNull(path.join(relayDir, name)); @@ -211,12 +216,7 @@ export function readJobMessages(jobDir) { function readClaimedMessages(jobDir) { const paths = jobPaths(jobDir); - let names = []; - try { - names = fs.readdirSync(paths.claimed).filter((name) => name.endsWith(".json")).sort(); - } catch { - return []; - } + const names = messageNames(paths.claimed).filter(name => name.endsWith(".json")).sort(); return names .map((name) => readJsonOrNull(path.join(paths.claimed, name))) .filter(Boolean) diff --git a/plugins/relay/scripts/lib/worker/worker.mjs b/plugins/relay/scripts/lib/worker/worker.mjs index 4b75716a..84343647 100644 --- a/plugins/relay/scripts/lib/worker/worker.mjs +++ b/plugins/relay/scripts/lib/worker/worker.mjs @@ -20,7 +20,7 @@ import { requestAccount } from "../jobs/request.mjs"; import { backendFor } from "../backends/index.mjs"; import { truncateCodePoints } from "../util/text.mjs"; import { acquireWorkerLease, releaseWorkerLease, readLease } from "../jobs/lease.mjs"; -import { claimInboxMessages, CLOSED_ACTIONS, newMessageId, owedMessages, pendingInboxMessages, readOutcome, writeOutcome } from "../jobs/messages.mjs"; +import { InboxStorageError, claimInboxMessages, CLOSED_ACTIONS, newMessageId, owedMessages, pendingInboxMessages, readOutcome, writeOutcome } from "../jobs/messages.mjs"; import { jobPaths, nowIso, readJob, writeFileAtomic, writeJob, readJsonOrNull, writeJsonAtomic } from "../jobs/store.mjs"; import { PLUGIN_VERSION } from "../util/plugin.mjs"; import { ownExecution, readProcessStartTime, listProcesses, readProcessTableCached, sampleExecution, executionBytes, isProcessIdentityAlive } from "../util/process.mjs"; @@ -187,6 +187,24 @@ export async function runWorker(jobDir, options = {}) { return result; } } + } catch (error) { + // A final handoff read can fail after the round saved its terminal state. + // Only this worker's lease authorizes correcting that provisional completion. + if (error instanceof InboxStorageError) { + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + const holder = readLease(jobDir); + const held = holder?.role === "worker" && sameProcess(holder, me); + // The final post-release read can also fail. Reacquire the fence before + // correcting our own completion; never overwrite a successor's record. + const acquired = !held && (!holder || !isProcessIdentityAlive(holder)) && acquireWorkerLease(jobDir, me, "storage"); + if (held || acquired) { + try { + const job = readJob(jobDir); + if (held || sameProcess(job?.worker, me)) writeJob(jobDir, { ...job, state: "failed", errorMessage: error.message, completedAt: nowIso() }); + } finally { releaseWorkerLease(jobDir, me); } + } + } + throw error; } finally { try { if (registryFile) fs.rmSync(registryFile, { force: true }); } catch {} } @@ -916,6 +934,9 @@ async function runWorkerRound(jobDir, options, held) { } clearInterval(poller); + // A successful turn is not completion evidence for unreadable message storage. + pendingInboxMessages(jobDir); + owedMessages(jobDir); const last = job.attempts.at(-1); if (cancelRequested) { if (last && !TERMINAL.has(last.state)) { diff --git a/tests/cancel-dead.test.mjs b/tests/cancel-dead.test.mjs index 44d9269f..67359234 100644 --- a/tests/cancel-dead.test.mjs +++ b/tests/cancel-dead.test.mjs @@ -238,7 +238,7 @@ fs.readFileSync = function(file, ...args) { }); -test("a completed job with a closing worker records one no-op cancel without delegating another", async t => { +test("a completed job with a closing worker records one cleanup receipt without delegating another", async t => { const ctx = setup(t); const worker = await sleeper(t, true), server = await sleeper(t, false); const { id, jobDir } = recorded(ctx, worker.identity, server.identity, { state: "completed", attempts: [{ attempt: 1, state: "completed" }] }); @@ -251,5 +251,5 @@ test("a completed job with a closing worker records one no-op cancel without del const messages = ["inbox", "inbox/claimed"].flatMap(dir => fs.readdirSync(path.join(jobDir, dir)).filter(n => n.endsWith(".json")).map(n => JSON.parse(fs.readFileSync(path.join(jobDir, dir, n))))).filter(m => m.kind === "cancel"); assert.equal(messages.length, 1); const outcome = JSON.parse(fs.readFileSync(path.join(jobDir, "inbox/outcomes", `${messages[0].id}.json`))); - assert.equal(outcome.action, "no-op"); + assert.equal(outcome.action, "cancelled"); }); diff --git a/tests/grok-delivery.test.mjs b/tests/grok-delivery.test.mjs index 84f21254..9628f26f 100644 --- a/tests/grok-delivery.test.mjs +++ b/tests/grok-delivery.test.mjs @@ -430,7 +430,8 @@ import { buildLaunchRequest } from "../plugins/relay/scripts/lib/jobs/request.mj } else { await waitFor(() => companion(ctx, ['status', id]).stdout.includes('state: dead')); } - assert.equal(companion(ctx, ['resume', id, '--message', '[quick] Recover.']).status, 0); + const resumed = companion(ctx, ['resume', id, '--message', '[quick] Recover.']); + assert.equal(resumed.status, 0, resumed.stdout + resumed.stderr); await finishJob(ctx, id); const after = readJob(ctx, id); assert.equal(after.sessionId, before.sessionId); diff --git a/tests/harness-workflow.test.mjs b/tests/harness-workflow.test.mjs index 478ddc6f..12e478cd 100644 --- a/tests/harness-workflow.test.mjs +++ b/tests/harness-workflow.test.mjs @@ -159,3 +159,40 @@ test('a SendMessage resume skips the first-step gate even when its initial start const step = h.handlers['turn.step'](h.$, { index: 0, agentId: 'a-resumed', turnId: 't-resumed' }, async function* (e) { return await h.next(e); }); await step.next(); assert.equal(h.runs.length, 0); }); + +for (const event of ['agent.spawn', 'turn.step']) test(`${event} gate storage uncertainty reaches bounded start and cannot admit a worker`, async t => { + const { setup, companion, promptFile, jobDirs } = await import('./support/job-helpers.mjs'); + const { readJob } = await import('../plugins/relay/scripts/lib/jobs/store.mjs'); + const { hasLiveWorker } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const ctx = setup(t, { env: { RELAY_ADMISSION: 'on', RELAY_ADMISSION_HOME: '/dev/null/unwritable', RELAY_JOBS_ACK_TIMEOUT_MS: '100', RELAY_ADMISSION_POLL_MS: '20' } }); + const h = hooks([]); + const gateResults = []; + h.$.process.run = async argv => { + if (argv[2] === 'check-header') return { exitCode: 0, stdout: 'ok\n' }; + const result = companion(ctx, argv.slice(2)); gateResults.push(result); + return { exitCode: result.status, stdout: result.stdout, stderr: result.stderr }; + }; + let starts = 0; + const boundedStart = () => { + starts++; + const result = companion(ctx, ['start', '--prompt-file', promptFile(ctx)]); + assert.equal(result.status, 6, result.stdout + result.stderr); + assert.match(result.stdout, /state: not started/); + return { agentId: 'a-refused' }; + }; + if (event === 'agent.spawn') await h.handlers[event](h.$, { ...spawnEvent, workflow: { runId: 'wf-storage' } }, boundedStart); + else { + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-unknown', transcript_path: '/missing/workflows/wf-storage/agent-a-unknown.jsonl' }, h.next); + const step = h.handlers[event](h.$, { index: 0, agentId: 'a-unknown', turnId: 't-storage' }, async function* () { return boundedStart(); }); + await step.next(); + } + assert.equal(starts, 1); + assert.equal(gateResults.length, 1); + assert.equal(gateResults[0].status, 1); + assert.match(gateResults[0].stderr, /guard unavailable/); + for (const dir of jobDirs(ctx)) { + assert.equal(readJob(dir).admission.state, 'not-started'); + assert.equal(hasLiveWorker(dir), false); + assert.equal(readJob(dir).attempts.length, 0); + } +}); diff --git a/tests/message-storage.test.mjs b/tests/message-storage.test.mjs new file mode 100644 index 00000000..5d4cfe88 --- /dev/null +++ b/tests/message-storage.test.mjs @@ -0,0 +1,76 @@ +import './support/isolate-env.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { setup } from './support/job-helpers.mjs'; +import { createJobDir, readJob, writeJob, jobPaths } from '../plugins/relay/scripts/lib/jobs/store.mjs'; +import { readLaunchPrompt } from '../plugins/relay/scripts/lib/jobs/request.mjs'; +import { messageAccount, postInboxMessage } from '../plugins/relay/scripts/lib/jobs/messages.mjs'; +import { readLease } from '../plugins/relay/scripts/lib/jobs/lease.mjs'; +import { isProcessIdentityAlive } from '../plugins/relay/scripts/lib/util/process.mjs'; +import { runWorker } from '../plugins/relay/scripts/lib/worker/worker.mjs'; + +function fixture(t) { + const ctx = setup(t, { turnMs: 50, env: { RELAY_ADMISSION: 'off', RELAY_WORKER_SCOPE: 'off' } }); + const { jobDir } = createJobDir(ctx.work, 'job-storage', ctx.env); + const { request, brief } = readLaunchPrompt('model: gpt-test\neffort: low\n---\n[quick] Finish.', { baseDir: ctx.work, env: ctx.env }); + writeJob(jobDir, { id: 'job-storage', kind: 'task', state: 'starting', pendingFirstTurn: true, attempts: [], request }); + fs.writeFileSync(jobPaths(jobDir).brief, brief); + return { ctx, jobDir, paths: jobPaths(jobDir) }; +} + +for (const code of ['EACCES', 'EIO']) for (const stage of ['inbox', 'claimed', 'handoff', 'released']) { + test(`${code} reading ${stage} during worker finalization fails explicitly and preserves owed work`, async t => { + const { ctx, jobDir, paths } = fixture(t); + const readdir = fs.readdirSync; + let armed = false, injected = false, message, completionReads = 0; + const mock = t.mock.method(fs, 'readdirSync', function (directory, ...args) { + const job = readJob(jobDir); + if (String(directory) === paths.inbox && job.state === 'completed') completionReads++; + const finalizing = stage === 'released' ? completionReads >= 2 : stage === 'handoff' ? completionReads >= 1 : job.attempts?.at(-1)?.state === 'completed'; + if (!armed && finalizing) { + message = postInboxMessage(jobDir, { kind: 'continue', text: 'Owed follow-up.' }); + if (stage === 'claimed') { + fs.mkdirSync(paths.claimed, { recursive: true }); + fs.renameSync(path.join(paths.inbox, `${message.id}.json`), path.join(paths.claimed, `${message.id}.json`)); + } + armed = true; + } + if (armed && String(directory) === (stage === 'claimed' ? paths.claimed : paths.inbox)) { + injected = true; + throw Object.assign(new Error('synthetic message storage failure'), { code }); + } + return readdir.call(this, directory, ...args); + }); + let rejected; + try { await runWorker(jobDir, { env: ctx.env }); } catch (error) { rejected = error; } + finally { mock.mock.restore(); } + assert.equal(injected, true); + const job = readJob(jobDir); + assert.equal(job.state, 'failed'); + assert.match(job.errorMessage, new RegExp(`storage uncertain:.*${code}`)); + if (rejected) assert.match(rejected.message, /storage uncertain/); + assert.deepEqual(messageAccount(jobDir).undelivered, [message.id]); + assert.equal(readLease(jobDir), null); + assert.equal(ctx.fakeState().appServerStarts, 1); + assert.equal(isProcessIdentityAlive(job.server), false); + }); +} + +test('absent message directories allow an empty worker to complete', async t => { + const { ctx, jobDir, paths } = fixture(t); + const readdir = fs.readdirSync; + let injected = false; + const mock = t.mock.method(fs, 'readdirSync', function (directory, ...args) { + if ([paths.inbox, paths.claimed].includes(String(directory)) && readJob(jobDir).attempts?.at(-1)?.state === 'completed') { + injected = true; + throw Object.assign(new Error('absent directory'), { code: 'ENOENT' }); + } + return readdir.call(this, directory, ...args); + }); + try { await runWorker(jobDir, { env: ctx.env }); } finally { mock.mock.restore(); } + assert.equal(injected, true); + assert.equal(readJob(jobDir).state, 'completed'); + assert.equal(readLease(jobDir), null); +}); From e9ba86617c0e2b545c060cae42f6251b7b1bb5a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 24 Oct 2026 23:20:35 +0200 Subject: [PATCH 08/10] adoption: refuse uncertain message storage before creating another job Propagate storage uncertainty from matching adoption candidates and record its delivery violation. Keep queued-workflow fixtures owned by a live lease. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/scripts/lib/harness/relay.mjs | 7 ++++++- tests/relay-delivery.test.mjs | 8 +++++--- tests/relay-guard.test.mjs | 10 ++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/plugins/relay/scripts/lib/harness/relay.mjs b/plugins/relay/scripts/lib/harness/relay.mjs index 4eb4d1c1..f387ede8 100644 --- a/plugins/relay/scripts/lib/harness/relay.mjs +++ b/plugins/relay/scripts/lib/harness/relay.mjs @@ -21,7 +21,7 @@ import path from "node:path"; import process from "node:process"; import { awaitOutcome, hasLiveWorker, COMPANION_SCRIPT } from "../worker/delivery.mjs"; -import { appendRelayViolation, messageExists, messageAccount, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; +import { InboxStorageError, appendRelayViolation, messageExists, messageAccount, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; import { absoluteJobsHomeEnv, listJobDirs, nowIso, readJob, jobPaths, readJsonOrNull, resolveJobsHome, TERMINAL_STATES, writeJsonAtomic } from "../jobs/store.mjs"; import { isTerminalView, JobViewBusy, loadJobView } from "../jobs/view.mjs"; import { acquireOwnership, releaseOwnership } from "../util/ownership.mjs"; @@ -526,6 +526,11 @@ export function findAdoptableRelayJob(env, { agentId, workflowRunId, journalKey, try { state = loadJobView(dir).state; } catch (error) { + // Uncertain message storage cannot turn an existing call into a new job. + if (error instanceof InboxStorageError) { + appendRelayViolation(dir, { kind: "delivery", reason: `messages could not be delivered: ${error.message}` }); + throw error; + } // A busy job is at work: it stays a candidate. A directory without job.json does not. if (!(error instanceof JobViewBusy)) continue; state = null; diff --git a/tests/relay-delivery.test.mjs b/tests/relay-delivery.test.mjs index ba59fdd4..7b76a2b9 100644 --- a/tests/relay-delivery.test.mjs +++ b/tests/relay-delivery.test.mjs @@ -497,14 +497,16 @@ test("setup before posting on the adoption start path records one violation", as fs.writeFileSync(inbox, "obstruct inbox setup"); try { const { started } = await relayStart(retry); - assert.notEqual(started.status, 0); + assert.notEqual(started.status, 0, started.stdout + started.stderr); + const { listJobDirs } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + assert.deepEqual(listJobDirs({ env: retry.env }), [dir], "uncertain adoption must never submit a duplicate job"); assert.equal(readRelayViolations(dir).length, 1); - assert.match(readRelayViolations(dir)[0].reason, /messages could not be delivered: EEXIST/); + assert.match(readRelayViolations(dir)[0].reason, /messages could not be delivered: Message storage uncertain:.*ENOTDIR/); } finally { fs.unlinkSync(inbox); fs.renameSync(saved, inbox); } - assert.match(companion(first, ["result", jobId]).stdout, /^relay-violation: delivery: .*EEXIST/m); + assert.match(companion(first, ["result", jobId]).stdout, /^relay-violation: delivery: .*storage uncertain:.*ENOTDIR/m); }); test("wait false reports no consumer on timeout", async () => { diff --git a/tests/relay-guard.test.mjs b/tests/relay-guard.test.mjs index 0bee99dc..db6264ed 100644 --- a/tests/relay-guard.test.mjs +++ b/tests/relay-guard.test.mjs @@ -1553,7 +1553,7 @@ test("a Workflow relay that never runs start gets the Workflow correction with i assert.equal(waitJob(ctx, jobId).status, 0); }); -test("a queued Workflow stays pending through repeated hand-back and stop attempts beyond the escape budget", async () => { +test("a queued Workflow stays pending through repeated hand-back and stop attempts beyond the escape budget", async t => { const ctx = relayContext("job", { workflow: WORKFLOW, turnMs: 100 }); ctx.append(ctx.userRequest(), ctx.environment(), ctx.workflowPrompt("computedTask", QUICK), ctx.snapshot("string")); const { jobId } = await relayStart(ctx); @@ -1563,11 +1563,17 @@ test("a queued Workflow stays pending through repeated hand-back and stop attemp await waitFor(() => !readLease(dir)); const file = path.join(dir, "job.json"); const job = JSON.parse(fs.readFileSync(file, "utf8")); + const { acquireWorkerLease, releaseWorkerLease, isWorkerAlive } = await import("../plugins/relay/scripts/lib/jobs/lease.mjs"); + const { readProcessStartTime } = await import("../plugins/relay/scripts/lib/util/process.mjs"); + await waitFor(() => !isWorkerAlive(job.worker)); + const worker = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + assert.equal(acquireWorkerLease(dir, worker), true); + t.after(() => { releaseWorkerLease(dir, worker); fs.writeFileSync(file, JSON.stringify(job)); }); ctx.env.RELAY_ADMISSION = "on"; ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root ?? path.dirname(dir), "synthetic-admission"); ctx.env.RELAY_TEST_ADMISSION_READINGS = path.join(dir, "synthetic-readings.json"); fs.writeFileSync(ctx.env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify({ availableBytes: 0, totalBytes: 40 * 2 ** 30, cores: 8, load5: 0 })); - fs.writeFileSync(file, JSON.stringify({ ...job, state: "queued", worker: null, admission: { state: "waiting", waitKind: "worker", reason: "memory low", since: new Date().toISOString() } })); + fs.writeFileSync(file, JSON.stringify({ ...job, state: "queued", worker, admission: { state: "waiting", waitKind: "memory", reason: "memory low", since: new Date().toISOString() } })); for (let i = 0; i < RELAY_BLOCK_BUDGET + 2; i++) { assert.equal(decision(await pre(ctx, "SubagentHandback", { message: "done" })), "deny"); const blocked = await stop(ctx, "done"); From a0ff97001fd02d6b7c4eee151da20cd3d289007d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sun, 25 Oct 2026 00:16:07 +0200 Subject: [PATCH 09/10] recovery: persist the retry allowance before publishing delivery receipts A caller can stop a worker as soon as it observes delivery. Save the user message recovery epoch before that receipt, and test the exact post-publication death window. Recovery tests wait for the successor turn before cancellation. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/CHANGELOG.md | 1 + plugins/relay/scripts/lib/worker/worker.mjs | 5 ++- tests/recovery.test.mjs | 44 ++++++++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/plugins/relay/CHANGELOG.md b/plugins/relay/CHANGELOG.md index 789f299c..8aaf5003 100644 --- a/plugins/relay/CHANGELOG.md +++ b/plugins/relay/CHANGELOG.md @@ -24,6 +24,7 @@ Changed: Fixed: +- Delivered corrections retain their automatic recovery allowance if the worker exits during acknowledgement. - Unreadable message storage fails explicitly and preserves owed work; it cannot be mistaken for an empty inbox at completion. - Continuations and recovery use the same bounded launch path. Undelivered pending messages are withdrawn rather than acknowledged without a worker, and interrupted recovery keeps its message id. diff --git a/plugins/relay/scripts/lib/worker/worker.mjs b/plugins/relay/scripts/lib/worker/worker.mjs index 84343647..9d4f7cc4 100644 --- a/plugins/relay/scripts/lib/worker/worker.mjs +++ b/plugins/relay/scripts/lib/worker/worker.mjs @@ -297,9 +297,12 @@ async function runWorkerRound(jobDir, options, held) { if (!message) { return; } + const reset = !message.automatic && ["steered", "turn-started", "delivered-before"].includes(result.action); + // A visible delivery receipt must already carry its durable recovery allowance. + // The caller can stop this worker immediately after observing the receipt. + if (reset && job.recoveryEpoch !== message.id) save({ recoveryEpoch: message.id }); writeOutcome(jobDir, message.id, result); const log = Array.isArray(job.messages) ? job.messages : []; - const reset = !message.automatic && ["steered", "turn-started", "delivered-before"].includes(result.action); save({ messages: [...log, { id: message.id, kind: message.kind, at: nowIso(), ...result }], ...(reset ? { recoveryEpoch: message.id } : {}) }); }; diff --git a/tests/recovery.test.mjs b/tests/recovery.test.mjs index 6580d74d..a3154adf 100644 --- a/tests/recovery.test.mjs +++ b/tests/recovery.test.mjs @@ -189,9 +189,49 @@ for (const persisted of [false, true]) { // Delivering a message into the automatic turn earns a fresh retry too. assert.equal(companion(ctx, ["steer", id, "--message", "User correction"]).status, 0); killWorker(ctx, id); - await waitFor(() => readJobJson(ctx, id).automaticResumes.length === 3 && readJobJson(ctx, id).turnId); + await waitFor(() => turnRecorded(ctx, id, 5)); + assert.equal(readJobJson(ctx, id).automaticResumes.length, 3); companion(ctx, ["cancel", id]); - assert.equal((await waitingAgain).status, 5); + const cancelled = await waitingAgain; + assert.equal(cancelled.status, 5, cancelled.stdout + cancelled.stderr); + }); + + test("a delivered steer rearms recovery before its receipt can be observed", async t => { + const ctx = relayContext("job", { turnMs: 30000 }); + const marker = path.join(ctx.home, "receipt-published"), release = path.join(ctx.home, "release-receipt"); + const preload = path.join(ctx.home, "freeze-after-receipt.mjs"); + fs.writeFileSync(preload, `import fs from "node:fs"; +const rename = fs.renameSync; +fs.renameSync = function (from, to, ...args) { + const result = rename.call(this, from, to, ...args); + if (process.argv[2] === "worker" && String(to).includes("/inbox/outcomes/") && fs.readFileSync(to, "utf8").includes('"action": "steered"')) { + fs.writeFileSync(${JSON.stringify(marker)}, ""); + while (!fs.existsSync(${JSON.stringify(release)})) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + return result; +};`); + ctx.env.NODE_OPTIONS = `${ctx.env.NODE_OPTIONS ?? ""} --import ${preload}`; + t.after(() => fs.writeFileSync(release, "")); + const id = await start(ctx); + killWorker(ctx, id); + const waiting = waitAsync(ctx, await waitInput(ctx, id)); + t.after(async () => { + fs.writeFileSync(release, ""); + companion(ctx, ["cancel", id]); + await waiting; + }); + await waitFor(() => turnRecorded(ctx, id, 2)); + const sent = companion(ctx, ["steer", id, "--message", "Durable user correction"]); + assert.equal(sent.status, 0, sent.stdout + sent.stderr); + await waitFor(() => fs.existsSync(marker)); + const messageId = /^message: steer (.+)$/m.exec(sent.stdout)[1]; + assert.equal(readJobJson(ctx, id).recoveryEpoch, messageId, "the receipt cannot precede its retry allowance"); + killWorker(ctx, id); + await waitFor(() => turnRecorded(ctx, id, 3)); + assert.equal(readJobJson(ctx, id).automaticResumes.length, 2); + companion(ctx, ["cancel", id]); + const cancelled = await waiting; + assert.equal(cancelled.status, 5, cancelled.stdout + cancelled.stderr); }); test("two concurrent relay waits reserve only one automatic resume", async () => { From 6fa691b4edbc29e0facac6d706bfc51bddfb526a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sun, 25 Oct 2026 00:36:23 +0200 Subject: [PATCH 10/10] recovery: never restore a dead view over cancellation or a successor Fence the unsuccessful handoff restore against terminal states and a newly published worker identity. Add deterministic interleavings at the final recovery lease. Co-Authored-By: GPT-6.1 Sol --- plugins/relay/CHANGELOG.md | 1 + plugins/relay/scripts/lib/worker/recovery.mjs | 9 ++++- tests/recovery.test.mjs | 37 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/plugins/relay/CHANGELOG.md b/plugins/relay/CHANGELOG.md index 8aaf5003..b87ff089 100644 --- a/plugins/relay/CHANGELOG.md +++ b/plugins/relay/CHANGELOG.md @@ -24,6 +24,7 @@ Changed: Fixed: +- Recovery preserves a cancellation or successor worker that wins during the dispatch handoff. - Delivered corrections retain their automatic recovery allowance if the worker exits during acknowledgement. - Unreadable message storage fails explicitly and preserves owed work; it cannot be mistaken for an empty inbox at completion. - Continuations and recovery use the same bounded launch path. Undelivered pending messages are diff --git a/plugins/relay/scripts/lib/worker/recovery.mjs b/plugins/relay/scripts/lib/worker/recovery.mjs index 212d9538..c963c834 100644 --- a/plugins/relay/scripts/lib/worker/recovery.mjs +++ b/plugins/relay/scripts/lib/worker/recovery.mjs @@ -8,7 +8,7 @@ import { releaseLeaseToConsumer, startWithinWindow } from "./delivery.mjs"; import { acquireWorkerLease, isWorkerAlive } from "../jobs/lease.mjs"; import { computeJobView } from "../jobs/view.mjs"; import { messageExists, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; -import { nowIso, readJob, readJsonOrNull, writeJob } from "../jobs/store.mjs"; +import { nowIso, readJob, readJsonOrNull, TERMINAL_STATES, writeJob } from "../jobs/store.mjs"; import { stopExecution, readProcessStartTime } from "../util/process.mjs"; // The sweep: stop those app-servers with every descendant, SIGTERM then SIGKILL. Returns @@ -92,7 +92,12 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { if (acquireWorkerLease(jobDir, me, "recovery")) { try { const job = readJob(jobDir), record = recoveryForEpoch(job); - if (!result.accepted && record?.state === "reserved") writeJob(jobDir, { ...job, state: original.state, worker: original.worker }); + // Cancellation or a successor can win while dispatch is being observed. + // Restoring the old dead view must never overwrite either publication. + const changedWorker = job.worker && (job.worker.pid !== original.worker?.pid || job.worker.startTime !== original.worker?.startTime); + if (!result.accepted && record?.state === "reserved" && !TERMINAL_STATES.has(job.state) && !changedWorker) { + writeJob(jobDir, { ...job, state: original.state, worker: original.worker }); + } if (record?.state === "reserved" && result.accepted) writeJob(jobDir, { ...job, automaticResumes: job.automaticResumes.map(entry => entry.id === record.id ? { ...entry, state: "launched" } : entry) }); } finally { releaseLeaseToConsumer(jobDir, me, env, { spawn: false }); } diff --git a/tests/recovery.test.mjs b/tests/recovery.test.mjs index a3154adf..842debf7 100644 --- a/tests/recovery.test.mjs +++ b/tests/recovery.test.mjs @@ -910,3 +910,40 @@ for (const gone of ["removed", "dead", "reused pid"]) { assert.equal(readJob(ctx, id).automaticResumes.length, 1); }); } + + +for (const winner of ["cancel", "successor"]) test(`recovery refusal preserves a ${winner} published before its final lease`, async t => { + const ctx = setup(t, { env: { RELAY_ADMISSION: "on", RELAY_MAX_WAITING_WORKERS: "0", RELAY_ADMISSION_POLL_MS: "20" } }); + ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root, "admission"); + const { jobDir } = createJobDir(ctx.work, "job-recovery-fence", ctx.env); + const original = { id: "job-recovery-fence", kind: "task", state: "running", pendingFirstTurn: true, attempts: [], + worker: { pid: 2147483647, startTime: "gone" }, + request: buildLaunchRequest({ model: "gpt-test", effort: "low", cd: ctx.work }, { env: ctx.env, baseDir: ctx.work }) }; + writeJob(jobDir, original); + fs.writeFileSync(path.join(jobDir, "brief.md"), "Never start under full capacity."); + let successor; + if (winner === "successor") { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + await once(child, "spawn"); + successor = { pid: child.pid, startTime: readProcessStartTime(child.pid) }; + t.after(() => child.kill("SIGKILL")); + } + const directory = ownershipDir(path.join(jobDir, "worker.lease")); + const link = fs.linkSync; + let recoveryLeases = 0, published = false; + const mock = t.mock.method(fs, "linkSync", function (from, to, ...args) { + if (path.dirname(String(to)) === directory && JSON.parse(fs.readFileSync(from, "utf8")).role === "recovery" && ++recoveryLeases === 2) { + const job = JSON.parse(fs.readFileSync(path.join(jobDir, "job.json"), "utf8")); + writeJob(jobDir, { ...job, ...(winner === "cancel" ? { state: "cancelled", completedAt: new Date().toISOString() } : { worker: successor }) }); + published = true; + } + return link.call(this, from, to, ...args); + }); + try { assert.equal(await resumeDeadJobOnce(jobDir, ctx.env), false); } + finally { mock.mock.restore(); } + assert.equal(published, true); + const job = JSON.parse(fs.readFileSync(path.join(jobDir, "job.json"), "utf8")); + if (winner === "cancel") assert.equal(job.state, "cancelled"); + else assert.deepEqual(job.worker, successor); + assert.equal(job.attempts.length, 0); +});