diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6b3c05c9..cfc7b7cf 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,13 +5,13 @@ }, "metadata": { "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", - "version": "3.0.0-dev.24" + "version": "3.0.0-dev.25" }, "plugins": [ { "name": "relay", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "author": { "name": "Libar" }, diff --git a/README.md b/README.md index 8bc495d9..fbc6f0ea 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ The plugin's hooks module answers `/relay:status`, `/relay:result`, `/relay:canc tokens spent. The answer starts with the command as you typed it, then the companion's lines in a code block, as it prints them, then a job's report as Markdown, or the job list as a table. `claude -p "/relay:result "` exits with the companion's exit code (0 completed with a report; -3 failed, dead, completed without a report, or messages not yet run; 4 queued or running; 5 cancelled), with +3 failed, dead, completed without a report, or messages not yet run; 4 queued or running; 5 cancelled; 6 not started), with `--output-format text` or `json`; under `stream-json` Claude Code exits 0. These four run only when you type them. When you ask in plain words, Claude reads jobs with the companion in Bash, which a session in `default` permission mode asks you about, and cancels a job by sending its agent the @@ -350,14 +350,12 @@ Defaults use physical total memory `T` in bytes, `GiB = 2^30` bytes and `max`/`ultra` on other models, never enters this pool. Opaque defaults remain unknown; only identified selections are covered. - At most `Q = max(8, 2F)` resident waiting workers, including workers being spawned, are kept. - Worker creation holds `64 MiB` for 30 seconds against the same host/scope budget. A full - worker budget or insufficient memory leaves durable work without a worker and returns its - original job ID. Keep calling `wait`: the submitter/adopter's wait, message or relay-hook - loop retries worker creation at most once per admission poll (3 seconds by default). - A bare `start` can remain dormant until someone waits; there is no background queue service. - `status` only reads jobs: use `wait ` to adopt deferred work. A workerless job with no - spawn retry for 15 minutes stops deferring the stop-time review gate; it remains adoptable. - The limit bounds workers, not all submissions, native transcripts or waiting clients. + Worker creation holds `64 MiB` for 30 seconds against the same host/scope budget. + A job is accepted only once its own worker owns it; a job that cannot get a worker within + a minute is reported as not started (exit 6), and nothing runs later. There is no durable queue. + Workflow relays wait at spawn as queued until a waiting place frees, without a model turn + or a job waiter. Plain `relay:job` calls wait up to 30 seconds, then report capacity. + Use a Workflow for large batches. Accepted jobs keep running after the caller ends. Linux reads `MemAvailable` (with a labelled Node >=22 fallback). macOS uses the `kern.memorystatus_level` percentage of total memory, with a labelled `vm_stat` free+inactive @@ -436,8 +434,8 @@ the relay's final text against the job's result instead. The PostToolUse hook ca relay that `TaskStop` stopped. The hooks module (`hooks/mod/register.js`) adds an `agent.spawn` hook, which checks a `relay:job` prompt's header and refuses the spawn with the parser's message when the header is bad, and a `session.send` hook: when a message is sent to a relay, it ends the relay's -current wait, and the guard delivers the message at the relay's next call. The module only checks -headers and wakes a waiting relay: delivery runs on command hooks and the transcript. Its +current wait, and the guard delivers the message at the relay's next call. The module checks +headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript. Its `command.run` hooks answer `/relay:status`, `/relay:result`, `/relay:cancel` and `/relay:transfer`, and its status line counts the repository's running and dead jobs and owed messages ([design.md](docs/design.md#the-status-line)). diff --git a/docs/guide.md b/docs/guide.md index 672a8026..e9ad8788 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -165,14 +165,16 @@ A Stop during the launch handoff may prevent dispatch; a later waiter can retry ## When a job waits -A job's block can show `state: queued` and a `queued:` line explaining why it is waiting. This is pending work, -before it starts or between turns. While its relay agent or the companion's `wait` command is waiting, the job -starts by itself when the reason clears. Leave that call running; do not launch the job again. If you started a -job directly without an agent or a `wait` call, ask Claude to wait for that job by its id. `/relay:status` shows -progress but does not start waiting jobs. You can still cancel the job or send it a message by its id. +A job's block can show `state: queued` and a `queued:` line explaining why its live worker is +waiting, before a backend starts or between turns. Its worker starts the work when the reason clears, +even if you close the waiting call. Leave the job running; collect its result by id. -In Claude Code's Workflow panel a relay agent can look idle for minutes: it sits in relay's `wait` -call, which returns within seconds of the job ending. +When relay's start queue is full, Workflow agents wait in the panel as queued, using no model tokens +and creating no job waiter until a place frees. A plain `relay:job` call waits up to 30 seconds, then +reports that relay is at capacity. Use a Workflow for large batches. A direct start that cannot get +its own worker within a minute reports `state: not started` (exit 6); nothing runs later. Submit it +again when capacity is available. A launcher that ends before handing off ownership shows as dead. +A legacy workerless queue is also dead; `wait --auto-resume` can recover it explicitly. The load guard shares capacity across your sessions. New backend starts and replacements are spaced at least 5 seconds apart; turns on a running backend are not paced. @@ -193,7 +195,7 @@ On macOS, tracking and cleanup use process groups plus sampled descendants; memo running tools are not paused or limited. A detached child that orphans between samples can escape tracking on systems without scopes. -A queued job still occupies its Claude agent and, in a Workflow, one of that run's agent slots. A Workflow can +After admission, a queued job occupies its Claude agent and one Workflow agent slot. A row held at spawn has no running relay model or waiter. A Workflow can run up to 16 agents at once, fewer with fewer CPUs; `CLAUDE_CODE_WORKFLOW_MAX_CONCURRENT_AGENTS` changes that limit. Additional agents wait. At Claude's usage limit, Agent-tool relays fail; Workflows pause when auto-continue is on. Their backend jobs keep running. Ask Claude to collect those jobs or resume the paused @@ -328,7 +330,7 @@ counts them: ⚠ relay: 2 running · 1 queued (memory low) · 1 dead · 3 owed ``` -- `queued` counts pending jobs and shows a queue reason. A job waiting to start is queued, even before its +- `queued` counts pending jobs with a live worker and shows a queue reason. A job waiting to start is queued while its background process starts. - `running` counts the jobs that are starting or running. - `dead` counts jobs whose worker died or never came up; a message or the companion's `resume` continues a dead job diff --git a/package-lock.json b/package-lock.json index a54569b0..7e4a3cf2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", diff --git a/package.json b/package.json index b14d6920..08375913 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@libar-dev/relay-plugin-cc", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "private": true, "type": "module", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", diff --git a/plugins/relay/.claude-plugin/plugin.json b/plugins/relay/.claude-plugin/plugin.json index 9f9a6327..5d7c02c2 100644 --- a/plugins/relay/.claude-plugin/plugin.json +++ b/plugins/relay/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "relay", - "version": "3.0.0-dev.24", + "version": "3.0.0-dev.25", "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.", "author": { "name": "Libar" diff --git a/plugins/relay/CHANGELOG.md b/plugins/relay/CHANGELOG.md index 5d3edc40..b87ff089 100644 --- a/plugins/relay/CHANGELOG.md +++ b/plugins/relay/CHANGELOG.md @@ -8,6 +8,9 @@ its entries here. Changed: +- Job acceptance requires a live worker lease within one minute; refused starts exit 6 and never run later. + Workflow relays wait at spawn for a waiting place, including stall retries at their first step; + plain relay calls report capacity after 30 seconds. Large batches should use Workflows. - Foreground `task` workers run independently of the caller; Ctrl-C leaves a dispatched job running. - Jobs own their execution groups and stop their tools at backend loss, cancellation and completion. Linux workers use per-job scopes whenever a user manager is available, and set OOM score 500. @@ -21,6 +24,15 @@ Changed: Fixed: +- Recovery preserves a cancellation or successor worker that wins during the dispatch handoff. +- Delivered corrections retain their automatic recovery allowance if the worker exits during acknowledgement. +- Unreadable message storage fails explicitly and preserves owed work; it cannot be mistaken for an empty inbox at completion. +- Continuations and recovery use the same bounded launch path. Undelivered pending messages are + withdrawn rather than acknowledged without a worker, and interrupted recovery keeps its message id. +- Repeated launch failures stop after three attempts and show the error and count. Handoff retries + preserve owed continuations, and messages racing worker exit retry without a five-second delay. +- Nested workers publish their identity independently of admission storage, including with the guard off. + Admission retains time per wait reason and reports the longest hold after admission. - Repeated cleanup preserves a predecessor's first observed death, keeping the recorded ordering of predecessor exit and replacement startup intact. - Worker heartbeats keep process accounting active with the load guard off without creating diff --git a/plugins/relay/agents/job.md b/plugins/relay/agents/job.md index 6a5d7658..17e6e95e 100644 --- a/plugins/relay/agents/job.md +++ b/plugins/relay/agents/job.md @@ -23,7 +23,7 @@ Run with Bash: node "${CLAUDE_PLUGIN_ROOT}/scripts/companion.mjs" start --from-relay ``` -If the output has a line `job: `, remember the id and go to 2. Otherwise the start failed: hand back the start output at once, every line, nothing before it and nothing after it, and run nothing else, unless a hook's reason tells you to. +If the output starts with `state: not started`, hand back that output at once, every line, and run nothing else. Otherwise, if the output has a line `job: `, remember the id and go to 2. Otherwise the start failed: hand back the start output at once, every line, nothing before it and nothing after it, and run nothing else, unless a hook's reason tells you to. ## 2. Wait diff --git a/plugins/relay/hooks/mod/register.js b/plugins/relay/hooks/mod/register.js index 4800417a..9edfffc8 100644 --- a/plugins/relay/hooks/mod/register.js +++ b/plugins/relay/hooks/mod/register.js @@ -1040,6 +1040,30 @@ async function drawJob($, { Box, Text, Button, Input, Markdown }, job, board) { return h(Box, { flexDirection: "column", marginTop: 1 }, ...nodes); } +// Host process waits do not spend the hook's ten-second JavaScript budget. +// Cancellation must reach the registered catch: a swallowed abort could spawn work +// after the user stopped it. Ordinary gate failures still fall back to start's bound. +async function waitForPlace($, workflow, agentId, transcript) { + for (;;) { + const argv = ["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--wait-seconds", workflow ? "540" : "30"]; + if (agentId) argv.push("--agent", agentId, "--retry"); + if (transcript) argv.push("--transcript", transcript); + let result; + try { result = await $.process.run(argv, { timeoutMs: workflow ? 600000 : 40000 }); } + catch (error) { + if (["abort", "cancelled", "timeout"].includes(error?.kind) || error?.name === "AbortError" || /abort|cancel|timeout/i.test(String(error?.message ?? error))) throw error; + return {}; + } + if (result.exitCode !== 0) return {}; + const output = String(result.stdout ?? "").trim(); + const pass = /^pass(?: ([a-f0-9-]{36}))?$/.exec(output); + if (pass) return { reservation: pass[1] ?? null }; + const hold = /^hold (.+)$/.exec(output); + if (!hold) return {}; + if (!workflow) return { deny: "relay is at capacity: " + hold[1] + ". Try again in a minute, or start these jobs from a Workflow, which waits for a place." }; + } +} + export function register(on) { // One timer and one read in flight per load of the module. `fastUntil` is the last tick of the // every-tick reads a relay event (a relay:job spawn, a wake, one of the commands below) asks for. @@ -1089,26 +1113,59 @@ export function register(on) { return next(e); }); + // In-memory identities only avoid duplicate hook work; all places and their + // expiry live under the coordinator's admission lock, never in mod counters. + const spawned = new Set(); + const relayAgents = new Map(); + const binding = new Map(); on("agent.spawn", async ($, e, next) => { - if (e?.subagentType === "relay:job") { - poller.fastUntil = poller.ticks + CLEARED_TICKS; - } - if (!isCheckedSpawn(e)) { - return next(e); - } - let refusal = null; - try { - refusal = await headerRefusal($, e); - } catch { - // A check that could not run says nothing about the header; the relay's start checks it. - refusal = null; + if (e?.subagentType !== "relay:job") return next(e); + poller.fastUntil = poller.ticks + CLEARED_TICKS; + if (e.resume != null) { if (e.resume.agentId) spawned.add(e.resume.agentId); return next(e); } + if (isCheckedSpawn(e)) { + let refusal = null; + try { refusal = await headerRefusal($, e); } catch {} + if (refusal !== null) return { deny: refusal }; } - if (refusal !== null) { - // Claude Code puts its own "spawn denied by a plugin" before the reason. - return { deny: refusal }; + const gate = await waitForPlace($, e.workflow != null); + if (gate.deny) return gate; + const result = await next(e); + if (result?.agentId) { + spawned.add(result.agentId); + if (gate.reservation) { + try { + const pending = $.process.run(["node", $.plugin.root + "/scripts/companion.mjs", "gate", "--bind", gate.reservation, "--agent", result.agentId], { timeoutMs: 10000 }); + binding.set(result.agentId, pending); + await pending; + } catch { + // The agent is already running; start's bounded admission remains authoritative. + } finally { binding.delete(result.agentId); } + } } + return result; + }).catch(($, e, next) => { + if (next.called) return next(e); + return { deny: "relay admission was interrupted. Try again, or resume the Workflow." }; + }); + + // Workflow stall retries bypass agent.spawn. Track each actual attempt and + // hold its first model step using the same host-process gate. + on("classic.SubagentStart", async ($, e, next) => { + if (e.agent_type === "relay:job") relayAgents.set(e.agent_id, e.transcript_path); return next(e); }); + on("turn.step", async function* ($, e, next) { + if (e.index !== 0 || !relayAgents.has(e.agentId)) return yield* next(e); + if (binding.has(e.agentId)) { try { await binding.get(e.agentId); } catch {} } + if (!spawned.delete(e.agentId)) await waitForPlace($, true, e.agentId, relayAgents.get(e.agentId)); + return yield* next(e); + }).catch(async function* ($, e, next) { + // The engine reuses an in-flight next result here; it does not run the model twice. + if (next.called) return yield* next(e); + const answer = "state: not started\nerror: relay admission was interrupted; resume the Workflow."; + yield { kind: "text", index: 0, text: answer }; + return { turnId: e.turnId, index: e.index, answer, toolUses: [], stopReason: "end_turn", usage: null }; + }); on("session.send", async ($, e, next) => { const result = await next(e); @@ -1161,6 +1218,7 @@ export function register(on) { // after it has returned, and a slow or stuck listing never holds Claude Code's stop. on("turn.complete", async ($, e, next) => { const completed = await next(e); + if (e.agentId != null) { relayAgents.delete(e.agentId); spawned.delete(e.agentId); binding.delete(e.agentId); } if (e.agentId != null && e.reason === "aborted") { try { $.clock.after(0, () => { diff --git a/plugins/relay/hooks/mod/register.test.ts b/plugins/relay/hooks/mod/register.test.ts index e867751e..4ec34d8c 100644 --- a/plugins/relay/hooks/mod/register.test.ts +++ b/plugins/relay/hooks/mod/register.test.ts @@ -165,6 +165,7 @@ function spawnEngine(on: any, check: () => unknown = () => ran(0, 'ok\n')) { const runs: any[] = [] const spawns: unknown[] = [] on('process.run', (_: unknown, e: any) => { + if (e.argv[2] === 'gate') return ran(0, 'pass\n') runs.push(e) return check() }) @@ -2891,3 +2892,68 @@ test('the board shows a closed wait with a fixed duration after admission', asyn expect(texts).not.toContain('queued:') await ui.unmount() }) + +function capacityEngine(on: any, answers: string[]) { + const gates: any[] = [], spawns: any[] = [] + on('process.run', (_: unknown, e: any) => { + if (e.argv[2] === 'check-header') return ran(0, 'ok\n') + gates.push(e) + if (e.argv.includes('--bind')) return ran(0, 'bound\n') + return ran(0, answers.shift() ?? 'pass\n') + }) + on('agent.spawn', (_: unknown, e: unknown) => { spawns.push(e); return STARTED }) + return { gates, spawns } +} + +test('Workflow admission waits on hold then binds the waiting place to the actual agent', async ($, on) => { + const id = '00000000-0000-4000-8000-000000000001' + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n', `pass ${id}\n`]) + expect(await $.agent.spawn(relay(GOOD, { workflow: { runId: 'wf-capacity', agentIndex: 1 } }) as any)).toEqual(STARTED) + expect(seen.spawns).toHaveLength(1) + expect(seen.gates).toHaveLength(3) + expect(seen.gates[0].argv.slice(2)).toEqual(['gate', '--wait-seconds', '540']) + expect(seen.gates[0].init).toEqual({ timeoutMs: 600000 }) + expect(seen.gates[2].argv.slice(2)).toEqual(['gate', '--bind', id, '--agent', 'a-job']) +}) + +test('a plain relay refuses after its thirty-second gate with the capacity explanation', async ($, on) => { + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n']) + expect(await $.agent.spawn(relay(GOOD))).toEqual({ deny: 'relay is at capacity: waiting worker limit: 2 of 2. Try again in a minute, or start these jobs from a Workflow, which waits for a place.' }) + expect(seen.spawns).toHaveLength(0) + expect(seen.gates[0].argv.slice(2)).toEqual(['gate', '--wait-seconds', '30']) +}) + +test('an invalid gate answer falls back to the bounded start', async ($, on) => { + const seen = capacityEngine(on, ['not a verdict\n']) + expect(await $.agent.spawn(relay(GOOD))).toEqual(STARTED) + expect(seen.spawns).toHaveLength(1) +}) + +test('a resumed relay bypasses capacity admission', async ($, on) => { + const seen = capacityEngine(on, ['hold no place\n']) + expect(await $.agent.spawn(relay('Continue.', { resume: { agentId: 'a-job' } }) as any)).toEqual(STARTED) + expect(seen.gates).toHaveLength(0) +}) + +test('a resumed relay first step makes no capacity call even with no existing job', async ($, on) => { + const seen = capacityEngine(on, ['hold waiting worker limit: 2 of 2\n']) + on('classic.SubagentStart', () => ({})) + on('turn.step', async function* (_: any, e: any) { + yield { kind: 'text', index: 0, text: 'ok' } + return { turnId: e.turnId, index: e.index, answer: 'ok', toolUses: [], stopReason: 'end_turn', usage: null } + }) + await $.agent.spawn(relay('Continue.', { resume: { agentId: 'a-job' } }) as any) + await $.classic.SubagentStart({ agent_type: 'relay:job', agent_id: 'a-job' } as any) + const stream = $.turn.step({ turnId: 't-resumed', index: 0, model: 'claude-test', messageCount: 1, agentId: 'a-job' } as any) + let step = await stream.next(); while (step.done !== true) step = await stream.next() + expect(step.value.answer).toBe('ok') + expect(seen.gates).toHaveLength(0) +}) + +test('a binding failure after spawn keeps the admitted agent result', async ($, on) => { + let spawned = 0 + on('process.run', (_: unknown, e: any) => e.argv.includes('--bind') ? { deny: 'synthetic bind failure' } : ran(0, e.argv[2] === 'check-header' ? 'ok\n' : 'pass 00000000-0000-4000-8000-000000000001\n')) + on('agent.spawn', () => { spawned++; return STARTED }) + expect(await $.agent.spawn(relay(GOOD))).toEqual(STARTED) + expect(spawned).toBe(1) +}) diff --git a/plugins/relay/scripts/cli/cancel.mjs b/plugins/relay/scripts/cli/cancel.mjs index f5c74e3e..afd40bd5 100644 --- a/plugins/relay/scripts/cli/cancel.mjs +++ b/plugins/relay/scripts/cli/cancel.mjs @@ -59,6 +59,10 @@ function markCancelledAfterKill(jobDir, reason, attempt = null, killed = false, const pause = () => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); const deadline = Date.now() + 2000; while (!acquireWorkerLease(jobDir, me, "cancel")) { + const current = readJob(jobDir); + // A finished worker may still be closing its backend. Its job needs one + // no-op receipt, not a delegated cancel plus a second receipt. + if (attempt === null && TERMINAL_STATES.has(current.state)) return { ended: current.state }; const holder = readLease(jobDir); if (!isWorkerLease(holder) && Date.now() < deadline) { pause(); diff --git a/plugins/relay/scripts/cli/gate.mjs b/plugins/relay/scripts/cli/gate.mjs new file mode 100644 index 00000000..d8d2f3d8 --- /dev/null +++ b/plugins/relay/scripts/cli/gate.mjs @@ -0,0 +1,53 @@ +import process from "node:process"; +import fs from "node:fs"; +import path from "node:path"; +import { parseCommandInput, sleep } from "./shared.mjs"; +import { bindReservation, readReservation, reserveWaitingPlace } from "../lib/worker/admission.mjs"; +import { findAdoptableRelayJob, findRelayJob, promptSha256, readJournalKey, readRelayTranscript, transcriptCandidates } from "../lib/harness/relay.mjs"; + +// A stalled Workflow attempt gets a fresh agent id without agent.spawn. At its +// first step, reuse an existing job or reservation before asking for another place. +function retryAlreadyCovered(agentId, transcriptPath, env) { + let workflow = false, plain = false; + if (findRelayJob(env, agentId)) return true; + for (const file of transcriptCandidates({ agent_id: agentId, transcript_path: transcriptPath })) { + if (!fs.existsSync(file)) continue; + const run = /[\\/]workflows[\\/]([^\\/]+)[\\/]agent-[^\\/]+\.jsonl$/.exec(file)?.[1]; + if (!run) { plain = true; continue; } + workflow = true; + const transcript = readRelayTranscript(file); + let meta; + try { meta = JSON.parse(fs.readFileSync(file.replace(/\.jsonl$/, ".meta.json"), "utf8")); } catch {} + if (findAdoptableRelayJob(env, { agentId, workflowRunId: run, journalKey: readJournalKey(file, agentId), + promptSha256: promptSha256(transcript.prompt?.text), label: meta?.description })) return true; + } + // Plain resumes are never capacity-gated, including a start previously refused. + return plain && !workflow; +} + +export async function handleGate(argv) { + const { options } = parseCommandInput(argv, { valueOptions: ["wait-seconds", "bind", "agent", "transcript"], booleanOptions: ["retry"] }); + if (options.bind) { + if (!options.agent) throw new Error("gate --bind needs --agent"); + process.stdout.write(bindReservation(options.bind, options.agent) ? "bound\n" : "expired\n"); + return; + } + if (options.agent) { + const reservation = readReservation(options.agent); + if (reservation) { process.stdout.write(`pass ${reservation}\n`); return; } + if (options.retry && retryAlreadyCovered(options.agent, options.transcript, process.env)) { process.stdout.write("pass\n"); return; } + } + const seconds = Number(options["wait-seconds"] ?? 0); + if (!Number.isFinite(seconds) || seconds < 0) throw new Error("--wait-seconds must be non-negative"); + const deadline = Date.now() + seconds * 1000; + for (;;) { + const result = reserveWaitingPlace(); + if (result.error) { process.stderr.write(`${result.reason}\n`); process.exitCode = 1; return; } + if (result.pass) { + if (result.reservation && options.agent) bindReservation(result.reservation, options.agent); + process.stdout.write(`pass${result.reservation ? ` ${result.reservation}` : ""}\n`); return; + } + if (Date.now() >= deadline) { process.stdout.write(`hold ${result.reason}\n`); return; } + await sleep(Math.min(2000 + Math.floor(Math.random() * 1000), Math.max(1, deadline - Date.now()))); + } +} diff --git a/plugins/relay/scripts/cli/message.mjs b/plugins/relay/scripts/cli/message.mjs index 487323d9..f5af486d 100644 --- a/plugins/relay/scripts/cli/message.mjs +++ b/plugins/relay/scripts/cli/message.mjs @@ -21,6 +21,7 @@ function renderDelivery(job, kind, delivery, state) { `message: ${kind} ${delivery.message.id}`, `outcome: ${delivery.outcome?.action ?? "pending (no worker has taken it yet)"}` ]; + if (delivery.outcome?.action === "not-started") lines.push(`not delivered: ${delivery.outcome.reason}; send it again later`); if (delivery.outcome?.turnId) { lines.push(`turn: ${delivery.outcome.turnId}`); } @@ -53,7 +54,8 @@ export async function handleSteerOrContinue(kind, argv) { const state = stateAfterDelivery(jobDir); if (options.json) printJson(kind, { job: job.id, ...delivery, state }); else process.stdout.write(renderDelivery(job, kind, delivery, state)); - if (!delivery.outcome) { + if (delivery.outcome?.action === "not-started") process.exitCode = EXIT.notStarted; + else if (!delivery.outcome) { process.exitCode = EXIT.running; } } @@ -76,7 +78,8 @@ export async function handleResume(argv) { const state = stateAfterDelivery(jobDir); if (options.json) printJson("resume", { job: job.id, ...delivery, state }); else process.stdout.write(renderDelivery(job, "resume", delivery, state)); - if (!delivery.outcome) { + if (delivery.outcome?.action === "not-started") process.exitCode = EXIT.notStarted; + else if (!delivery.outcome) { process.exitCode = EXIT.running; } } diff --git a/plugins/relay/scripts/cli/start.mjs b/plugins/relay/scripts/cli/start.mjs index 139a8ba8..c8f6a0b1 100644 --- a/plugins/relay/scripts/cli/start.mjs +++ b/plugins/relay/scripts/cli/start.mjs @@ -3,10 +3,12 @@ import { readInputFile, createTaskJob, parseCommandInput, printJson } from "./sh import { default as fs } from "node:fs"; import { default as path } from "node:path"; import { default as process } from "node:process"; -import { spawnWorker } from "../lib/worker/delivery.mjs"; +import { startWithinWindow } from "../lib/worker/delivery.mjs"; import { appendRelayViolation } from "../lib/jobs/messages.mjs"; import { nowIso, readJob } from "../lib/jobs/store.mjs"; import { buildLaunchRequest, readLaunchPrompt, UsageError } from "../lib/jobs/request.mjs"; +import { readReservation, releaseReservation } from "../lib/worker/admission.mjs"; +import { resumeDeadJobOnce } from "../lib/worker/recovery.mjs"; import { SCHEMA_SENTENCE } from "../lib/harness/workflow.mjs"; import { backendJson, backendLines, backendSettingLines, EXIT, loadJobViewSettled as loadJobView, relayAdoptionLines } from "../lib/jobs/view.mjs"; import { adoptRelayJob, deliverRelayMessages, findAdoptableRelayJob, promptSha256, readJournalKey, readRelayPointer, readRelayTranscript, writeRelayPointer } from "../lib/harness/relay.mjs"; @@ -154,7 +156,11 @@ async function handleStartFromRelay(options) { for (const problem of transcript.problems) { appendRelayViolation(adoptable, { kind: "transcript", reason: `messages could not be checked: ${problem}`, agentId }); } - await deliverRelayMessages(adoptable, transcript.messages, { wait: false }); + const reservation = readReservation(agentId); + try { + if (loadJobView(adoptable).state === "dead") await resumeDeadJobOnce(adoptable); + await deliverRelayMessages(adoptable, transcript.messages, { wait: false, reservation }); + } finally { if (reservation) releaseReservation(reservation); } const adopted = readJob(adoptable); if (options.json) printJson("start", { jobId: adopted.id, state: loadJobView(adoptable).state, jobDir: adoptable, notes: adopted.request?.notes ?? [], ...backendJson(adopted.request) }); else process.stdout.write(adoptedBlock(adopted, adoptable)); @@ -165,12 +171,15 @@ async function handleStartFromRelay(options) { }); let job = readJob(jobDir); writeRelayPointer(process.env, { agentId, transcriptPath, jobId: job.id, jobDir, startError: null }); - spawnWorker(jobDir, request.cwd); + await startWithinWindow(jobDir, request.cwd, process.env, { reservation: agentId ? readReservation(agentId) : null }); job = readJob(jobDir); for (const problem of transcript.problems) { appendRelayViolation(jobDir, { kind: "transcript", reason: `messages could not be checked: ${problem}`, agentId }); } - await deliverRelayMessages(jobDir, transcript.messages, { wait: false }); + if (job.admission?.state !== "not-started") await deliverRelayMessages(jobDir, transcript.messages, { wait: false }); + if (job.admission?.state === "not-started") { + fail(job.errorMessage, "not-started"); process.exitCode = EXIT.notStarted; return; + } if (options.json) printJson("start", { jobId: job.id, state: job.state, jobDir, notes: request.notes, ...backendJson(request) }); else process.stdout.write(startBlock(job, jobDir, request, job.state)); } @@ -184,8 +193,14 @@ export async function handleStart(argv) { } const { request, brief } = readLaunchFromOptions(options); const jobDir = createTaskJob(request, brief); - spawnWorker(jobDir, request.cwd); + await startWithinWindow(jobDir, request.cwd); const job = readJob(jobDir); + if (job.admission?.state === "not-started") { + process.exitCode = EXIT.notStarted; + if (options.json) printJson("start", { jobId: job.id, state: "not started", jobDir, error: { kind: "not-started", message: job.errorMessage } }); + else process.stdout.write(`state: not started\nerror: ${job.errorMessage}\n`); + return; + } if (options.json) { printJson("start", { jobId: job.id, state: job.state, jobDir, notes: request.notes, ...backendJson(request) }); return; diff --git a/plugins/relay/scripts/cli/task.mjs b/plugins/relay/scripts/cli/task.mjs index 9fc5e5c3..fe44ce18 100644 --- a/plugins/relay/scripts/cli/task.mjs +++ b/plugins/relay/scripts/cli/task.mjs @@ -4,10 +4,10 @@ import { default as fs } from "node:fs"; import { default as path } from "node:path"; import { default as process } from "node:process"; import { readStdinIfPiped } from "../lib/util/fs.mjs"; -import { spawnWorker, retryQueuedWorker, hasLiveWorker } from "../lib/worker/delivery.mjs"; +import { startWithinWindow, hasLiveWorker } from "../lib/worker/delivery.mjs"; import { requireJobDirInHome, jobPaths, readJob } from "../lib/jobs/store.mjs"; import { buildLaunchRequest, UsageError } from "../lib/jobs/request.mjs"; -import { loadJobViewSettled as loadJobView, renderJobWithReport, isTerminalView } from "../lib/jobs/view.mjs"; +import { EXIT, loadJobViewSettled as loadJobView, renderJobWithReport, isTerminalView } from "../lib/jobs/view.mjs"; import { runWorker } from "../lib/worker/worker.mjs"; const STOP_REVIEW_TASK_MARKER = "Run a stop-gate review of the previous Claude turn."; @@ -49,16 +49,20 @@ export async function handleTask(argv) { prefix: "task", summary: prompt.includes(STOP_REVIEW_TASK_MARKER) ? `${backendWith("stopGate").label} Stop Gate Review` : null }); + await startWithinWindow(jobDir, request.cwd); + const started = readJob(jobDir); + if (started.admission?.state === "not-started") { + if (options.json) printJson("task", { jobId: started.id, state: "not started", jobDir, errorMessage: started.errorMessage }); + else process.stdout.write(`state: not started\nerror: ${started.errorMessage}\n`); + process.exitCode = EXIT.notStarted; return; + } if (options.background) { - spawnWorker(jobDir, request.cwd); const job = readJob(jobDir); if (options.json) printJson("task", { jobId: job.id, state: job.state, jobDir, backend: backendFor(request).name }); else process.stdout.write(`job: ${job.id}\nstate: ${job.state}\ndir: ${jobDir}\n`); return; } - spawnWorker(jobDir, request.cwd); for (;;) { - retryQueuedWorker(jobDir); const current = loadJobView(jobDir); if (isTerminalView(current) && !hasLiveWorker(jobDir)) break; await sleep(50); diff --git a/plugins/relay/scripts/companion.mjs b/plugins/relay/scripts/companion.mjs index 7d7f24be..a2e5a166 100644 --- a/plugins/relay/scripts/companion.mjs +++ b/plugins/relay/scripts/companion.mjs @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { handleGate } from "./cli/gate.mjs"; import { handleStart } from "./cli/start.mjs"; import { handleCheckHeader } from "./cli/check-header.mjs"; import { handleWait } from "./cli/wait.mjs"; @@ -70,6 +71,9 @@ async function main() { } switch (subcommand) { + case "gate": + await handleGate(argv); + break; case "check-header": handleCheckHeader(argv); break; diff --git a/plugins/relay/scripts/hooks/stop-review-gate.mjs b/plugins/relay/scripts/hooks/stop-review-gate.mjs index 540a7622..a4f760a4 100644 --- a/plugins/relay/scripts/hooks/stop-review-gate.mjs +++ b/plugins/relay/scripts/hooks/stop-review-gate.mjs @@ -17,7 +17,7 @@ import { readRepoConfig } from "../lib/jobs/config.mjs"; const STOP_REVIEW_TIMEOUT_MS = 10 * 60 * 1000; const STOP_PROBE_TIMEOUT_MS = 10 * 1000; // Starting the review job, the margin of its wait, and the cancel after a timeout. -export const STOP_COMMAND_TIMEOUT_MS = 60 * 1000; +export const STOP_COMMAND_TIMEOUT_MS = 90 * 1000; const REVIEW_BACKEND = backendWith("stopGate"); const MANUAL_REVIEW = "Run /relay:review --wait to review by hand."; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); diff --git a/plugins/relay/scripts/lib/harness/relay.mjs b/plugins/relay/scripts/lib/harness/relay.mjs index 725d3f06..f387ede8 100644 --- a/plugins/relay/scripts/lib/harness/relay.mjs +++ b/plugins/relay/scripts/lib/harness/relay.mjs @@ -21,7 +21,7 @@ import path from "node:path"; import process from "node:process"; import { awaitOutcome, hasLiveWorker, COMPANION_SCRIPT } from "../worker/delivery.mjs"; -import { appendRelayViolation, messageExists, messageAccount, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; +import { InboxStorageError, appendRelayViolation, messageExists, messageAccount, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; import { absoluteJobsHomeEnv, listJobDirs, nowIso, readJob, jobPaths, readJsonOrNull, resolveJobsHome, TERMINAL_STATES, writeJsonAtomic } from "../jobs/store.mjs"; import { isTerminalView, JobViewBusy, loadJobView } from "../jobs/view.mjs"; import { acquireOwnership, releaseOwnership } from "../util/ownership.mjs"; @@ -526,6 +526,11 @@ export function findAdoptableRelayJob(env, { agentId, workflowRunId, journalKey, try { state = loadJobView(dir).state; } catch (error) { + // Uncertain message storage cannot turn an existing call into a new job. + if (error instanceof InboxStorageError) { + appendRelayViolation(dir, { kind: "delivery", reason: `messages could not be delivered: ${error.message}` }); + throw error; + } // A busy job is at work: it stays a candidate. A directory without job.json does not. if (!(error instanceof JobViewBusy)) continue; state = null; @@ -685,6 +690,7 @@ export async function deliverRelayMessages(jobDir, messages, options = {}) { async function deliverRelayMessagesUnderLock(jobDir, messages, options) { const env = options.env ?? process.env; const wait = options.wait ?? true; + const reservation = options.reservation ?? null; if (undeliveredMessages(jobDir, messages).length === 0) { return { busy: false, delivered: [], undelivered: [] }; } @@ -761,7 +767,7 @@ async function deliverRelayMessagesUnderLock(jobDir, messages, options) { postInboxMessage(jobDir, { id: record.messageId, kind: record.kind, text: message.text, relayUuid: message.uuid }); } const ackTimeoutMs = envNumber(env, "RELAY_ACK_TIMEOUT_MS", 20000); - const { outcome, consumer } = await awaitOutcome(jobDir, record.messageId, { env, timeoutMs: ackTimeoutMs, wait }); + const { outcome, consumer } = await awaitOutcome(jobDir, record.messageId, { env, timeoutMs: ackTimeoutMs, wait, reservation }); if (!consumer) { appendRelayViolation(jobDir, { kind: "delivery", reason: `message ${message.uuid} acknowledgement timed out; will retry with message id ${record.messageId}` }); break; diff --git a/plugins/relay/scripts/lib/jobs/lease.mjs b/plugins/relay/scripts/lib/jobs/lease.mjs index 8321fb65..0652d67b 100644 --- a/plugins/relay/scripts/lib/jobs/lease.mjs +++ b/plugins/relay/scripts/lib/jobs/lease.mjs @@ -1,5 +1,6 @@ -import { jobPaths } from "./store.mjs"; +import { jobPaths, readJob } from "./store.mjs"; import { acquireOwnership, readOwnership, releaseOwnership } from "../util/ownership.mjs"; +import { pendingInboxMessages, owedMessages } from "./messages.mjs"; import { isProcessIdentityAlive } from "../util/process.mjs"; @@ -13,7 +14,16 @@ export const isWorkerAlive = isProcessIdentityAlive; // holder that consumes inbox messages, from a cancel or a recovery that takes the lease to // write job.json. `worker` in the release is kept for the callers' symmetry. export function acquireWorkerLease(jobDir, worker, role = "worker") { - return Boolean(acquireOwnership(jobPaths(jobDir).lease, { ...worker, role })); + const file = jobPaths(jobDir).lease; + if (!acquireOwnership(file, { ...worker, role })) return false; + // A dispatch already in flight may arrive after the submitting window closed. + // Under the same fence as refusal, prevent it from reviving an unstarted brief. + // A later explicit message is work the user has asked to submit again. + if (role === "worker" && readJob(jobDir)?.admission?.state === "not-started" && + pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0) { + releaseOwnership(file); return false; + } + return true; } diff --git a/plugins/relay/scripts/lib/jobs/messages.mjs b/plugins/relay/scripts/lib/jobs/messages.mjs index a93d2771..fc5f21d8 100644 --- a/plugins/relay/scripts/lib/jobs/messages.mjs +++ b/plugins/relay/scripts/lib/jobs/messages.mjs @@ -27,18 +27,26 @@ export function postInboxMessage(jobDir, message) { } -export function pendingInboxMessages(jobDir) { - const paths = jobPaths(jobDir); - try { - return fs - .readdirSync(paths.inbox) - .filter((name) => name.endsWith(".json") && !name.startsWith(".")) - .sort(); - } catch { - return []; +export class InboxStorageError extends Error { + constructor(directory, cause) { + super(`Message storage uncertain: cannot list ${directory} (${cause.code ?? cause.message}); completion cannot be confirmed.`, { cause }); + this.name = "InboxStorageError"; } } +function messageNames(directory) { + try { return fs.readdirSync(directory); } + catch (error) { + if (error.code === "ENOENT") return []; + throw new InboxStorageError(directory, error); + } +} + +export function pendingInboxMessages(jobDir) { + return messageNames(jobPaths(jobDir).inbox) + .filter(name => name.endsWith(".json") && !name.startsWith(".")).sort(); +} + export function claimInboxMessages(jobDir) { const paths = jobPaths(jobDir); @@ -175,10 +183,7 @@ const WORK_KINDS = new Set(["steer", "continue", "resume"]); export function readJobMessages(jobDir) { const paths = jobPaths(jobDir); const relays = new Map(); - const namesIn = (dir) => { - try { return fs.readdirSync(dir).filter(name => name.endsWith(".json") && !name.startsWith(".")); } - catch { return []; } - }; + const namesIn = dir => messageNames(dir).filter(name => name.endsWith(".json") && !name.startsWith(".")); const relayDir = path.join(paths.inbox, "relay"); for (const name of namesIn(relayDir).sort()) { const record = readJsonOrNull(path.join(relayDir, name)); @@ -211,12 +216,7 @@ export function readJobMessages(jobDir) { function readClaimedMessages(jobDir) { const paths = jobPaths(jobDir); - let names = []; - try { - names = fs.readdirSync(paths.claimed).filter((name) => name.endsWith(".json")).sort(); - } catch { - return []; - } + const names = messageNames(paths.claimed).filter(name => name.endsWith(".json")).sort(); return names .map((name) => readJsonOrNull(path.join(paths.claimed, name))) .filter(Boolean) @@ -250,3 +250,20 @@ export function messageAccount(jobDir) { .map(({ message, outcome }) => ({ id: message.id, reason: outcome.reason ?? "dropped" })); return { undelivered, dropped }; } + +// Renaming races the worker's claim atomically. A refused message is retained for +// inspection but never appears in pending/owed accounting or a future worker's input. +export function withdrawMessage(jobDir, messageId) { + const paths = jobPaths(jobDir), directory = path.join(jobDir, "withdrawn"); + fs.mkdirSync(directory, { recursive: true }); + try { fs.renameSync(path.join(paths.inbox, `${messageId}.json`), path.join(directory, `${messageId}.json`)); } + catch (error) { if (error.code === "ENOENT") return fs.existsSync(path.join(directory, `${messageId}.json`)); throw error; } + const relay = path.join(paths.inbox, "relay"); + let names; + try { names = fs.readdirSync(relay); } catch (error) { if (error.code !== "ENOENT") throw error; names = []; } + for (const name of names) { + const file = path.join(relay, name); + if (readJsonOrNull(file)?.messageId === messageId) fs.rmSync(file, { force: true }); + } + return true; +} diff --git a/plugins/relay/scripts/lib/jobs/view.mjs b/plugins/relay/scripts/lib/jobs/view.mjs index 8e1921e8..9618790e 100644 --- a/plugins/relay/scripts/lib/jobs/view.mjs +++ b/plugins/relay/scripts/lib/jobs/view.mjs @@ -15,7 +15,8 @@ export const EXIT = { usage: 1, noReport: 3, running: 4, - cancelled: 5 + cancelled: 5, + notStarted: 6 }; // How many times a view reads the job and its parts before it gives up on this call. @@ -127,6 +128,7 @@ export function exitCodeFor(view) { if (view.state === "cancelled") { return EXIT.cancelled; } + if (view.job?.admission?.state === "not-started") return EXIT.notStarted; if (view.undelivered?.length > 0) { return EXIT.noReport; } @@ -254,6 +256,7 @@ export function renderJobBlock(view) { } else if (job.errorMessage && view.state !== "completed") { lines.push(`error: ${String(job.errorMessage).replace(/\s+/g, " ").trim()}`); } + if (job.admission?.launchFailures) lines.push(`worker launch: ${job.admission.launchFailures} failed launch${job.admission.launchFailures === 1 ? "" : "es"} of 3${job.admission.launchError ? ` (${job.admission.launchError})` : ""}`); if (job.launch?.note) lines.push(`note: ${job.launch.note}`); // Cleanup still changes live accounting after a turn ends. A terminal result is copied // verbatim by relay agents, so do not include that changing measurement in its text. @@ -489,7 +492,12 @@ export function computeJobView(job, options = {}) { state = "dead"; deadReason = `worker ${job.worker.pid} is gone`; } - } else if (!(stored === "queued" && job?.admission?.waitKind === "worker")) { + } else if (stored === "queued" && job?.admission?.waitKind === "worker") { + if (!isWorkerAlive(job.admission.launcher)) { + state = "dead"; + deadReason = job.admission.launcher ? "not started: the process starting it ended" : "queued by an older relay with nobody to start it"; + } + } else { const created = Date.parse(job?.launchPendingAt ?? job?.createdAt ?? ""); if (Number.isFinite(created) && now - created > (options.workerStartGraceMs ?? WORKER_START_GRACE_MS)) { state = "dead"; diff --git a/plugins/relay/scripts/lib/util/process.mjs b/plugins/relay/scripts/lib/util/process.mjs index 7174c01b..4bc7fa60 100644 --- a/plugins/relay/scripts/lib/util/process.mjs +++ b/plugins/relay/scripts/lib/util/process.mjs @@ -433,7 +433,10 @@ export function readProcessTableCached({ home = null, maxAgeMs = 5000, now = Dat const cached = readJsonOrNull(file); if (cached && now >= cached.at && now - cached.at < maxAgeMs) return cached.rows; const lock = `${file}.lock`; - if (!acquireOwnership(lock)) return cached?.rows ?? null; + if (!acquireOwnership(lock)) { + if (maxAgeMs > 0 && cached && now >= cached.at && now - cached.at < 2 * maxAgeMs) return cached.rows; + return list(); + } try { const rows = list(); if (rows) writeJsonAtomic(file, { at: now, rows }); @@ -485,16 +488,28 @@ function scopeMemberStarted(row, execution) { return !row.startTime || !Number.isFinite(start) || row.startedAtMs >= Math.floor(start / 1000) * 1000; } +// Independent of admission storage, including when the guard is explicitly off. +export const independentWorkerRegistry = () => path.join("/tmp", `relay-workers-${process.getuid?.() ?? "user"}`); +export function safeWorkerRegistry(directory) { + try { + const stat = fs.lstatSync(directory); + return stat.isDirectory() && !stat.isSymbolicLink() && + (process.getuid == null || stat.uid === process.getuid()) && (stat.mode & 0o077) === 0; + } catch { return false; } +} + + // Published workers form ownership boundaries, including during a nested launch handoff. function foreignWorkerPids(job, table) { const excluded = new Set(); - for (const directory of new Set(executions(job).map(e => e.workerRegistry).filter(Boolean))) { - let names; - try { names = fs.readdirSync(directory); } catch { continue; } - for (const name of names) { - const worker = readJsonOrNull(path.join(directory, name)); - if (worker?.jobDir !== job.jobDir && worker?.startTime && table.some(row => - row.pid === worker.pid && row.startTime === worker.startTime && !row.zombie)) excluded.add(worker.pid); + const independent = independentWorkerRegistry(); + for (const directory of new Set([...(safeWorkerRegistry(independent) ? [independent] : []), ...executions(job).map(e => e.workerRegistry).filter(Boolean)])) { + // Lookup only current process ids. Historical crash records cannot increase + // sampling work, and a newly published nested worker is visible immediately. + for (const row of table) { + const worker = readJsonOrNull(path.join(directory, `${row.pid}.json`)); + if (worker?.jobDir !== job.jobDir && worker?.startTime && + row.startTime === worker.startTime && !row.zombie) excluded.add(row.pid); } } // Never walk through another job's worker to capture its backend or tools either. @@ -551,6 +566,11 @@ export function executionMembers(job, table, { jobDir = job?.jobDir, excludeSelf const pids = new Set(fs.readFileSync(file, "utf8").trim().split(/\s+/).map(Number)); entries.push(...table.filter(row => pids.has(row.pid) && !row.zombie && scopeMemberStarted(row, execution))); + // Scope membership disproves emptiness even when the process table missed + // a concurrently forked pid. Never signal it without a verified identity. + for (const pid of pids) if (pid > 0 && !table.some(row => row.pid === pid)) { + entries.push({ pid, startTime: null, identityUnknown: true }); + } } catch (error) { if (error.code !== "ENOENT") cause = "the execution scope could not be read"; } } const leader = table.find(row => row.pid === execution.leader?.pid && row.startTime === execution.leader.startTime && !row.zombie); @@ -562,7 +582,7 @@ export function executionMembers(job, table, { jobDir = job?.jobDir, excludeSelf const recorded = jobDir ? readJsonOrNull(path.join(jobDir, "processes.json"))?.members ?? [] : []; const knownRoots = verifiedRoots([...roots(job), ...recorded], table, excluded); entries.push(...roots(job), ...recorded, ...snapshotProcessTree(knownRoots.map(root => root.pid), { table, excludePids: excluded })); - const live = liveSnapshotMembers(entries, { table }); + const live = [...liveSnapshotMembers(entries, { table }), ...entries.filter(entry => entry.identityUnknown && !entry.startTime && !table.some(row => row.pid === entry.pid))]; const result = [...new Map(live.filter(entry => !excluded.has(entry.pid) && (!excludeSelf || entry.pid !== process.pid)).map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; if (cause) Object.assign(result, { cause }); return result; @@ -597,12 +617,22 @@ export async function stopExecution(job, options = {}) { } } const stopped = await stopProcessSnapshot(members, options); - // Tools may fork while TERM is being handled. Verify the execution itself again, - // rather than declaring it empty just because the original snapshot is gone. - const remaining = executionMembers(job, (options.listProcesses ?? listProcesses)(), options); + // A member may have forked after the first table read. Stop verified members + // first, then take one fresh discovery pass for survivors. Unknown scope pids + // retain capacity, but never prevent signals to processes we can identify. + let finalTable = (options.listProcesses ?? listProcesses)(); + let remaining = executionMembers(job, finalTable, options); + const discovered = remaining.filter(entry => entry.startTime && !entry.identityUnknown && + !members.some(old => old.pid === entry.pid && old.startTime === entry.startTime)); + if (!remaining.cause && discovered.length) { + await stopProcessSnapshot(discovered, options); + finalTable = (options.listProcesses ?? listProcesses)(); + remaining = executionMembers(job, finalTable, options); + } + const stillStopped = liveSnapshotMembers(stopped, { table: finalTable }); /** @type {ProcessSnapshot} */ - const survivors = [...new Map([...stopped, ...remaining].map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; - if (stopped.cause || remaining.cause) survivors.cause = stopped.cause ?? remaining.cause; + const survivors = [...new Map([...stillStopped, ...remaining].map(entry => [`${entry.pid}:${entry.startTime}`, entry])).values()]; + if (stillStopped.cause || remaining.cause) survivors.cause = stillStopped.cause ?? remaining.cause; for (const server of roots(job)) { if (server.pid !== process.pid && isProcessIdentityAlive(server) && !survivors.some(p => p.pid === server.pid)) survivors.push(server); } diff --git a/plugins/relay/scripts/lib/worker/admission.mjs b/plugins/relay/scripts/lib/worker/admission.mjs index 46c78c51..8673eb45 100644 --- a/plugins/relay/scripts/lib/worker/admission.mjs +++ b/plugins/relay/scripts/lib/worker/admission.mjs @@ -7,7 +7,7 @@ import process from "node:process"; import { randomUUID, createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; import { listJobDirs, readJob, readJsonOrNull, writeJsonAtomic, writeJob, TERMINAL_STATES } from "../jobs/store.mjs"; -import { pendingInboxMessages } from "../jobs/messages.mjs"; +import { pendingInboxMessages, owedMessages } from "../jobs/messages.mjs"; import { acquireOwnership, releaseOwnership, readOwnership } from "../util/ownership.mjs"; import { executionMembers, readProcessTableCached, isProcessAlive, isProcessIdentityAlive, readProcessStartTime } from "../util/process.mjs"; @@ -24,7 +24,7 @@ export const DEFAULTS = Object.freeze({ weights: Object.freeze({ light: 1, medium: 2, "very-heavy": 3 }), heavyModel: /astra/i, kimiHeavyModel: /(^|[^a-z0-9])k3([^a-z0-9]|$)/i, lightEfforts: ["low", "minimal", "none"], lightModel: /luna|flash|highspeed|-fast$/i, - pollMs: 3000, claimantPolls: 4, probeMs: 120000, sensorCacheMs: 2000, discoveryMs: 60000, + reservationMs: 45000, maxLaunchFailures: 3, pollMs: 3000, claimantPolls: 4, probeMs: 120000, sensorCacheMs: 2000, discoveryMs: 60000, sensorStaleMs: 10000, processCacheMs: 60000, spawnGraceMs: 10000, stuckLockMs: 60000, pauseExpiryMs: 6 * 3600000, clockSkewMs: 300000, usagePauseMs: 30 * 60000, ratePauseMs: 60000, maxRatePauseMs: 15 * 60000, @@ -69,7 +69,7 @@ export function guardSettings(config = {}, machine = { totalBytes: os.totalmem() return { maxActive: DEFAULTS.maxActive, busySeats, guaranteedVeryHeavy: DEFAULTS.guaranteedVeryHeavy, guaranteedOther: DEFAULTS.guaranteedOther, headroomBytes: Math.max(DEFAULTS.headroomGiB * GiB, machine.totalBytes * DEFAULTS.headroomFraction), - maxWaitingWorkers: Math.max(DEFAULTS.minWaitingWorkers, DEFAULTS.waitingPerBusySeat * busySeats) }; + maxWaitingWorkers: config.maxWaitingWorkers ?? Math.max(DEFAULTS.minWaitingWorkers, DEFAULTS.waitingPerBusySeat * busySeats) }; } export function validModelClasses(entries) { return Array.isArray(entries) ? entries.filter(e => e && typeof e.pattern === "string" && /^[0-9A-Za-z._:/*-]{1,100}$/.test(e.pattern) && Object.hasOwn(WEIGHTS, e.class)) : []; @@ -317,7 +317,7 @@ function slotFile(home, id) { if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error( function writeSlot(home, slot) { fs.mkdirSync(path.join(home, "slots"), { recursive: true }); writeJsonAtomic(slotFile(home, slot.jobId), slot); } export function executionAlive(job, deps) { if (job?.execution || job?.worker) { - const members = executionMembers(job, readProcessTableCached({ home: admissionHome(deps.env), maxAgeMs: deps.freshExecution ? 0 : 5000 }), { jobDir: job.jobDir }); + const members = executionMembers(job, (deps.listProcesses ? deps.listProcesses() : readProcessTableCached({ home: admissionHome(deps.env), maxAgeMs: deps.freshExecution ? 0 : 5000 })), { jobDir: job.jobDir }); if (members.cause || members.some(p => p.pid !== job.worker?.pid)) return true; } return [job?.server, ...(job?.attempts ?? []).flatMap(a => a.servers ?? [])].some(s => s?.pid && !s.endedAt && deps.isAlive(s)); @@ -341,6 +341,12 @@ function slots(home, deps, clean = true) { if (name.endsWith(".tmp")) { if (clean && deps.now() - fs.statSync(file).mtimeMs > 60000) fs.rmSync(file, { force: true }); continue; } if (!/^[A-Za-z0-9_-]+\.json$/.test(name)) continue; const s = readJsonOrNull(file); + if (s?.v === 1 && s.state === "reserved" && s.jobId === name.slice(0, -5) && + /^r-[a-f0-9-]{36}$/.test(s.jobId) && Number.isFinite(Date.parse(s.expiresAt))) { + if (deps.now() < Date.parse(s.expiresAt) && Date.parse(s.expiresAt) - deps.now() <= DEFAULTS.reservationMs + DEFAULTS.clockSkewMs) result.push(s); + else if (clean) fs.rmSync(file, { force: true }); + continue; + } if (!s || s.v !== 1 || typeof s.jobDir !== "string" || s.jobId !== name.slice(0, -5) || !Object.hasOwn(WEIGHTS, s.class) || !["waiting", "running", "spawning"].includes(s.state) || typeof s.backend !== "string" || !Number.isFinite(Date.parse(s.since)) || (s.state !== "spawning" && (!Number.isInteger(s.pid) || s.pid <= 0)) || @@ -364,7 +370,7 @@ function slots(home, deps, clean = true) { const job = readJob(s.jobDir); s.quotaHeavy = isVeryHeavy({ backend: s.backend, model: s.model, effort: s.effort }) || isVeryHeavy(job?.request); s.weight = WEIGHTS[s.class]; - const backendLive = executionAlive({ ...job, jobDir: s.jobDir }, deps); + const backendLive = !live && executionAlive({ ...job, jobDir: s.jobDir }, { ...deps, freshExecution: clean }); if (live || backendLive) result.push(!live && backendLive ? { ...s, state: "running" } : s); else if (clean) fs.rmSync(file, { force: true }); } @@ -460,8 +466,25 @@ function jobAccount(home, jobId, deps) { } const kindLabel = kind => ({ usage: "usage limit", rate: "rate limit", concurrency: "concurrent request limit" })[kind] ?? kind; function refusalReason(p) { if (p.reason) return p.reason; return `${p.backend}${p.account ? ` (account ${p.account})` : ""} paused: ${kindLabel(p.kind)}${p.code ? ` (${p.code})` : ""} until ${p.until}`; } +function waitHistory(admission, now) { + const totals = {}; + // Reason text includes changing readings. Aggregate by cause, with bounded + // keys, rather than making a new durable entry for every sample. + for (const old of Object.values(admission?.waitTotals ?? {})) { + if (typeof old?.kind !== "string" || !Number.isFinite(old.ms) || old.ms < 0) continue; + totals[old.kind] = { ...old, ms: (totals[old.kind]?.ms ?? 0) + old.ms }; + } + if (admission?.state === "waiting" && admission.reason) { + const from = Date.parse(admission.reasonSince ?? admission.since); + const elapsed = Number.isFinite(from) ? Math.max(0, now - from) : 0; + const kind = admission.causeKind ?? admission.waitKind ?? "unknown"; + totals[kind] = { kind, reason: admission.reason, ms: (totals[kind]?.ms ?? 0) + elapsed }; + } + return totals; +} +function mainWait(totals) { return Object.values(totals).sort((a, b) => b.ms - a.ms)[0]; } function recordFor(slot, notes, reason = null, waitKind = null) { - return { state: reason ? "waiting" : "admitted", reason: reason ?? slot.waitReason ?? null, waitKind: waitKind ?? slot.waitKind ?? null, since: slot.waitingSince ?? null, admittedAt: reason ? null : slot.admittedAt ?? null, class: slot.class, weight: slot.weight, probe: Boolean(slot.probe), notes }; + return { waitTotals: slot.waitTotals ?? {}, reasonSince: slot.reasonSince ?? null, state: reason ? "waiting" : "admitted", reason: reason ?? slot.waitReason ?? null, waitKind: waitKind ?? slot.waitKind ?? null, since: slot.waitingSince ?? null, admittedAt: reason ? null : slot.admittedAt ?? null, class: slot.class, weight: slot.weight, probe: Boolean(slot.probe), notes }; } function bypass(ctx, platform = process.platform) { const notes = [["linux", "darwin"].includes(platform) ? "load guard off" : `load guard bypassed: admission unsupported on ${platform}`]; @@ -543,6 +566,7 @@ export function admit(ctx, injected = {}) { mine.waitingSince ??= iso(now); const old = readJsonOrNull(slotFile(home, ctx.jobId)); const slot = existing?.state === "running" ? { ...existing, checkedAt: iso(now) } : { ...mine, since: old?.since ?? mine.since, spawnAt: old?.spawnAt ?? null, holdAt: old?.holdAt ?? old?.spawnAt ?? null, holdBytes: old?.holdBytes ?? 0, state: "waiting", waitKind }; + slot.waitTotals = waitHistory(ctx.admission, now); slot.reasonSince = iso(now); slot.waitReason = reason; slot.waitKind = waitKind; deps.writeSlot(home, slot); return { admit: false, reason, waitKind, retryMs: Number(deps.env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs * (0.67 + 0.67 * deps.random()), slot, record: recordFor({ ...slot, waitingSince: mine.waitingSince }, [...notes, ...deps.notes], reason, waitKind) }; @@ -605,8 +629,9 @@ export function admit(ctx, injected = {}) { mine.probe = true; mine.probeGeneration = pause.probe.generation; mine.probeAt = pause.probe.at; } const previousWait = ctx.admission?.state === "admitted" ? ctx.admission : null; - const slot = { ...mine, waitingSince: ctx.waitingSince ?? previousWait?.since ?? null, - waitReason: ctx.admission?.reason ?? own?.waitReason ?? null, waitKind: (["worker", "spawned"].includes(ctx.admission?.waitKind) ? ctx.admission.causeKind : ctx.admission?.waitKind) ?? own?.waitKind ?? null, + const waitTotals = waitHistory(ctx.admission, now), longest = mainWait(waitTotals); + const slot = { ...mine, waitTotals, waitingSince: ctx.waitingSince ?? previousWait?.since ?? null, + waitReason: longest?.reason ?? ctx.admission?.reason ?? own?.waitReason ?? null, waitKind: longest?.kind ?? (["worker", "spawned"].includes(ctx.admission?.waitKind) ? ctx.admission.causeKind : ctx.admission?.waitKind) ?? own?.waitKind ?? null, spawnAt: own?.spawnAt ?? null, state: "running", admittedAt: ctx.waitingSince ? iso(now) : previousWait?.admittedAt ?? iso(now), holdAt: retainedTurn ? existing.holdAt : iso(now), holdBytes: retainedTurn ? existing.holdBytes : startupBytes(mine.backend) }; deps.writeSlot(home, slot); @@ -808,10 +833,15 @@ export function renderGuard(snapshot) { // Reserve a waiting-worker place and spawn while holding the admission lock. The // caller owns retries; completing workers never launch another session's jobs. -export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { +export function spawnWithAdmission(jobDir, env, spawn, injected = {}, { reservation = null } = {}) { const deps = dependencies({ ...injected, env }); let spawnAttempted = false; - const launch = () => { spawnAttempted = true; return spawn(); }; + const launch = () => { + const job = readJob(jobDir); + if (!job || (TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0)) return null; + spawnAttempted = true; return spawn(); + }; + const launcher = self(deps); try { const platform = deps.platform ?? process.platform; if (!["linux", "darwin"].includes(platform)) { process.stderr.write(`relay load guard: admission unsupported on ${platform}; worker bypassed\n`); return launch(); } @@ -824,11 +854,15 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { deps.lockSleep(25 + Math.floor(50 * deps.random())); machine = machineReading(home, deps); } - const settings = guardSettings(config, machine), job = readJob(jobDir), candidates = legacyCandidates(deps); + const settings = spawnSettings(config, machine, env), job = readJob(jobDir), candidates = legacyCandidates(deps); const decision = retryLock(home, deps, () => { const job = readJob(jobDir); - if (TERMINAL_STATES.has(job?.state) && pendingInboxMessages(jobDir).length === 0) return null; - const all = slots(home, deps), existing = all.find(s => s.jobId === job.id); + if (TERMINAL_STATES.has(job?.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0) return null; + let all = slots(home, deps); + const reserved = all.find(s => s.state === "reserved" && s.jobId === `r-${reservation}`); + // Conversion is atomic with the spawning slot: nobody can take the place in between. + if (reserved) all = all.filter(s => s !== reserved); + const existing = all.find(s => s.jobId === job.id); if (existing && ((existing.state === "spawning" && recent(deps.now(), existing.since, DEFAULTS.spawnGraceMs)) || deps.isAlive({ pid: existing.pid, startTime: existing.processStart }))) return null; const waiting = all.filter(s => s.state !== "running").length; const cls = modelClass(job.request); @@ -837,21 +871,22 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { const check = memoryCheck(machine, settings, holds(all, job.id, deps.now()), DEFAULTS.workerBytes, guaranteed); const reason = check?.reason ?? (waiting >= settings.maxWaitingWorkers ? `waiting worker limit: ${waiting} of ${settings.maxWaitingWorkers}` : null); if (reason) { - writeJob(jobDir, { ...job, state: check?.permanent ? "failed" : "queued", ...(check?.permanent ? { completedAt: iso(deps.now()), errorMessage: reason } : {}), worker: null, - admission: { state: check?.permanent ? "failed" : "waiting", reason, waitKind: "worker", causeKind: check?.kind ?? "workers", since: job.admission?.state === "waiting" ? job.admission.since : iso(deps.now()), class: cls, weight: WEIGHTS[cls], notes: machine.notes } }); + writeJob(jobDir, { ...job, state: check?.permanent ? "failed" : TERMINAL_STATES.has(job.state) ? job.state : "queued", ...(check?.permanent ? { completedAt: iso(deps.now()), errorMessage: reason } : {}), worker: null, + admission: { ...job.admission, waitTotals: waitHistory(job.admission, deps.now()), reasonSince: iso(deps.now()), launcher, state: check?.permanent ? "failed" : job.admission?.state === "not-started" ? "not-started" : "waiting", reason, waitKind: "worker", causeKind: check?.kind ?? "workers", since: job.admission?.state === "waiting" ? job.admission.since : iso(deps.now()), class: cls, weight: WEIGHTS[cls], notes: machine.notes } }); return null; } const slot = { v: 1, jobId: job.id, jobDir, pid: null, processStart: null, backend: job.request.backend ?? "codex", account: job.request.account ?? null, class: cls, weight: WEIGHTS[cls], state: "spawning", spawnAt: iso(deps.now()), since: iso(deps.now()), holdAt: iso(deps.now()), holdBytes: DEFAULTS.workerBytes, holdGroup: machine.group, scopes: machine.scopes }; + if (reserved) fs.rmSync(slotFile(home, reserved.jobId), { force: true }); deps.writeSlot(home, slot); if (job.state === "queued" && job.admission?.waitKind === "worker") writeJob(jobDir, { ...job, state: "starting", worker: null, launchPendingAt: iso(deps.now()), - admission: { ...job.admission, waitKind: "spawned" } }); + admission: { ...job.admission, launcher, waitKind: "spawned" } }); return slot; }); if (!decision.locked) { const previousLock = job.admission?.waitKind === "worker" && /admission lock (busy|stuck)/.test(job.admission.reason); const lockSince = previousLock ? job.admission.lockSince ?? job.admission.since : iso(deps.now()); - writeJob(jobDir, { ...job, state: "queued", worker: null, admission: { state: "waiting", waitKind: "worker", causeKind: "lock", + writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, launcher, state: "waiting", waitKind: "worker", causeKind: "lock", reason: lockReason(home, job.id, decision.holder, deps, lockSince), lockSince, since: job.admission?.since ?? iso(deps.now()), notes: [] } }); return null; @@ -871,10 +906,80 @@ export function spawnWithAdmission(jobDir, env, spawn, injected = {}) { } catch (e) { if (spawnAttempted) throw e; const job = readJob(jobDir); - if (job && !(TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0)) { + if (job && !(TERMINAL_STATES.has(job.state) && pendingInboxMessages(jobDir).length === 0 && owedMessages(jobDir).length === 0)) { const held = guardError({ jobId: job.id, home: admissionHome(env), waitingSince: job.admission?.since, admission: job.admission }, e, deps); - try { writeJob(jobDir, { ...job, state: "queued", worker: null, admission: { ...held.record, waitKind: "worker", causeKind: "guard-error" } }); } catch {} + try { writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, ...held.record, launcher, waitKind: "worker", causeKind: "guard-error" } }); } catch {} } return null; } } + +// Waiting places live in the same lock domain as workers. An environment override is +// useful for deliberately smaller queues; absent it, the generic CPU formula applies. +function spawnSettings(config, machine, env) { + const value = env.RELAY_MAX_WAITING_WORKERS; + return guardSettings({ ...config, ...(value != null && /^\d+$/.test(value) ? { maxWaitingWorkers: Number(value) } : {}) }, machine); +} +const validReservation = id => typeof id === "string" && /^[a-f0-9-]{36}$/.test(id); +const bindingFile = (env, agentId) => { + if (typeof agentId !== "string" || !/^[A-Za-z0-9_-]+$/.test(agentId)) throw new Error("invalid agent id"); + return path.join(admissionHome(env), "bindings", `${agentId}.json`); +}; +export function readReservation(agentId, env = process.env) { + const binding = readJsonOrNull(bindingFile(env, agentId)); + return binding && Date.parse(binding.expiresAt) > Date.now() && validReservation(binding.reservation) ? binding.reservation : null; +} +export function bindReservation(reservation, agentId, env = process.env, injected = {}) { + if (!validReservation(reservation)) throw new Error("invalid reservation id"); + const deps = dependencies({ ...injected, env }), home = admissionHome(env); + const result = retryLock(home, deps, () => { + const slot = slots(home, deps).find(s => s.jobId === `r-${reservation}` && s.state === "reserved"); + if (!slot || (slot.agentId && slot.agentId !== agentId)) return false; + deps.writeSlot(home, { ...slot, agentId }); + const file = bindingFile(env, agentId); + fs.mkdirSync(path.dirname(file), { recursive: true }); + writeJsonAtomic(file, { reservation, expiresAt: slot.expiresAt }); + return true; + }); + return result.value ?? false; +} +export function reserveWaitingPlace(env = process.env, injected = {}) { + const deps = dependencies({ ...injected, env }); + try { + if (!admissionEnabled(env, undefined, deps.platform)) return { pass: true, reservation: null }; + const home = admissionHome(env), machine = machineReading(home, deps); + const settings = spawnSettings({}, machine, env); + const result = retryLock(home, deps, () => { + const waiting = slots(home, deps).filter(s => s.state !== "running").length; + if (waiting >= settings.maxWaitingWorkers) return { pass: false, reason: `waiting worker limit: ${waiting} of ${settings.maxWaitingWorkers}` }; + const reservation = randomUUID(), now = deps.now(); + deps.writeSlot(home, { v: 1, jobId: `r-${reservation}`, state: "reserved", since: iso(now), expiresAt: iso(now + DEFAULTS.reservationMs) }); + return { pass: true, reservation }; + }); + return result.locked ? result.value : { pass: false, reason: "admission lock busy" }; + } catch (error) { return { pass: false, error: true, reason: `guard unavailable (${error.message})` }; } +} + +// Called only under the job lease after its refusal has become terminal. +export function discardSpawnReservation(jobDir, env = process.env) { + const deps = dependencies({ env }), home = admissionHome(env); + try { + if (!admissionEnabled(env)) return; + retryLock(home, deps, () => { + const job = readJob(jobDir), file = slotFile(home, job.id), slot = readJsonOrNull(file); + if (job.admission?.state === "not-started" && slot?.state === "spawning") fs.rmSync(file, { force: true }); + }); + } catch {} +} + +export function releaseReservation(reservation, env = process.env) { + if (!validReservation(reservation)) return false; + const deps = dependencies({ env }), home = admissionHome(env); + try { return retryLock(home, deps, () => { + const file = slotFile(home, `r-${reservation}`), slot = readJsonOrNull(file); + if (slot?.state !== "reserved") return false; + fs.rmSync(file, { force: true }); + if (slot.agentId) fs.rmSync(bindingFile(env, slot.agentId), { force: true }); + return true; + }).value ?? false; } catch { return false; } +} diff --git a/plugins/relay/scripts/lib/worker/delivery.mjs b/plugins/relay/scripts/lib/worker/delivery.mjs index f472e0b9..49f754ff 100644 --- a/plugins/relay/scripts/lib/worker/delivery.mjs +++ b/plugins/relay/scripts/lib/worker/delivery.mjs @@ -10,9 +10,9 @@ import process from "node:process"; import { fileURLToPath } from "node:url"; import { absoluteJobsHomeEnv, jobPaths, readJob, writeJob, readJsonOrNull, writeJsonAtomic, TERMINAL_STATES } from "../jobs/store.mjs"; -import { isMessagePending, messageAccount, pendingInboxMessages, postInboxMessage, readOutcome, writeOutcome } from "../jobs/messages.mjs"; +import { isMessagePending, messageAccount, pendingInboxMessages, postInboxMessage, readOutcome, withdrawMessage } from "../jobs/messages.mjs"; import { isWorkerLease, readLease, releaseWorkerLease } from "../jobs/lease.mjs"; -import { spawnWithAdmission, DEFAULTS } from "./admission.mjs"; +import { spawnWithAdmission, discardSpawnReservation, DEFAULTS } from "./admission.mjs"; import { acquireOwnership, releaseOwnership } from "../util/ownership.mjs"; import { holderAlive } from "./admission.mjs"; import { readProcessStartTime, isProcessIdentityAlive } from "../util/process.mjs"; @@ -28,30 +28,45 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -export function spawnWorker(jobDir, cwd, env = process.env) { +export function spawnWorker(jobDir, cwd, env = process.env, { reservation = null } = {}) { const lease = jobPaths(jobDir).lease; if (!acquireOwnership(lease, { pid: process.pid, startTime: null, role: "spawn" }, { isAlive: holderAlive })) return null; try { let job = readJob(jobDir); + pruneLaunches(jobDir); const latest = latestLaunch(jobDir); if (latest?.worker && ["pending", "acknowledged"].includes(latest.state) && isProcessIdentityAlive(latest.worker)) return null; - if (job && !TERMINAL_STATES.has(job.state) && launchInterrupted(job, latest) && job.admission?.causeKind !== "launch") { - job = { ...job, state: "queued", worker: null, admission: { ...job.admission, state: "waiting", - waitKind: "worker", causeKind: "launch", reason: "worker launch interrupted; retrying", since: new Date().toISOString() } }; + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && launchInterrupted(job, latest)) { + job = { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, state: "waiting", + waitKind: "worker", causeKind: "launch", launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, reason: "worker launch interrupted; retrying", since: new Date().toISOString() } }; writeJob(jobDir, job); } + const pendingId = pendingInboxMessages(jobDir).at(0) ?? null; + if (pendingId && job?.admission?.launchMessageId !== pendingId && job?.admission?.launchFailures) { + job = { ...job, admission: { ...job.admission, launchFailures: 0, launchError: null, launchMessageId: pendingId } }; + writeJob(jobDir, job); + } + if ((job?.admission?.launchFailures ?? 0) >= DEFAULTS.maxLaunchFailures) return null; const retryAt = Date.parse(job?.admission?.workerRetryAt); const poll = envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs); - if (job?.state === "queued" && job.admission?.waitKind === "worker" && Date.now() >= retryAt && Date.now() - retryAt < poll) return null; - const result = spawnWithAdmission(jobDir, env, () => spawnWorkerProcess(jobDir, cwd, env)); + if (job?.state === "queued" && job.admission?.waitKind === "worker" && Date.now() - retryAt >= 0 && Date.now() - retryAt < poll) return null; + const result = spawnWithAdmission(jobDir, env, () => spawnWorkerProcess(jobDir, cwd, env), {}, { reservation }); const deferred = readJob(jobDir); - if (deferred?.state === "queued" && deferred.admission?.waitKind === "worker") writeJob(jobDir, { ...deferred, launchPendingAt: new Date(Date.now()).toISOString(), admission: { ...deferred.admission, workerRetryAt: new Date(Date.now()).toISOString() } }); + if (deferred?.state === "queued" && deferred.admission?.waitKind === "worker") writeJob(jobDir, { ...deferred, launchPendingAt: new Date(Date.now()).toISOString(), admission: { ...deferred.admission, launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, workerRetryAt: new Date(Date.now()).toISOString() } }); return result; } catch (error) { + // Dispatch acknowledgement is an observation, not worker ownership. An + // observation timeout cannot revoke a worker that may already have started. + const uncertain = latestLaunch(jobDir); + if (hasLiveWorker(jobDir) || isProcessIdentityAlive(uncertain?.worker)) { + process.stderr.write(`relay: worker dispatch observation uncertain (${error.message}); waiting for its owner\n`); + return uncertain?.state === "acknowledged" ? uncertain.worker.pid : null; + } const job = readJob(jobDir); - if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length > 0)) writeJob(jobDir, { ...job, state: "queued", worker: null, + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length > 0)) writeJob(jobDir, { ...job, state: TERMINAL_STATES.has(job.state) ? job.state : "queued", worker: null, admission: { ...job.admission, state: "waiting", waitKind: "worker", causeKind: "launch", - reason: "worker launch interrupted; retrying", launchError: error.message, + reason: `worker launch failed (${(job.admission?.launchFailures ?? 0) + 1} of ${DEFAULTS.maxLaunchFailures}): ${error.message}`, launchError: error.message, + launchFailures: (job.admission?.launchFailures ?? 0) + 1, launchMessageId: pendingInboxMessages(jobDir).at(0) ?? null, since: new Date().toISOString(), workerRetryAt: new Date().toISOString() } }); process.stderr.write(`relay load guard: deferred worker spawn failed (${error.message}); work remains queued for retry\n`); return null; @@ -60,6 +75,7 @@ export function spawnWorker(jobDir, cwd, env = process.env) { } function spawnWorkerProcess(jobDir, cwd, env) { + pruneLaunches(jobDir, 19); const file = path.join(jobDir, `launch-${randomUUID()}.json`); writeJsonAtomic(file, { jobId: path.basename(jobDir), trigger: env.RELAY_WORKER_TRIGGER ?? process.argv[2] ?? "dispatch", caller: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, requestedAt: new Date().toISOString(), state: "pending" }); @@ -82,6 +98,8 @@ function spawnWorkerProcess(jobDir, cwd, env) { if (launch?.state?.startsWith("handoff-")) throw new Error("Worker launch handoff failed; no backend was started."); Atomics.wait(pause, 0, 0, 10); } + const uncertain = readJsonOrNull(file); + if (isProcessIdentityAlive(uncertain?.worker)) return uncertain.state === "acknowledged" ? uncertain.worker.pid : null; throw new Error("Worker launch acknowledgement timed out; no dispatch was confirmed."); } @@ -106,15 +124,6 @@ export function releaseLeaseToConsumer(jobDir, holder, env = process.env, { spaw return true; } -// A worker was spawned for this job a moment ago and has not taken its lease yet. -function workerJustSpawned(job, now = Date.now()) { - if (job?.worker?.pid || ["completed", "failed", "cancelled"].includes(job?.state)) { - return false; - } - const created = Date.parse(job?.createdAt ?? ""); - return Number.isFinite(created) && now - created < 10000; -} - // Whether the job ended after an attempt that ran this message. function attemptEnded(jobDir, messageId) { const job = readJob(jobDir); @@ -125,44 +134,84 @@ function attemptEnded(jobDir, messageId) { // Make sure a worker will take a message already in the inbox, and wait up to timeoutMs // for its outcome. With wait:false, wait only until a live consumer exists. // A timeout leaves consumer:false so a relay keeps the original message pending. +// A submission is acknowledged only after a live worker takes the lease. Refusal +// is fenced by the same lease as dispatch; a later reader cannot resurrect the brief. +export async function startWithinWindow(jobDir, cwd, env = process.env, options = {}) { + const now = options.now ?? Date.now, pause = options.sleep ?? sleep; + const windowMs = options.windowMs ?? envNumber(env, "RELAY_JOBS_ACK_TIMEOUT_MS", 60000); + const poll = Math.max(10, envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs)); + const deadline = now() + windowMs; + let nextSpawn = -Infinity, spawned = 0; + for (;;) { + if (hasLiveWorker(jobDir)) return { accepted: true, spawned }; + const job = readJob(jobDir); + if (!job || (TERMINAL_STATES.has(job.state) && !(options.messageId && messageAccount(jobDir).undelivered.includes(options.messageId)))) return { accepted: false, terminal: true, spawned }; + if (now() >= nextSpawn && now() <= deadline) { + if ((options.spawn ?? spawnWorker)(jobDir, cwd, env, { reservation: options.reservation })) spawned++; + nextSpawn = now() + poll; + if (hasLiveWorker(jobDir)) return { accepted: true, spawned }; + } + const current = readJob(jobDir); + if (TERMINAL_STATES.has(current?.state) && !(options.messageId && messageAccount(jobDir).undelivered.includes(options.messageId))) return { accepted: false, terminal: true, spawned }; + if (now() >= deadline || (current?.admission?.launchFailures ?? 0) >= DEFAULTS.maxLaunchFailures) { + const reason = current?.admission?.reason ?? "no live worker took ownership"; + if (options.refuse === false) return { accepted: false, reason, spawned }; + const lease = jobPaths(jobDir).lease; + if (acquireOwnership(lease, { pid: process.pid, startTime: readProcessStartTime(process.pid), role: "refusal" }, { isAlive: holderAlive })) { + try { + const final = readJob(jobDir); + if (final && !TERMINAL_STATES.has(final.state)) writeJob(jobDir, { ...final, state: "failed", worker: null, + completedAt: new Date().toISOString(), errorMessage: `Not started: ${reason}. Nothing ran; start it again later.`, + admission: { ...final.admission, state: "not-started", reason } }); + discardSpawnReservation(jobDir, env); + } finally { releaseOwnership(lease); } + return { accepted: false, reason, spawned }; + } + // An in-flight dispatch owns the fence. Wait for its live lease or recheck + // the terminal record before refusing; never race its backend start. + } + await pause(Math.max(10, Math.min(50, deadline - now()))); + } +} + export async function awaitOutcome(jobDir, messageId, options = {}) { const env = options.env ?? process.env; const timeoutMs = options.timeoutMs ?? envNumber(env, "RELAY_JOBS_ACK_TIMEOUT_MS", 60000); - const pollMs = envNumber(env, "RELAY_JOBS_POLL_MS", 200); + const pollMs = Math.max(10, envNumber(env, "RELAY_JOBS_POLL_MS", 200)); + const spawnPoll = Math.max(10, envNumber(env, "RELAY_ADMISSION_POLL_MS", DEFAULTS.pollMs)); const deadline = Date.now() + timeoutMs; - let lastSpawnAt = 0; let spawned = 0; + if (!hasLiveWorker(jobDir) && messageAccount(jobDir).undelivered.includes(messageId) && timeoutMs > 0) { + const job = readJob(jobDir); + spawned += (await startWithinWindow(jobDir, job?.request?.cwd ?? process.cwd(), env, + { windowMs: timeoutMs, messageId, refuse: false, reservation: options.reservation })).spawned; + } + let nextSpawn = -Infinity, wasLive = false; for (;;) { - retryQueuedWorker(jobDir, env); const outcome = readOutcome(jobDir, messageId); const live = hasLiveWorker(jobDir); - // A worker that ran this message in an attempt and then ended the job (a failed resume, - // say) leaves it owed for `resume`, which the state block names. Only a worker that died - // before it ended such an attempt gets a successor here; otherwise the attempt count would - // depend on whether this loop read the outcome before or after that worker exited. if (outcome && (live || !messageAccount(jobDir).undelivered.includes(messageId) || attemptEnded(jobDir, messageId))) { return { outcome, spawned, consumer: true }; } - const deferred = readJob(jobDir); - if (!live && deferred?.state === "queued" && deferred.admission?.waitKind === "worker") { - const queued = outcome ?? { action: "queued-next-turn", reason: deferred.admission.reason }; - if (!outcome) writeOutcome(jobDir, messageId, queued); - return { outcome: queued, spawned, consumer: true }; - } - if (options.wait === false && live) { - return { outcome: null, spawned, consumer: true }; - } + if (options.wait === false && live) return { outcome: null, spawned, consumer: true }; if (Date.now() >= deadline) { + if (live) return { outcome: null, spawned, consumer: false }; + const reason = readJob(jobDir)?.admission?.reason ?? "no live worker took the message"; + if (withdrawMessage(jobDir, messageId)) return { outcome: { action: "not-started", reason }, spawned, consumer: false }; + // A claim beat the rename. Re-read its outcome before deciding: the worker + // owns delivery now, and the caller must not invite a duplicate send. + const claimedOutcome = readOutcome(jobDir, messageId); + if (claimedOutcome && (hasLiveWorker(jobDir) || !messageAccount(jobDir).undelivered.includes(messageId) || attemptEnded(jobDir, messageId))) return { outcome: claimedOutcome, spawned, consumer: true }; + if (hasLiveWorker(jobDir)) { await sleep(pollMs); continue; } return { outcome: null, spawned, consumer: false }; } - if ((isMessagePending(jobDir, messageId) || messageAccount(jobDir).undelivered.includes(messageId)) && !live && Date.now() - lastSpawnAt > 5000) { + if (!live && (wasLive || Date.now() >= nextSpawn) && + (isMessagePending(jobDir, messageId) || messageAccount(jobDir).undelivered.includes(messageId))) { const job = readJob(jobDir); - if (job?.admission?.waitKind === "worker" || !workerJustSpawned(job)) { - const pid = spawnWorker(jobDir, job?.request?.cwd ?? process.cwd(), env); - lastSpawnAt = Date.now(); - if (pid) spawned += 1; - } + if (spawnWorker(jobDir, job?.request?.cwd ?? process.cwd(), env)) spawned++; + nextSpawn = Date.now() + (isProcessIdentityAlive(latestLaunch(jobDir)?.worker) ? pollMs : spawnPoll); } + wasLive = live; await sleep(Math.max(10, Math.min(pollMs, deadline - Date.now()))); } } @@ -174,10 +223,21 @@ export async function deliverMessage(jobDir, kind, text, extra = {}, options = { return { message, outcome, spawned }; } +export function pruneLaunches(jobDir, keep = 20) { + const files = fs.readdirSync(jobDir).filter(name => /^launch-.*\.json$/.test(name)); + const records = files.map(name => ({ name, record: readJsonOrNull(path.join(jobDir, name)) })) + .sort((a, b) => String(b.record?.requestedAt ?? "").localeCompare(String(a.record?.requestedAt ?? "")) || b.name.localeCompare(a.name)); + for (const entry of records.slice(keep)) { + // An unresolved live handoff is never discarded, regardless of history age. + if (entry.record?.state === "pending" && isProcessIdentityAlive(entry.record.worker)) continue; + fs.rmSync(path.join(jobDir, entry.name), { force: true }); + } + return records.slice(0, keep).map(entry => entry.record).filter(Boolean); +} function latestLaunch(jobDir) { return fs.readdirSync(jobDir).filter(name => /^launch-.*\.json$/.test(name)) .map(name => readJsonOrNull(path.join(jobDir, name))).filter(Boolean) - .sort((a, b) => String(a.requestedAt).localeCompare(String(b.requestedAt))).at(-1); + .sort((a, b) => String(b.requestedAt).localeCompare(String(a.requestedAt)))[0] ?? null; } function launchInterrupted(job, latest) { @@ -187,7 +247,7 @@ function launchInterrupted(job, latest) { export function retryQueuedWorker(jobDir, env = process.env) { const job = readJob(jobDir); - if (job && !TERMINAL_STATES.has(job.state) && !hasLiveWorker(jobDir) && !isProcessIdentityAlive(job.worker)) { + if (job && (!TERMINAL_STATES.has(job.state) || pendingInboxMessages(jobDir).length) && !hasLiveWorker(jobDir) && !isProcessIdentityAlive(job.worker)) { const latest = latestLaunch(jobDir); if (launchInterrupted(job, latest)) { // Re-read and reserve under the spawn lease; concurrent waiters share that fence. diff --git a/plugins/relay/scripts/lib/worker/recovery.mjs b/plugins/relay/scripts/lib/worker/recovery.mjs index 42ce91e1..c963c834 100644 --- a/plugins/relay/scripts/lib/worker/recovery.mjs +++ b/plugins/relay/scripts/lib/worker/recovery.mjs @@ -3,11 +3,12 @@ import process from "node:process"; import fs from "node:fs"; import path from "node:path"; -import { releaseLeaseToConsumer, spawnWorker } from "./delivery.mjs"; +import { DEFAULTS, readReservation } from "./admission.mjs"; +import { releaseLeaseToConsumer, startWithinWindow } from "./delivery.mjs"; import { acquireWorkerLease, isWorkerAlive } from "../jobs/lease.mjs"; import { computeJobView } from "../jobs/view.mjs"; import { messageExists, newMessageId, postInboxMessage } from "../jobs/messages.mjs"; -import { nowIso, readJob, readJsonOrNull, writeJob } from "../jobs/store.mjs"; +import { nowIso, readJob, readJsonOrNull, TERMINAL_STATES, writeJob } from "../jobs/store.mjs"; import { stopExecution, readProcessStartTime } from "../util/process.mjs"; // The sweep: stop those app-servers with every descendant, SIGTERM then SIGKILL. Returns @@ -51,9 +52,13 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { let launch = false; let cancelling = false; let cwd; + let original; + let reservation; try { let job = readJob(jobDir); if (computeJobView(job).state !== "dead" || (job.kind && job.kind !== "task")) return false; + original = job; + reservation = job.relay?.agentId ? readReservation(job.relay.agentId, env) : null; const epoch = job.recoveryEpoch ?? "start"; const previous = recoveryForEpoch(job); // A reservation interrupted before posting is repaired with the same message id. @@ -64,26 +69,40 @@ export async function resumeDeadJobOnce(jobDir, env = process.env) { cancelling = cancelIsRunning(jobDir); if (cancelling) return "busy"; if (previous && previous.state !== "reserved") { - writeJob(jobDir, { ...job, server: null, serverPid: null }); + writeJob(jobDir, { ...job, server: null, serverPid: null, + automaticResumes: job.automaticResumes.map(entry => entry.id === previous.id ? previous : entry) }); return false; } - const record = previous ?? { id: newMessageId(), epoch, at: nowIso(), fromAttempt: job.attempts?.length ?? 0, reason: "worker", state: "reserved" }; + const record = previous ?? { id: newMessageId(), epoch, at: nowIso(), fromAttempt: job.attempts?.length ?? 0, fromWorkerCount: job.workers?.length ?? 0, reason: "worker", state: "reserved" }; const records = previous ? job.automaticResumes : [...(job.automaticResumes ?? []), record]; writeJob(jobDir, { ...job, automaticResumes: records }); if (!messageExists(jobDir, record.id)) { postInboxMessage(jobDir, { id: record.id, kind: "resume", text: DEFAULT_RESUME_MESSAGE, isDefault: true, automatic: true }); } - writeJob(jobDir, { - ...job, automaticResumes: records.map(entry => entry.id === record.id ? { ...entry, state: "launched" } : entry), - state: "starting", worker: null, server: null, serverPid: null, launchPendingAt: nowIso() - }); cwd = job.request.cwd; launch = true; } finally { releaseLeaseToConsumer(jobDir, me, env, { spawn: !launch && !cancelling }); } - if (launch) spawnWorker(jobDir, cwd, { ...env, RELAY_WORKER_TRIGGER: "recovery" }); - return launch; + if (!launch) return false; + const result = await startWithinWindow(jobDir, cwd, { ...env, RELAY_WORKER_TRIGGER: "recovery" }, + { windowMs: Number(env.RELAY_ADMISSION_POLL_MS) || DEFAULTS.pollMs, refuse: false, reservation }); + // Workers update the job concurrently. Take its lease before changing the + // recovery record, or let the worker's own attempt bookkeeping be authoritative. + if (acquireWorkerLease(jobDir, me, "recovery")) { + try { + const job = readJob(jobDir), record = recoveryForEpoch(job); + // Cancellation or a successor can win while dispatch is being observed. + // Restoring the old dead view must never overwrite either publication. + const changedWorker = job.worker && (job.worker.pid !== original.worker?.pid || job.worker.startTime !== original.worker?.startTime); + if (!result.accepted && record?.state === "reserved" && !TERMINAL_STATES.has(job.state) && !changedWorker) { + writeJob(jobDir, { ...job, state: original.state, worker: original.worker }); + } + if (record?.state === "reserved" && result.accepted) writeJob(jobDir, { ...job, + automaticResumes: job.automaticResumes.map(entry => entry.id === record.id ? { ...entry, state: "launched" } : entry) }); + } finally { releaseLeaseToConsumer(jobDir, me, env, { spawn: false }); } + } + return result.accepted; } // One automatic recovery per user-work epoch, shared by worker and app-server deaths. @@ -91,5 +110,10 @@ export const DEFAULT_RESUME_MESSAGE = "Your session was cut off. Continue the ta export function recoveryForEpoch(job) { const epoch = job.recoveryEpoch ?? "start"; - return (job.automaticResumes ?? []).find(entry => entry.epoch === epoch); + const record = (job.automaticResumes ?? []).find(entry => entry.epoch === epoch); + // Worker-owned history proves that a reservation was taken without a launcher + // overwriting job.json while the worker held it. Preserve the single retry epoch. + if (record?.state === "reserved" && ((job.attempts ?? []).some(a => a.messageId === record.id) || + (record.fromWorkerCount != null && (job.workers?.length ?? 0) > record.fromWorkerCount))) return { ...record, state: "launched" }; + return record; } diff --git a/plugins/relay/scripts/lib/worker/registry.mjs b/plugins/relay/scripts/lib/worker/registry.mjs index 740e309a..591d7964 100644 --- a/plugins/relay/scripts/lib/worker/registry.mjs +++ b/plugins/relay/scripts/lib/worker/registry.mjs @@ -1,20 +1,35 @@ import fs from "node:fs"; import path from "node:path"; -import { admissionEnabled, admissionHome } from "./admission.mjs"; -import { writeJsonAtomic } from "../jobs/store.mjs"; +import { independentWorkerRegistry, safeWorkerRegistry, isProcessIdentityAlive } from "../util/process.mjs"; +import { readJsonOrNull, writeJsonAtomic } from "../jobs/store.mjs"; -export function registerWorker(worker, { env = process.env, platform = process.platform } = {}) { +export function registerWorker(worker, { env = process.env, platform = process.platform, directory = independentWorkerRegistry() } = {}) { if (!["linux", "darwin"].includes(platform)) return null; - const directory = `${admissionHome(env)}.workers`; + // Cleanup boundaries must survive an unwritable admission directory, even with + // the guard off. Publish before handoff; inability to publish prevents launch. + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + if (!safeWorkerRegistry(directory)) throw new Error("Worker identity directory must be private, owned by this user and not a symlink."); + if (!worker?.startTime) throw new Error("Worker process identity could not be read; no backend was started."); + pruneWorkerRegistry(directory); const file = path.join(directory, `${worker.pid}.json`); - try { - fs.mkdirSync(directory, { recursive: true }); - writeJsonAtomic(file, worker); - return file; - } catch (error) { - if (admissionEnabled(env, undefined, platform)) throw error; - // Guard-off launches cannot require admission storage. The launch intent and its - // reparenting acknowledgement remain mandatory writes in the job's own directory. - return null; + writeJsonAtomic(file, worker); + return file; +} + +// Registration performs a small rotating maintenance pass, independently of +// heartbeat sampling. Identity checks retain live or unreadable identities. +export function pruneWorkerRegistry(directory = independentWorkerRegistry(), { limit = 128, isAlive = isProcessIdentityAlive } = {}) { + let names; + try { names = fs.readdirSync(directory).filter(name => /^\d+\.json$/.test(name)).sort(); } + catch (error) { if (error.code === "ENOENT") return 0; throw error; } + const cursorFile = path.join(directory, ".sweep.json"); + const cursor = readJsonOrNull(cursorFile)?.after ?? ""; + const ordered = [...names.filter(name => name > cursor), ...names.filter(name => name <= cursor)].slice(0, limit); + let removed = 0; + for (const name of ordered) { + const file = path.join(directory, name), worker = readJsonOrNull(file); + if (worker?.startTime && !isAlive(worker)) { fs.rmSync(file, { force: true }); removed++; } } + if (ordered.length) writeJsonAtomic(cursorFile, { after: ordered.at(-1) }); + return removed; } diff --git a/plugins/relay/scripts/lib/worker/worker.mjs b/plugins/relay/scripts/lib/worker/worker.mjs index 4b75716a..9d4f7cc4 100644 --- a/plugins/relay/scripts/lib/worker/worker.mjs +++ b/plugins/relay/scripts/lib/worker/worker.mjs @@ -20,7 +20,7 @@ import { requestAccount } from "../jobs/request.mjs"; import { backendFor } from "../backends/index.mjs"; import { truncateCodePoints } from "../util/text.mjs"; import { acquireWorkerLease, releaseWorkerLease, readLease } from "../jobs/lease.mjs"; -import { claimInboxMessages, CLOSED_ACTIONS, newMessageId, owedMessages, pendingInboxMessages, readOutcome, writeOutcome } from "../jobs/messages.mjs"; +import { InboxStorageError, claimInboxMessages, CLOSED_ACTIONS, newMessageId, owedMessages, pendingInboxMessages, readOutcome, writeOutcome } from "../jobs/messages.mjs"; import { jobPaths, nowIso, readJob, writeFileAtomic, writeJob, readJsonOrNull, writeJsonAtomic } from "../jobs/store.mjs"; import { PLUGIN_VERSION } from "../util/plugin.mjs"; import { ownExecution, readProcessStartTime, listProcesses, readProcessTableCached, sampleExecution, executionBytes, isProcessIdentityAlive } from "../util/process.mjs"; @@ -187,6 +187,24 @@ export async function runWorker(jobDir, options = {}) { return result; } } + } catch (error) { + // A final handoff read can fail after the round saved its terminal state. + // Only this worker's lease authorizes correcting that provisional completion. + if (error instanceof InboxStorageError) { + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + const holder = readLease(jobDir); + const held = holder?.role === "worker" && sameProcess(holder, me); + // The final post-release read can also fail. Reacquire the fence before + // correcting our own completion; never overwrite a successor's record. + const acquired = !held && (!holder || !isProcessIdentityAlive(holder)) && acquireWorkerLease(jobDir, me, "storage"); + if (held || acquired) { + try { + const job = readJob(jobDir); + if (held || sameProcess(job?.worker, me)) writeJob(jobDir, { ...job, state: "failed", errorMessage: error.message, completedAt: nowIso() }); + } finally { releaseWorkerLease(jobDir, me); } + } + } + throw error; } finally { try { if (registryFile) fs.rmSync(registryFile, { force: true }); } catch {} } @@ -279,9 +297,12 @@ async function runWorkerRound(jobDir, options, held) { if (!message) { return; } + const reset = !message.automatic && ["steered", "turn-started", "delivered-before"].includes(result.action); + // A visible delivery receipt must already carry its durable recovery allowance. + // The caller can stop this worker immediately after observing the receipt. + if (reset && job.recoveryEpoch !== message.id) save({ recoveryEpoch: message.id }); writeOutcome(jobDir, message.id, result); const log = Array.isArray(job.messages) ? job.messages : []; - const reset = !message.automatic && ["steered", "turn-started", "delivered-before"].includes(result.action); save({ messages: [...log, { id: message.id, kind: message.kind, at: nowIso(), ...result }], ...(reset ? { recoveryEpoch: message.id } : {}) }); }; @@ -916,6 +937,9 @@ async function runWorkerRound(jobDir, options, held) { } clearInterval(poller); + // A successful turn is not completion evidence for unreadable message storage. + pendingInboxMessages(jobDir); + owedMessages(jobDir); const last = job.attempts.at(-1); if (cancelRequested) { if (last && !TERMINAL.has(last.state)) { diff --git a/plugins/relay/skills/jobs/SKILL.md b/plugins/relay/skills/jobs/SKILL.md index 76f6192f..0f82f1f0 100644 --- a/plugins/relay/skills/jobs/SKILL.md +++ b/plugins/relay/skills/jobs/SKILL.md @@ -171,7 +171,7 @@ dir: ~/.relay-jobs// ``` -- `state` is `completed`, `failed` (with `error:`), `cancelled`, `dead` (the job has not ended, but its worker is gone, or none started within 60 s of the launch, except a workerless job queued for admission; whether a recovery was tried is another matter, shown by `automatic-resume:`), or `queued` (waiting for the load guard; `queued:` says why), `starting`/`running`. A failed start is `state: not started` with `error:`. A block with no `---` has no report. +- `state` is `completed`, `failed` (with `error:`), `cancelled`, `dead` (the job has not ended, but its worker is gone, or its launch process ended before ownership, whether a recovery was tried is another matter, shown by `automatic-resume:`), or `queued` (waiting for the load guard; `queued:` says why), `starting`/`running`. A failed start is `state: not started` with `error:`. A block with no `---` has no report. - `completed` means the job finished its answer, not that its commands passed. Read the report. - Before you act on a result or show it, load the `relay:result-handling` skill: it reads the state before the report. - Lines that may appear: `queued:`, `refusal:`, `backend:`, `grok-home:` and `kimi-home:` (see Grok jobs, GLM jobs and Kimi jobs), `kind:` (a review), `tier: fast`, `mcp-off:` (the MCP servers this job turned off), `relay:`, `note:`, `stalled:` (no event for ten minutes; information only), `capacity-retries:`, `automatic-resume:` (a recovery and its cause), `relay-adopted:`, `undelivered:` (messages to the job, acknowledged and not run yet; `resume` runs them; it says nothing about the report), `not-delivered:` (dropped, for example by a cancel), and `relay-violation:` (the hook let the relay go, or a delivery or a cancel failed; read the job yourself with `C result `). @@ -256,10 +256,12 @@ A person stopping a relay (`x`, Esc or the app's Stop) leaves its job running. S A wide fan-out that needs no steering can start its jobs with `C start` a few seconds apart and collect each with `C wait `; `C status` only reads jobs and does not start deferred work; no Claude agent is held per job. Jobs started within the same second can hit a provider's request-rate limit. For Codex and Grok routes, the worker retries at 30, 60 and 120 s. ZCode and Kimi retry model calls themselves. -A job can wait as `queued` before it starts or between turns. Keep calling `wait`: queued work -remains pending and adoptable, including in a Workflow. Cancel, steer and continue still work. -Jobs without a worker need a submitter/adopter's wait, message or relay hook to retry spawning; -a bare `start` can stay dormant. `status` alone does not start deferred work: use `C wait `. +A job can wait as `queued` before its backend starts or between turns; its live worker owns that wait. +When the start queue is full, Workflow agents wait at spawn as queued without model tokens or job +waiters. A plain `relay:job` call waits up to 30 seconds then reports capacity. Use a Workflow for +large batches. A direct start that cannot get a worker within a minute reports not started (exit 6); +nothing runs later. A dead launch process or legacy workerless queue is dead and can be recovered +with `C wait --auto-resume`. Cancel, steer and continue still work for accepted jobs. The guard paces new backend starts and replacements 5 seconds apart across this user's sessions. Turns on a running backend are not paced and wait only for critical memory/pressure or provider pauses/probes. The first two identified very-heavy and first three other active jobs skip the host reserve and the busy-seat allowance, subject to critical checks, startup allowance, scope/fit limits, provider protection, ordering and start pacing. There is no very-heavy maximum. @@ -269,20 +271,21 @@ A macOS pressure warning is shown but holds nothing. Memory and load can delay n healthy machines have no fixed active ceiling. Turn the guard off only when the user asks. Sensor details, small-host fit limits and formulas are in [internals.md](internals.md). -`--json` prints one object whose first fields are `schema` (`relay./1`), `relayVersion` and `relayBuild`. An error under `--json` is that header with `error: { kind, message }` on stdout, `kind` one of `busy` (exit 4: try again), `not-found`, `usage`, `attempt-changed` or `error`. `C status --json` lists slim rows, newest first, 50 a page (`--limit N`, up to 200), with `nextCursor` for `--cursor`; `--active` lists only jobs that are queued, run, died or owe messages. One job (`C status --json`, `result`, `wait`, `cancel`) has every field, among them `attemptRecords` and `messages`, each message with how the job took it. +`--json` prints one object whose first fields are `schema` (`relay./1`), `relayVersion` and `relayBuild`. An error under `--json` is that header with `error: { kind, message }` on stdout, `kind` one of `busy` (exit 4: try again), `not-found`, `usage`, `attempt-changed`, `not-started` (exit 6) or `error`. `C status --json` lists slim rows, newest first, 50 a page (`--limit N`, up to 200), with `nextCursor` for `--cursor`; `--active` lists only jobs that are queued, run, died or owe messages. One job (`C status --json`, `result`, `wait`, `cancel`) has every field, among them `attemptRecords` and `messages`, each message with how the job took it. -Exit codes of `wait` and `result`: `0` completed with a report; `3` ended with no answer to take, which is `failed` or `dead` (even with a report), `completed` with no report, or messages not yet run (`undelivered:`), so read the state, then any report; `4` non-terminal (`queued`, `starting` or `running`); `5` cancelled, including with a pending continuation because cancellation takes precedence; `1` usage error. +Exit codes of `wait` and `result`: `0` completed with a report; `3` ended with no answer to take, which is `failed` or `dead` (even with a report), `completed` with no report, or messages not yet run (`undelivered:`), so read the state, then any report; `4` non-terminal (`queued`, `starting` or `running`); `5` cancelled, including with a pending continuation because cancellation takes precedence; `6` not started; `1` usage error. ### Continue a job by its id `C steer`, `C continue` and `C resume` post a new message each time they run, and print a receipt: `job:`, `message: `, `outcome:` and `state:`. - Exit `0`: the job recorded how it took the message, such as `steered`, `queued-next-turn`, `accepted-next-turn` or `turn-started`; `dropped` and `skipped` did not run, and `reason:` says why. -- Exit `4`: the message was posted, but no outcome came within the acknowledgement window (60 s), so `outcome:` is `pending`. It stays in the job's inbox for a worker to take. That is neither a refusal nor a failure. (`resume` also exits 4 when the job is busy, before it posts anything.) +- Exit `6`: no worker took the message within a minute. `not delivered: …; send it again later` means it was withdrawn before claim and will not run later. +- Exit `4`: the message has a live worker or remains claimed, but no outcome came within the acknowledgement window (60 s), so `outcome:` is `pending`. That is neither a refusal nor a failure. (`resume` also exits 4 when the job is busy, before it posts anything.) - Exit `1` can also follow posting: delivery or the state read can fail before a receipt is printed. Absence of a receipt alone proves nothing. Only a known pre-post validation refusal (such as an empty message or unknown job) proves nothing was posted. - Whether the turn then failed is the job's state, which `wait` reports. Before you send again after any nonzero exit, read the job's `messages` in `C status --json`: your message may have run since. -For a workerless queued job, `queued-next-turn` with a queue `reason:` acknowledges the message; it does not mean it ran. +Only a live worker writes `queued-next-turn` or `accepted-next-turn`. These receipts acknowledge ownership; they do not prove the turn ran. A continue, in full: diff --git a/plugins/relay/skills/jobs/internals.md b/plugins/relay/skills/jobs/internals.md index 8cf3846c..2bc79706 100644 --- a/plugins/relay/skills/jobs/internals.md +++ b/plugins/relay/skills/jobs/internals.md @@ -29,7 +29,7 @@ The harness marks each `agent()` call by the key it writes to the run's `journal The relay reads two things Claude Code does not publish as an interface: the agent transcript and the Workflow tool's run journal (`journal.jsonl`). Its markers were read from Claude Code 2.1.287 and 2.1.288 (`scripts/lib/harness/workflow.mjs`), and its hooks need `SubagentHandback`. Its hooks module needs the `session.send`, `agent.spawn` and `command.run` events of the Mod API, so the floor is Claude Code 2.1.307. A Claude Code update can break it. The check after an update is one workflow that runs a `relay:job` agent: the job's `job.json` must hold the call's key as `relay.journalKey` (once another relay has adopted the job, its `relay.json` holds the binding), and the script must get the state block back. -The module only checks headers and wakes a waiting relay: delivery runs on command hooks and the transcript. When a message is sent to a relay, the module's `session.send` hook runs `wake --agent `, which writes `relay-wake.json` in the job directory if that agent is the job's current relay, also when the job has ended, since a message can continue it. The guard passes each relay wait `--wake-after `, the time of the last transcript entry before the assistant message that holds the call, where Claude Code last put queued messages into the relay's conversation. The wait returns within a second of a marker newer than that, and the hook delivers the message at the relay's next call. Nothing removes a marker; a marker covered by the next boundary ends no more waits. +The module checks headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript. When a message is sent to a relay, the module's `session.send` hook runs `wake --agent `, which writes `relay-wake.json` in the job directory if that agent is the job's current relay, also when the job has ended, since a message can continue it. The guard passes each relay wait `--wake-after `, the time of the last transcript entry before the assistant message that holds the call, where Claude Code last put queued messages into the relay's conversation. The wait returns within a second of a marker newer than that, and the hook delivers the message at the relay's next call. Nothing removes a marker; a marker covered by the next boundary ends no more waits. Before a `relay:job` agent starts, the module's `agent.spawn` hook runs `check-header` on its prompt, the parser and request builder `start` uses, and denies the spawn with the parser's message when the header is bad. It leaves relative paths to `start`, since the relay's directory is not known at the spawn, and lets the spawn through when it cannot run the check, for a resumed agent, and for a Workflow agent whose prompt has no `---` line (a run that keeps its script from hooks puts a placeholder there); `start` checks again. A `PostToolUse` hook on `TaskStop` (`scripts/hooks/relay-stop.mjs`) takes the stopped agent's id from the tool's result and, when that agent is its job's current relay and the job has not ended, runs `cancel` and puts the result beside the TaskStop's; a failed cancel is a `relay-violation:`. A TaskStop on a whole Workflow cancels nothing, so a resume adopts the running jobs; the hook lists the run's jobs that keep running. @@ -99,10 +99,16 @@ Grok home. A job keeps its tier even when the `fast-tier` setting changes while The guard shares one admission home across this user's sessions, independently of jobs homes. For `C` CPU cores, the busy allowance is `F = min(16, max(2, C−2))`; the resident waiting-worker -limit is `Q = max(8, 2F)`. A workerless queued job remains valid; a wait, message or relay hook -retries its spawn at most once per admission poll, with a 64 MiB startup hold per waiting worker. -A workerless job with no spawn retry for 15 minutes stops deferring the stop-time review gate; -that changes gate deferral, not queue validity. +limit is `Q = max(8, 2F)`. Reservations, spawning workers and resident waiting workers share Q. +A reservation expires after 45 seconds and turns into a spawning slot atomically under the +admission lock. `RELAY_MAX_WAITING_WORKERS` explicitly overrides Q; the default follows CPU count. +Starts wait up to 60 seconds for a live worker lease or report not started (exit 6), with no later +execution. Workflow relays wait at `agent.spawn` through host processes, plain relays refuse after +30 seconds, and Workflow stall retries are gated at their first step. Every dispatch uses the +short-lived launcher. Failed launches stop after three failures and show their error and count. +A dead launcher or older workerless queue is dead, with explicit recovery through `wait --auto-resume`. +Undelivered messages refused before claim move to `withdrawn/`; a claim racing that rename retains +worker ownership. Admission keeps a tally of time per wait reason and reports the longest hold. Outside the minimum, new backend start admission requires `available − otherHolds − J ≥ R`, where `R = max(2 GiB, 0.15 × total)`. `J = 0.5 GiB` for Codex/Grok and `1 GiB` for Kimi/ZCode/unknown, held for 30 seconds without an diff --git a/plugins/relay/skills/jobs/orchestration.md b/plugins/relay/skills/jobs/orchestration.md index c61e5eb5..5d77a04c 100644 --- a/plugins/relay/skills/jobs/orchestration.md +++ b/plugins/relay/skills/jobs/orchestration.md @@ -28,7 +28,7 @@ outgrows the budget or finding counts keep rising. More rounds alone may not con | Claude-only side task | native `Agent` | one native agent | | Codex, Grok, ZCode or Kimi job needing messages | `relay:job` | one Haiku relay for the whole job | | Waves, branches, loops or recovery after caller loss | Workflow with `relay:job` | deterministic control flow and `resumeFromRunId` adoption | -| Wide fan-out with no relay steering needed | `C start` then `C wait ` | no Claude agent held per job; keep a waiter for deferred work | +| Wide fan-out with no relay steering needed | `C start` then `C wait ` | no Claude agent held per job; accepted jobs own their workers | A relay holds its Claude agent for the job's whole run, including queued waits, and spends some Claude usage. A Workflow slot stays occupied too. The documented per-run cap is up to 16 agents, @@ -62,9 +62,13 @@ probe goes first; a live probe is never replaced because of its age. These are b That job may finish `failed`; inspect its report and use same-job recovery where appropriate. Newly queued work keeps waiting. Acknowledgement of a message is not evidence its next turn ran. -`queued` is pending, never a result or a reason to submit the job again. Keep a wait/adopter: -`status` only reads; a wait, message or relay hook retries a workerless queue. Control Workflow jobs -by job id. Send a continue once, reconcile message records after uncertain errors, wait, then collect. +`queued` is pending under a live worker, never a result or a reason to submit the job again. +When the start queue is full, Workflow agents wait at spawn as queued, using no model tokens and +creating no job waiter until a place frees. A plain `relay:job` call waits up to 30 seconds then +reports capacity. Use a Workflow for large batches. A direct start without a worker after a minute +reports not started (exit 6); nothing runs later. Submit it again when capacity frees. +A dead launcher or an older workerless queue reads as dead; `wait --auto-resume` can recover it. +Control Workflow jobs by job id. Send a continue once, reconcile message records after uncertain errors, wait, then collect. At Claude's usage limit plain relays fail; Workflow runs pause where auto-continue is on. Backend jobs run on. Collect with `C wait ` or resume the Workflow with `resumeFromRunId`, never restart those jobs. Provider subscription windows and API dollar costs are different constraints; check diff --git a/tests/admission-e2e.test.mjs b/tests/admission-e2e.test.mjs index 034a725d..a2b3c400 100644 --- a/tests/admission-e2e.test.mjs +++ b/tests/admission-e2e.test.mjs @@ -3,7 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import assert from "node:assert/strict"; import test from "node:test"; -import { setup, companion, startJob } from "./support/job-helpers.mjs"; +import { setup, companion, startJob, waitFor } from "./support/job-helpers.mjs"; import { DEFAULTS as D } from "../plugins/relay/scripts/lib/worker/admission.mjs"; import { relayContext, relayStart, jobDirOf, pre, stop, decision } from "./support/relay-fixture.mjs"; const GiB = 2 ** 30; @@ -12,45 +12,45 @@ function readings(env, root) { env.RELAY_ADMISSION_HOME = path.join(root, "domain"); env.RELAY_TEST_ADMISSION_READINGS = path.join(root, "readings.json"); env.RELAY_ADMISSION_POLL_MS = "25"; + env.RELAY_JOBS_ACK_TIMEOUT_MS = "200"; const write = availableBytes => fs.writeFileSync(env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify({ availableBytes, totalBytes: 128 * GiB, cores: 64, load5: 0 })); write(0); return write; } -test("start defers a worker at critical pressure; continue is durable, wait owns retry and cancel works", t => { +test("workerless starts and continuations refuse; an explicit resume after pressure clears runs the work", t => { const ctx = setup(t), write = readings(ctx.env, ctx.root); - const { id, start } = startJob(ctx); - assert.match(start, /state: queued/); + const file = path.join(ctx.root, "prompt.md"); + fs.writeFileSync(file, "model: synthetic-model\neffort: low\n---\n[quick] Initial brief."); + const start = companion(ctx, ["start", "--prompt-file", file, "--json"]); + assert.equal(start.status, 6, start.stdout + start.stderr); + const id = JSON.parse(start.stdout).jobId; const initial = JSON.parse(companion(ctx, ["status", id, "--json"]).stdout); - assert.equal(initial.state, "queued"); assert.equal(initial.attempts, 0); + assert.equal(initial.state, "failed"); assert.equal(initial.attempts, 0); assert.equal(fs.existsSync(path.join(ctx.bin, "fake-codex-state.json")), false); - const next = companion(ctx, ["continue", id, "--message", "[quick] Queued follow-up.", "--json"]); - assert.equal(next.status, 0, next.stderr); - assert.equal(JSON.parse(next.stdout).outcome.action, "queued-next-turn"); - const pending = companion(ctx, ["wait", id, "--timeout-seconds", "0", "--exit-zero-while-running"]); - assert.equal(pending.status, 0); assert.match(pending.stdout, /still running: not an error/); + const next = companion(ctx, ["continue", id, "--message", "[quick] Withdraw this.", "--json"]); + assert.equal(next.status, 6, next.stdout + next.stderr); + assert.equal(JSON.parse(next.stdout).outcome.action, "not-started"); write(100 * GiB); - const done = companion(ctx, ["wait", id, "--json", "--timeout-seconds", String((D.spacingMs * 3) / 1000 + 10)]); + assert.equal(companion(ctx, ["wait", id, "--timeout-seconds", "1"]).status, 6); + assert.equal(fs.existsSync(path.join(ctx.bin, "fake-codex-state.json")), false); + ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = "5000"; + assert.equal(companion(ctx, ["resume", id]).status, 0); + const done = companion(ctx, ["wait", id, "--timeout-seconds", "15", "--json"]); assert.equal(done.status, 0, done.stdout + done.stderr); - assert.equal(JSON.parse(done.stdout).attempts, 2); - write(0); - const second = startJob(ctx); - const cancelled = companion(ctx, ["cancel", second.id, "--json"]); - assert.equal(cancelled.status, 0, cancelled.stdout + cancelled.stderr); - assert.equal(JSON.parse(cancelled.stdout).state, "cancelled"); - const ended = JSON.parse(companion(ctx, ["status", second.id, "--json"]).stdout); - assert.equal(ended.admission.state, "cancelled"); - assert.ok(ended.admission.since); assert.match(ended.admission.reason, /memory/); + assert.equal(JSON.parse(done.stdout).state, "completed"); }); for (const workflow of [false, true]) test(`relay ${workflow ? "Workflow" : "Agent"} queued result never finishes before admission`, async () => { const ctx = relayContext("job", { workflow: workflow ? "synthetic-run" : null, turnMs: 50 }); // Fixture paths are beneath a synthetic temporary home. const write = readings(ctx.env, ctx.env.RELAY_JOBS_HOME); + write(128 * GiB * D.criticalFraction + 2 * D.workerBytes); + ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = "5000"; if (workflow) ctx.append(ctx.userRequest(), ctx.environment(), ctx.workflowPrompt("computedTask", "[quick] Queued relay."), ctx.snapshot()); else ctx.append(ctx.prompt(`model: synthetic-model\neffort: ${D.lightEfforts[0]}\n---\n[quick] Queued relay.`)); const { jobId: id, started } = await relayStart(ctx); assert.ok(id, started.stdout); - assert.equal(JSON.parse(fs.readFileSync(path.join(jobDirOf(ctx, id), "job.json"))).state, "queued"); + await waitFor(() => JSON.parse(fs.readFileSync(path.join(jobDirOf(ctx, id), "job.json"))).state === "queued"); assert.equal(decision(await pre(ctx, "SubagentHandback", { message: "done" })), "deny"); assert.equal((await stop(ctx, "done"))?.decision, "block"); write(100 * GiB); @@ -59,10 +59,10 @@ for (const workflow of [false, true]) test(`relay ${workflow ? "Workflow" : "Age assert.equal(JSON.parse(done.stdout).state, "completed"); }); -for (const json of [false, true]) test(`background task reports deferred state (${json ? "JSON" : "text"})`, { skip: !["linux", "darwin"].includes(process.platform) }, t => { +for (const json of [false, true]) test(`background task refuses without a worker (${json ? "JSON" : "text"})`, { skip: !["linux", "darwin"].includes(process.platform) }, t => { const ctx = setup(t); readings(ctx.env, ctx.root); const result = companion(ctx, ["task", "--background", ...(json ? ["--json"] : []), "[quick] Deferred task."]); - assert.equal(result.status, 0, result.stderr); - if (json) assert.equal(JSON.parse(result.stdout).state, "queued"); - else assert.match(result.stdout, /state: queued/); + assert.equal(result.status, 6, result.stderr); + if (json) assert.equal(JSON.parse(result.stdout).state, "not started"); + else assert.match(result.stdout, /state: not started/); }); diff --git a/tests/admission-surfaces.test.mjs b/tests/admission-surfaces.test.mjs index 8bb3eb52..e3a3ff88 100644 --- a/tests/admission-surfaces.test.mjs +++ b/tests/admission-surfaces.test.mjs @@ -60,8 +60,12 @@ test("corrupt stored settings normalize; JSON listing reads no sensors or lock", test("a worker-spawn failure is never retried as a guard failure", async t => { const { spawnWithAdmission } = await import("../plugins/relay/scripts/lib/worker/admission.mjs"); + const ctx = setup(t); + const { createJobDir, writeJob } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { jobDir } = createJobDir(ctx.work, "spawn-failure", ctx.env); + writeJob(jobDir, { id: "spawn-failure", state: "starting", request: {} }); let calls = 0; - assert.throws(() => spawnWithAdmission("/synthetic/absent", { RELAY_ADMISSION: "off" }, () => { calls++; throw new Error("synthetic spawn failure"); }), /synthetic spawn failure/); + assert.throws(() => spawnWithAdmission(jobDir, { RELAY_ADMISSION: "off" }, () => { calls++; throw new Error("synthetic spawn failure"); }), /synthetic spawn failure/); assert.equal(calls, 1); }); diff --git a/tests/admission-worker.test.mjs b/tests/admission-worker.test.mjs index c25c92b9..cfcdf37d 100644 --- a/tests/admission-worker.test.mjs +++ b/tests/admission-worker.test.mjs @@ -254,18 +254,13 @@ test("first dispatch consumes its start grant once and preserves the actual queu }); -test("dormant workerless jobs stop deferring the repository review gate after fifteen minutes", async t => { +test("workerless queues without a live launcher do not defer the review gate", async t => { const { findActiveJob } = await import("../plugins/relay/scripts/hooks/stop-review-gate.mjs"); const s = seed(t), oldHome = process.env.RELAY_JOBS_HOME; process.env.RELAY_JOBS_HOME = s.ctx.env.RELAY_JOBS_HOME; t.after(() => { if (oldHome === undefined) delete process.env.RELAY_JOBS_HOME; else process.env.RELAY_JOBS_HOME = oldHome; }); - const now = Date.now(), at = new Date(now).toISOString(); - writeJob(s.dir, { ...readJob(s.dir), state: "queued", worker: null, admission: { waitKind: "worker", since: at, workerRetryAt: at } }); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000).id, readJob(s.dir).id); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000 + 1), null); - const job = readJob(s.dir); - writeJob(s.dir, { ...job, admission: { ...job.admission, workerRetryAt: new Date(now + 15 * 60000 + 1).toISOString() } }); - assert.equal(findActiveJob(s.ctx.work, now + 15 * 60000 + 1).id, job.id, "an adopter makes it active again"); + writeJob(s.dir, { ...readJob(s.dir), state: "queued", worker: null, admission: { waitKind: "worker" } }); + assert.equal(findActiveJob(s.ctx.work), null); }); test("a heartbeat during slow probe startup releases the claim without provider success", async t => { diff --git a/tests/admission.test.mjs b/tests/admission.test.mjs index 547ca8df..f3bead95 100644 --- a/tests/admission.test.mjs +++ b/tests/admission.test.mjs @@ -280,6 +280,7 @@ test("domain ignores custom jobs homes; spawn reservations bound waiting workers test("Windows admission is a visible no-op even with stale files and failed sensors", t => { const s = setup(t), c = s.ctx(); + writeJob(c.jobDir, { id: c.jobId, state: "starting", request: c.request }); const before = fs.readdirSync(s.root); const deps = { ...s.deps, platform: "win32", readAdmissionConfig() { throw new Error("no config read"); }, readMachine() { throw new Error("no sensor read"); } }; assert.match(admit(c, deps).record.notes.join(" "), /unsupported on win32/); @@ -1059,6 +1060,7 @@ test("the environment off switch survives configuration read errors", t => { const s = setup(t), c = s.ctx(), env = { ...s.env, RELAY_ADMISSION: "off" }; const deps = { ...s.deps, env, readAdmissionConfig: () => { throw new Error("unreadable configuration"); } }; assert.equal(admit(c, deps).admit, true); + writeJob(c.jobDir, { id: c.jobId, state: "starting", request: c.request }); assert.equal(spawnWithAdmission(c.jobDir, env, () => 42, deps), 42); }); @@ -1132,3 +1134,53 @@ test("a burst can reserve other seats while the first launch handshake is still assert.equal(sleeps, 0); assert.ok(jobs.every(c => readJob(c.jobDir).admission?.causeKind !== "lock")); }); + +test('admitted records retain the longest hold instead of the last short pacing hold', t => { + const s = setup(t), c = s.ctx({ model: 'synthetic-model' }); + s.setMachine({ availableBytes: 0 }); + let result = admit(c, s.deps); + c.admission = result.record; c.waitingSince = result.record.since; + assert.equal(result.waitKind, 'critical'); + s.tick(120000); s.setMachine({ availableBytes: 100 * GiB }); + fs.writeFileSync(path.join(s.home, 'timing.json'), JSON.stringify({ lastStart: new Date(s.deps.now()).toISOString() })); + result = admit(c, s.deps); assert.equal(result.waitKind, 'stagger'); c.admission = result.record; + s.tick(D.spacingMs); result = admit(c, s.deps); + assert.equal(result.admit, true); assert.equal(result.record.waitKind, 'critical'); + assert.match(result.record.reason, /memory/); + assert.equal(Object.values(result.record.waitTotals).find(x => x.kind === 'critical').ms, 120000); + assert.equal(Object.values(result.record.waitTotals).find(x => x.kind === 'stagger').ms, D.spacingMs); +}); + +test('slot cleaning and release require fresh live execution evidence even with a busy process cache', async t => { + const { executionAlive } = await import('../plugins/relay/scripts/lib/worker/admission.mjs'); + const s = setup(t), c = s.ctx(), dir = c.jobDir, file = path.join(s.home, 'slots', `${c.jobId}.json`); + const leader = { pid: 600000000, startTime: 'linux:100' }; + const tool = { pid: 600000001, ppid: 1, pgid: leader.pid, startTime: 'linux:200' }; + writeJob(dir, { id: c.jobId, request: c.request, worker: leader, server: tool, state: 'running', execution: { leader, pgid: leader.pid } }); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, JSON.stringify({ v: 1, jobId: c.jobId, jobDir: dir, pid: leader.pid, processStart: leader.startTime, + backend: 'codex', class: 'light', state: 'running', since: new Date(s.deps.now()).toISOString() })); + fs.writeFileSync(path.join(s.home, '.ps-table.json'), JSON.stringify({ at: Date.now() - 60000, rows: [] })); + const lock = path.join(s.home, '.ps-table.json.lock'); assert.ok(acquireOwnership(lock)); t.after(() => releaseOwnership(lock)); + const deps = { ...s.deps, listProcesses: () => [tool] }; + assert.equal(executionAlive({ ...readJob(dir), jobDir: dir }, deps), true); + assert.equal(describeGuard(s.env, {}, deps).activeSeats, 1); assert.equal(fs.existsSync(file), true); + assert.equal(releaseSlot(s.home, c.jobId, leader, deps), false); assert.equal(fs.existsSync(file), true); +}); + +test('changing memory readings accumulate one bounded hold kind and outweigh a short final stagger', t => { + const s = setup(t); fillOtherMinimum(s); + const c = s.ctx({ model: 'synthetic-model' }); + for (let i = 0; i < 100; i++) { + s.setMachine({ availableBytes: (10 + i / 100) * GiB }); + const held = admit(c, s.deps); assert.equal(held.waitKind, 'memory'); + c.admission = held.record; c.waitingSince = held.record.since; s.tick(1000); + } + s.setMachine({ availableBytes: 100 * GiB }); + fs.writeFileSync(path.join(s.home, 'timing.json'), JSON.stringify({ lastStart: new Date(s.deps.now()).toISOString() })); + const stagger = admit(c, s.deps); c.admission = stagger.record; + s.tick(D.spacingMs); const result = admit(c, s.deps); + assert.equal(result.admit, true); assert.equal(result.record.waitKind, 'memory'); + assert.equal(result.record.waitTotals.memory.ms, 100000); + assert.equal(Object.keys(result.record.waitTotals).length, 2); +}); diff --git a/tests/cancel-dead.test.mjs b/tests/cancel-dead.test.mjs index 3798a29b..67359234 100644 --- a/tests/cancel-dead.test.mjs +++ b/tests/cancel-dead.test.mjs @@ -5,6 +5,7 @@ import { once } from "node:events"; import test from "node:test"; import assert from "node:assert/strict"; import { setup, companion, waitFor } from "./support/job-helpers.mjs"; +import { acquireWorkerLease, releaseWorkerLease } from "../plugins/relay/scripts/lib/jobs/lease.mjs"; import { writeExecutable } from "./support/helpers.mjs"; import { createJobDir, readJob, writeJob } from "../plugins/relay/scripts/lib/jobs/store.mjs"; import { isProcessIdentityAlive, listProcesses, readProcessStartTime } from "../plugins/relay/scripts/lib/util/process.mjs"; @@ -235,3 +236,20 @@ fs.readFileSync = function(file, ...args) { assert.equal(isProcessIdentityAlive(identity), false); assert.equal(isProcessIdentityAlive(member), false); }); + + +test("a completed job with a closing worker records one cleanup receipt without delegating another", async t => { + const ctx = setup(t); + const worker = await sleeper(t, true), server = await sleeper(t, false); + const { id, jobDir } = recorded(ctx, worker.identity, server.identity, { state: "completed", attempts: [{ attempt: 1, state: "completed" }] }); + assert.equal(acquireWorkerLease(jobDir, worker.identity), true); + t.after(() => releaseWorkerLease(jobDir, worker.identity)); + const result = companion(ctx, ["cancel", id]); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(isProcessIdentityAlive(worker.identity), true); + assert.equal(isProcessIdentityAlive(server.identity), false); + const messages = ["inbox", "inbox/claimed"].flatMap(dir => fs.readdirSync(path.join(jobDir, dir)).filter(n => n.endsWith(".json")).map(n => JSON.parse(fs.readFileSync(path.join(jobDir, dir, n))))).filter(m => m.kind === "cancel"); + assert.equal(messages.length, 1); + const outcome = JSON.parse(fs.readFileSync(path.join(jobDir, "inbox/outcomes", `${messages[0].id}.json`))); + assert.equal(outcome.action, "cancelled"); +}); diff --git a/tests/commands.test.mjs b/tests/commands.test.mjs index dbc9f1a5..c61140c1 100644 --- a/tests/commands.test.mjs +++ b/tests/commands.test.mjs @@ -253,7 +253,7 @@ test("the jobs skill and design.md say a message that arrives while a turn is st test("dead and exit 3 are defined as view.mjs computes them, and the continue recipe waits within the Bash timeout", () => { const skill = flat(read("skills/jobs/SKILL.md")); - for (const fact of ["none started within 60 s of the launch", "except a workerless job queued for admission", "(even with a report)", "`completed` with no report", + for (const fact of ["its launch process ended before ownership", "not started (exit 6)", "(even with a report)", "`completed` with no report", "`completed` is only the first gate"]) { assert.ok(skill.includes(fact), fact); } @@ -265,7 +265,7 @@ test("dead and exit 3 are defined as view.mjs computes them, and the continue re const recipe = section(skill, "### Continue a job by its id", "How the hook carries the relay"); for (const fact of ["Exit `4`", "acknowledgement window (60 s)", "`skipped`", "--timeout-seconds 100", "the Bash tool's default timeout of 120 s", "the `undelivered` list", "After a `resume`, go back to step 2.", - "`report: none` is no new report", "Absence of a receipt alone proves nothing", "after any nonzero exit", "`queued-next-turn` with a queue `reason:`"]) { + "`report: none` is no new report", "Absence of a receipt alone proves nothing", "after any nonzero exit", "Only a live worker writes `queued-next-turn` or `accepted-next-turn`"]) { assert.ok(recipe.includes(fact), fact); } assert.doesNotMatch(recipe, /`no-op`/); @@ -510,7 +510,7 @@ test("README carries the first hour, stop gate, Claude Code dependency and devel assert.match(source, /A Claude Code update can break it\./); assert.ok(source.includes("the job's `job.json` must hold the call's key as `relay.journalKey`")); assert.ok(source.includes("once another relay has adopted the job, its `relay.json` holds the binding")); - assert.match(source, /The module only checks headers and wakes a waiting relay: delivery runs on command hooks and the transcript/); + assert.match(source, /The module checks headers, gates relay starts and wakes a waiting relay: delivery runs on command hooks and the transcript/); } assert.ok(internals.includes("stores that key as `relay.journalKey` in the job's `job.json`")); assert.doesNotMatch(internals, /binds the job to that key in the job's `relay\.json`/); diff --git a/tests/execution-groups.test.mjs b/tests/execution-groups.test.mjs index abcef172..f71f24d6 100644 --- a/tests/execution-groups.test.mjs +++ b/tests/execution-groups.test.mjs @@ -5,7 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { spawn, spawnSync } from "node:child_process"; import { once } from "node:events"; -import { setup, startJob, readJob, jobDir, companion, waitFor, finishJob, ROOT } from "./support/job-helpers.mjs"; +import { setup, startJob, promptFile, jobDirs, readJob, jobDir, companion, waitFor, finishJob, ROOT } from "./support/job-helpers.mjs"; import { listProcesses, executionMembers, sampleExecution, stopExecution, readProcessStartTime, isProcessIdentityAlive, readProcessTableCached } from "../plugins/relay/scripts/lib/util/process.mjs"; import { describeGuard, renderGuard } from "../plugins/relay/scripts/lib/worker/admission.mjs"; import { writeJob, readJob as readRecord } from "../plugins/relay/scripts/lib/jobs/store.mjs"; @@ -176,7 +176,19 @@ for (const phase of ["running", "admission", "recovery"]) test(`a long-lived wai ctx.env.RELAY_TEST_ADMISSION_READINGS = fixture; fs.writeFileSync(fixture, JSON.stringify({ totalBytes: 16 * 2 ** 30, availableBytes: 0, load5: 0, cores: 8 })); } - const { id } = startJob(ctx, undefined, "[long] Work"); + let id; + if (phase === "recovery") ({ id } = startJob(ctx, undefined, "[long] Work")); + else { + // Model a launcher interrupted during its bounded start window, before receipt. + const { createJobDir, jobPaths } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { buildLaunchRequest } = await import("../plugins/relay/scripts/lib/jobs/request.mjs"); + id = `job-deferred-${phase}`; + const { jobDir: dir } = createJobDir(ctx.work, id, ctx.env); + writeJob(dir, { id, kind: "task", state: "starting", createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: buildLaunchRequest({ model: "gpt-test", effort: "low", cd: ctx.work }, { baseDir: ctx.work, env: ctx.env }) }); + fs.writeFileSync(jobPaths(dir).brief, "[long] Work"); + assert.equal(spawnWorker(dir, ctx.work, ctx.env), null); + } const dir = jobDir(ctx, id); let old = readJob(ctx, id); if (phase === "recovery") { @@ -254,7 +266,8 @@ if(process.argv[1]?.endsWith('/launcher.mjs')) { // A pending handoff remains retryable. Concurrent waiters use the same lease fence. const waiters = Array.from({ length: 3 }, () => spawn(process.execPath, [script, "wait", id, "--timeout-seconds", "30"], { env: ctx.env, cwd: ctx.work, stdio: "ignore" })); t.after(() => { for (const waiter of waiters) { try { waiter.kill("SIGKILL"); } catch {} } }); - await waitFor(() => readJob(ctx, id).worker?.pid && readJob(ctx, id).launch?.state === "acknowledged", 20000); + try { await waitFor(() => readJob(ctx, id).worker?.pid && readJob(ctx, id).launch?.state === "acknowledged", 20000); } + catch (error) { const job = readJob(ctx, id); throw new Error(`${error.message}: ${JSON.stringify({ state: job.state, worker: job.worker, admission: job.admission, launch: job.launch, error: job.errorMessage, intents: fs.readdirSync(dir).filter(name => name.startsWith("launch-")).map(name => JSON.parse(fs.readFileSync(path.join(dir, name)))) })}`); } assert.equal(readJob(ctx, id).workers.filter(w => w.pid !== old.worker.pid).length, 1); assert.ok(readJob(ctx, id).launches.some(launch => launch.state === "death-observed" && launch.deathObservedAt && !launch.endedAt)); }); @@ -414,20 +427,19 @@ test("a nested worker and its backend are excluded even if sampled during handof assert.deepEqual(result.survivors, []); }); -test("a failed launch stays queued and plain wait dispatches it without auto-resume", async t => { - const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: "off" } }); +for (const command of ["start", "task"]) test(`a permanently failing ${command} launcher stops after three failures and exposes its error`, async t => { + const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: "off", RELAY_ADMISSION_POLL_MS: "20" } }); const preload = path.join(ctx.root, "fail-launch.mjs"); - fs.writeFileSync(preload, `if(process.argv[1]?.endsWith('/launcher.mjs')) process.exit(1);`); - const { id } = startJob(ctx, undefined, "[quick] Retry the launch.", { NODE_OPTIONS: `--import=${preload}` }); - const queued = readJob(ctx, id); - assert.equal(queued.state, "queued"); - assert.equal(queued.admission.causeKind, "launch"); - assert.equal(queued.admission.waitKind, "worker"); - assert.equal(companion(ctx, ["wait", id, "--timeout-seconds", "15"]).status, 0); - await finishJob(ctx, id); - assert.equal(readJob(ctx, id).attempts.length, 1); - assert.equal(readJob(ctx, id).workers.length, 1); - assert.equal(readJob(ctx, id).autoRecoveries?.length ?? 0, 0); + fs.writeFileSync(preload, `if(process.argv[1]?.endsWith('/launcher.mjs')) { console.error('synthetic launch failure'); process.exit(1); }`); + const result = companion(ctx, command === "start" ? ["start", "--prompt-file", promptFile(ctx)] : ["task", "[quick] Dummy task."], { NODE_OPTIONS: `--import=${preload}` }); + assert.equal(result.status, 6, result.stdout + result.stderr); + const dir = jobDirs(ctx)[0], failed = JSON.parse(fs.readFileSync(path.join(dir, "job.json"))); + assert.equal(failed.admission.launchFailures, 3); assert.match(failed.admission.launchError, /synthetic launch failure/); + const waited = companion(ctx, ["wait", failed.id, "--timeout-seconds", "1"]); + assert.equal(waited.status, 6); assert.match(waited.stdout, /3 failed launches of 3.*synthetic launch failure/); + assert.equal(fs.readdirSync(dir).filter(name => name.startsWith("launch-")).length, 3); + assert.equal(companion(ctx, ["resume", failed.id, "--message", "[quick] Retry explicitly."]).status, 0); + await finishJob(ctx, failed.id); }); for (const cancel of [false, true]) test(`an actual nested task survives outer cleanup after sampling its launcher handoff (cancel=${cancel})`, async t => { @@ -435,7 +447,7 @@ for (const cancel of [false, true]) test(`an actual nested task survives outer c const inner = setup(t, { turnMs: 30000, env: { RELAY_WORKER_SCOPE: "off" } }); const shared = path.join(outer.root, "admission"); outer.env.RELAY_ADMISSION_HOME = shared; - inner.env.RELAY_ADMISSION_HOME = shared; + inner.env.RELAY_ADMISSION_HOME = cancel ? "/dev/null/unwritable" : shared; const marker = path.join(outer.root, "nested-started"), blocked = path.join(inner.root, "blocked"), release = path.join(inner.root, "release"); const preload = path.join(inner.root, "hold-launcher.mjs"); fs.writeFileSync(preload, `import fs from 'node:fs'; @@ -486,7 +498,7 @@ test("scope cleanup excludes older members and preserves nested workers sharing const scopeKills = [], signals = []; t.mock.method(fs, "readFileSync", (file, ...args) => { if (file === "/proc/sys/kernel/random/boot_id") return "current-boot\n"; - if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return `${older.pid}\n${worker.pid}\n${tool.pid}\n`; + if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return table.map(row => row.pid).join("\n"); return read(file, ...args); }); t.mock.method(fs, "statSync", (file, ...args) => file === `/sys/fs/cgroup${group}` ? { ino: 37 } : stat(file, ...args)); @@ -502,3 +514,122 @@ test("scope cleanup excludes older members and preserves nested workers sharing assert.deepEqual(scopeKills, []); assert.deepEqual(result.survivors, []); }); + +test('scope-only pids absent from the process table remain unresolved and keep their seat', async t => { + const { executionAlive } = await import('../plugins/relay/scripts/lib/worker/admission.mjs'); + const group = '/synthetic/relay-jobs.slice/run-unresolved.scope', leader = { pid: 600000000, startTime: 'linux:100' }, pid = 600000001; + const read = fs.readFileSync, stat = fs.statSync; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + if (file === '/proc/sys/kernel/random/boot_id') return 'synthetic-boot\n'; + if (file === `/sys/fs/cgroup${group}/cgroup.procs`) return `${pid}\n`; + return read(file, ...args); + }); + t.mock.method(fs, 'statSync', (file, ...args) => file === `/sys/fs/cgroup${group}` ? { ino: 37 } : stat(file, ...args)); + const job = { worker: leader, execution: { leader, pgid: leader.pid, cgroup: group, cgroupId: 37, bootId: 'synthetic-boot' } }; + const members = executionMembers(job, []); + assert.deepEqual([...members], [{ pid, startTime: null, identityUnknown: true }]); + assert.equal(members.cause, undefined, "unknown scope pids retain capacity without suppressing verified signals"); + assert.equal(executionAlive(job, { env: {}, listProcesses: () => [], isAlive: () => false }), true); +}); + +test('launch history and crash registry maintenance stay bounded without scanning history on sample', async t => { + const { pruneLaunches } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const { pruneWorkerRegistry } = await import('../plugins/relay/scripts/lib/worker/registry.mjs'); + const ctx = setup(t), directory = path.join(ctx.root, 'launch-history'), registry = path.join(ctx.root, 'registry-history'); + fs.mkdirSync(directory); fs.mkdirSync(registry); + for (let i = 0; i < 100; i++) { + fs.writeFileSync(path.join(directory, `launch-${i}.json`), JSON.stringify({ state: 'acknowledged', requestedAt: new Date(i * 1000).toISOString() })); + fs.writeFileSync(path.join(registry, `${600000000 + i}.json`), JSON.stringify({ pid: 600000000 + i, startTime: 'gone', jobDir: '/synthetic/other' })); + } + pruneLaunches(directory); assert.equal(fs.readdirSync(directory).length, 20); + let checks = 0; + while (pruneWorkerRegistry(registry, { limit: 16, isAlive: () => { checks++; return false; } })) {} + assert.equal(checks, 100); assert.equal(fs.readdirSync(registry).filter(name => /^\d/.test(name)).length, 0); + const readdir = fs.readdirSync; + t.mock.method(fs, 'readdirSync', (dir, ...args) => { assert.notEqual(dir, registry, 'sampling cannot enumerate historical workers'); return readdir(dir, ...args); }); + const worker = { pid: 600000001, startTime: 'linux:200', ppid: 1, pgid: 600000001 }; + fs.writeFileSync(path.join(registry, `${worker.pid}.json`), JSON.stringify({ ...worker, jobDir: '/synthetic/other' })); + assert.deepEqual(executionMembers({ jobDir: directory, server: worker, execution: { leader: worker, pgid: worker.pid, workerRegistry: registry } }, [worker]), []); +}); + +test('caller acknowledgement observation timeout never erases a live dispatched worker', async t => { + const ctx = setup(t, { env: { RELAY_WORKER_SCOPE: 'off' } }); + const { createJobDir, jobPaths } = await import('../plugins/relay/scripts/lib/jobs/store.mjs'); + const { buildLaunchRequest } = await import('../plugins/relay/scripts/lib/jobs/request.mjs'); + const { startWithinWindow } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const { jobDir: dir } = createJobDir(ctx.work, 'job-delayed-observation', ctx.env); + writeJob(dir, { id: 'job-delayed-observation', kind: 'task', state: 'starting', createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: buildLaunchRequest({ model: 'gpt-test', effort: 'low', cd: ctx.work }, { baseDir: ctx.work, env: ctx.env }) }); + fs.writeFileSync(jobPaths(dir).brief, '[held] Wait for observation.'); + const read = fs.readFileSync, now = Date.now; let offset = 0, delayed = false; + t.mock.method(fs, 'readFileSync', (file, ...args) => { + const result = read(file, ...args); + if (!delayed && String(file).startsWith(path.join(dir, 'launch-')) && String(result).includes('"acknowledged"')) { + delayed = true; offset = 11000; + return JSON.stringify({ ...JSON.parse(String(result)), state: 'pending' }); + } + return result; + }); + t.mock.method(Date, 'now', () => now() + offset); + const result = await startWithinWindow(dir, ctx.work, ctx.env, { windowMs: 20000 }); + assert.equal(delayed, true); assert.equal(result.accepted, true); + const job = JSON.parse(fs.readFileSync(path.join(dir, 'job.json'))); + assert.ok(job.worker?.pid); assert.notEqual(job.state, 'queued'); + await waitFor(() => ctx.fakeState().appServerStarts === 1); + assert.equal(job.workers.length, 1); assert.equal(ctx.fakeState().appServerStarts, 1); + assert.equal(fs.readdirSync(dir).filter(name => /^launch-/.test(name)).length, 1); +}); + +test('a fork between process-table and scope reads does not stop cancellation of verified processes', { skip: process.platform !== 'linux' }, async t => { + const probe = spawnSync('systemd-run', ['--user', '--scope', '--quiet', '--collect', '--slice=relay-jobs.slice', '--', 'true']); + if (probe.status !== 0) return t.skip('user systemd manager unavailable'); + const ctx = setup(t, { turnMs: 30000 }); + const marker = path.join(ctx.root, 'fork-now'), childFile = path.join(ctx.root, 'forked-child.json'); + const preload = path.join(ctx.root, 'fork-during-cancel.mjs'); + const processModule = new URL('../plugins/relay/scripts/lib/util/process.mjs', import.meta.url).href; + fs.writeFileSync(preload, `import fs from 'node:fs'; import {spawn} from 'node:child_process'; +import {readProcessStartTime} from ${JSON.stringify(processModule)}; +const marker=${JSON.stringify(marker)}, childFile=${JSON.stringify(childFile)}; +if(process.argv[1]?.endsWith('/bin/codex')) { + let forked=false; + const timer=setInterval(()=>{if(!forked && fs.existsSync(marker)) { + forked=true; const child=spawn('sleep',['300'],{detached:true,stdio:'ignore'}); + child.on('spawn',()=>{fs.writeFileSync(childFile,JSON.stringify({pid:child.pid,startTime:readProcessStartTime(child.pid)}));child.unref();}); + }},5); timer.unref(); +} +if(process.argv[2]==='cancel') { + const read=fs.readFileSync; + let paused=false; + fs.readFileSync=function(file,...args) { + if(!paused && String(file).endsWith('/cgroup.procs')) { + paused=true;fs.writeFileSync(marker,'fork'); const until=Date.now()+5000; + while(!fs.existsSync(childFile) && Date.now() readJob(ctx, id).turnId); + assert.ok(readJob(ctx, id).execution.cgroup); + let child; + t.after(() => { if (child && isProcessIdentityAlive(child)) process.kill(child.pid, 'SIGKILL'); }); + const cancelled = companion(ctx, ['cancel', id]); + child = JSON.parse(fs.readFileSync(childFile)); + assert.equal(cancelled.status, 0, cancelled.stdout + cancelled.stderr); + assert.equal(readJob(ctx, id).state, 'cancelled'); + await waitFor(() => !isProcessIdentityAlive(child)); +}); + +test('worker identity storage rejects symlinks, shared modes and missing identities', async t => { + const { registerWorker } = await import('../plugins/relay/scripts/lib/worker/registry.mjs'); + const ctx = setup(t), directory = path.join(ctx.root, 'identity-storage'), link = path.join(ctx.root, 'identity-link'); + fs.mkdirSync(directory, { mode: 0o700 }); fs.symlinkSync(directory, link); + const worker = { pid: process.pid, startTime: readProcessStartTime(process.pid), jobDir: ctx.root }; + assert.throws(() => registerWorker(worker, { directory: link }), /private.*symlink/); + fs.chmodSync(directory, 0o777); + assert.throws(() => registerWorker(worker, { directory }), /private.*symlink/); + fs.chmodSync(directory, 0o700); + assert.throws(() => registerWorker({ ...worker, startTime: null }, { directory }), /identity could not be read/); + const file = registerWorker(worker, { directory }); assert.equal(JSON.parse(fs.readFileSync(file)).startTime, worker.startTime); +}); diff --git a/tests/gate.test.mjs b/tests/gate.test.mjs new file mode 100644 index 00000000..7fa54d4c --- /dev/null +++ b/tests/gate.test.mjs @@ -0,0 +1,255 @@ +import './support/isolate-env.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { setup, companion, promptFile, jobDirs, waitFor, ROOT } from './support/job-helpers.mjs'; +import { reserveWaitingPlace, bindReservation, readReservation, describeGuard, spawnWithAdmission, admit, DEFAULTS as D } from '../plugins/relay/scripts/lib/worker/admission.mjs'; +import { readJob, writeJob, jobPaths } from '../plugins/relay/scripts/lib/jobs/store.mjs'; +import { computeJobView, loadJobView, exitCodeFor } from '../plugins/relay/scripts/lib/jobs/view.mjs'; +import { hasLiveWorker, startWithinWindow, awaitOutcome } from '../plugins/relay/scripts/lib/worker/delivery.mjs'; +import { acquireWorkerLease, releaseWorkerLease } from '../plugins/relay/scripts/lib/jobs/lease.mjs'; +import { claimInboxMessages, messageAccount, postInboxMessage, writeOutcome } from '../plugins/relay/scripts/lib/jobs/messages.mjs'; +import { readProcessStartTime } from '../plugins/relay/scripts/lib/util/process.mjs'; +import { resumeDeadJobOnce } from '../plugins/relay/scripts/lib/worker/recovery.mjs'; + +function domain(t, limit = 2) { + const ctx = setup(t, { env: { RELAY_ADMISSION: 'on', RELAY_MAX_WAITING_WORKERS: String(limit), RELAY_ADMISSION_POLL_MS: '20', RELAY_JOBS_ACK_TIMEOUT_MS: '200' } }); + ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root, 'admission'); + ctx.env.RELAY_TEST_ADMISSION_READINGS = path.join(ctx.root, 'readings.json'); + const machine = { totalBytes: 18 * 2**30, availableBytes: 16 * 2**30, cores: 12, load5: 0 }; + fs.writeFileSync(ctx.env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify(machine)); + let now = Date.now(); + const deps = { now: () => now, readMachine: () => machine, scanUnslotted: () => [], env: ctx.env }; + return { ctx, deps, tick: (ms) => now += ms, machine }; +} +function job(s, id = 'job-window', state = 'starting') { + const dir = path.join(s.ctx.jobs, 'repo', id); fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(jobPaths(dir).brief, '[quick] Dummy job.'); + writeJob(dir, { id, kind: 'task', state, createdAt: new Date().toISOString(), pendingFirstTurn: true, attempts: [], + request: { backend: 'codex', cwd: s.ctx.work, model: 'synthetic-model', effort: 'low', notes: [] } }); + return dir; +} + +test('gate reserves Q places, binds once, converts without double counting, and expires after 45 seconds', t => { + const s = domain(t), a = reserveWaitingPlace(s.ctx.env, s.deps), b = reserveWaitingPlace(s.ctx.env, s.deps); + assert.ok(a.pass && b.pass); assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, false); + assert.equal(bindReservation(a.reservation, 'a-first', s.ctx.env, s.deps), true); + assert.equal(readReservation('a-first', s.ctx.env), a.reservation); + assert.equal(bindReservation(a.reservation, 'a-other', s.ctx.env, s.deps), false); + const dir = job(s); + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => 123, s.deps, { reservation: a.reservation }), 123); + const files = fs.readdirSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots')); + assert.equal(files.length, 2); assert.ok(files.includes('job-window.json')); + writeJob(dir, { ...readJob(dir), state: 'completed' }); + s.tick(D.reservationMs + 1); + assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, true); +}); + +test('guard-off gate passes without reading or writing admission storage', t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_ADMISSION_HOME = '/dev/null/unwritable'; + assert.deepEqual(reserveWaitingPlace(s.ctx.env, s.deps), { pass: true, reservation: null }); + assert.equal(companion(s.ctx, ['gate']).stdout, 'pass\n'); +}); + +test('window refusal exits 6 and readers never resurrect its brief after capacity frees', t => { + const s = domain(t, 0); + const result = companion(s.ctx, ['start', '--prompt-file', promptFile(s.ctx)]); + assert.equal(result.status, 6, result.stderr + result.stdout); + assert.match(result.stdout, /state: not started.*\nerror: Not started: waiting worker limit: 0 of 0/); + const dir = jobDirs(s.ctx)[0], refused = readJob(dir); + assert.equal(refused.state, 'failed'); assert.equal(refused.admission.state, 'not-started'); assert.equal(hasLiveWorker(dir), false); + assert.equal(fs.existsSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `${refused.id}.json`)), false); + s.ctx.env.RELAY_MAX_WAITING_WORKERS = '4'; + for (const argv of [['status', refused.id], ['result', refused.id], ['wait', refused.id, '--timeout-seconds', '1'], ['status', '--json', '--all']]) { + const read = companion(s.ctx, argv); assert.doesNotMatch(read.stderr, /spawn|launch failed/); + } + assert.equal(readJob(dir).attempts.length, 0); +}); + +test('pressure clearing inside the start window acknowledges a real live worker lease', async t => { + const s = domain(t), dir = job(s); s.machine.availableBytes = 0; + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + let attempts = 0; + const result = await startWithinWindow(dir, s.ctx.work, s.ctx.env, { windowMs: 200, spawn: () => { + attempts++; + if (attempts === 1) return spawnWithAdmission(dir, s.ctx.env, () => assert.fail('pressure'), s.deps); + s.machine.availableBytes = 16 * 2**30; + assert.equal(acquireWorkerLease(dir, me), true); + } }); + assert.equal(result.accepted, true); assert.equal(hasLiveWorker(dir), true); releaseWorkerLease(dir, me); +}); + +test('a killed launcher is dead rather than an ownerless durable queue', async t => { + const s = domain(t, 0); s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '60000'; + const child = spawn(process.execPath, [path.join(ROOT, 'plugins/relay/scripts/companion.mjs'), 'start', '--prompt-file', promptFile(s.ctx)], { cwd: s.ctx.work, env: s.ctx.env, stdio: 'ignore' }); + t.after(() => { try { child.kill('SIGKILL'); } catch {} }); + const dir = await waitFor(() => jobDirs(s.ctx).find(dir => readJob(dir).admission?.launcher)); + const ended = once(child, 'exit'); child.kill('SIGKILL'); await ended; + const view = computeJobView(readJob(dir)); assert.equal(view.state, 'dead'); assert.match(view.deadReason, /process starting it ended/); +}); + +test('a completed job continuation without a place is withdrawn and never runs later', async t => { + const s = domain(t, 0), dir = job(s, 'job-completed', 'completed'); + const message = postInboxMessage(dir, { kind: 'continue', text: 'Never run this.' }); + const relay = path.join(jobPaths(dir).inbox, 'relay'); fs.mkdirSync(relay); + fs.writeFileSync(path.join(relay, 'entry.json'), JSON.stringify({ messageId: message.id })); + const result = await awaitOutcome(dir, message.id, { env: s.ctx.env, timeoutMs: 100 }); + assert.equal(result.consumer, false); assert.equal(result.outcome.action, 'not-started'); + assert.ok(fs.existsSync(path.join(dir, 'withdrawn', `${message.id}.json`))); + assert.equal(fs.existsSync(path.join(relay, 'entry.json')), false); + assert.deepEqual(messageAccount(dir).undelivered, []); + s.ctx.env.RELAY_MAX_WAITING_WORKERS = '4'; + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => assert.fail('withdrawn message ran'), s.deps), null); +}); + +test('a worker claim beating withdrawal delivers once rather than refusing the claimed message', async t => { + const s = domain(t, 0), dir = job(s, 'job-race', 'completed'); + const msg = postInboxMessage(dir, { kind: 'continue', text: 'Once.' }); + const rename = fs.renameSync; let claims = 0; + t.mock.method(fs, 'renameSync', (from, to) => { + if (to === path.join(dir, 'withdrawn', `${msg.id}.json`)) { + claims += claimInboxMessages(dir).length; + writeOutcome(dir, msg.id, { action: 'turn-started', turnId: 'synthetic-turn' }); + } + return rename(from, to); + }); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, timeoutMs: 0 }); + assert.equal(result.consumer, true); assert.equal(result.outcome.action, 'turn-started'); assert.equal(claims, 1); +}); + +test('recovery with no place preserves dead state and retries the same reserved message', async t => { + const s = domain(t, 0), dir = job(s, 'job-recovery', 'running'); + writeJob(dir, { ...readJob(dir), worker: { pid: 2147483647, startTime: 'gone' } }); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), false); + const first = readJob(dir); assert.equal(computeJobView(first).state, 'dead'); assert.equal(first.automaticResumes[0].state, 'reserved'); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), false); + assert.equal(readJob(dir).automaticResumes[0].id, first.automaticResumes[0].id); + assert.equal(fs.readdirSync(jobPaths(dir).inbox).filter(n => n.endsWith('.json')).length, 1); +}); + +test('33 gated submissions reach backend admission with no refused or unowned work', t => { + const s = domain(t, 20), waiting = [], running = [], me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + let submitted = 0, admitted = 0, held = 0; + while (admitted < 33) { + while (submitted < 33) { + const place = reserveWaitingPlace(s.ctx.env, s.deps); + if (!place.pass) { held++; break; } + const dir = job(s, `job-floor-${++submitted}`); + const request = { ...readJob(dir).request, model: submitted <= 30 ? 'synthetic-sol' : 'synthetic-astra', effort: 'high' }; + writeJob(dir, { ...readJob(dir), request }); + assert.equal(spawnWithAdmission(dir, s.ctx.env, () => 1, s.deps, { reservation: place.reservation }), 1); + // A synthetic worker owns each waiting slot while the fake backend is gated. + const file = path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `${readJob(dir).id}.json`); + const slot = JSON.parse(fs.readFileSync(file)); fs.writeFileSync(file, JSON.stringify({ ...slot, pid: me.pid, processStart: me.startTime, state: 'waiting' })); + waiting.push({ home: s.ctx.env.RELAY_ADMISSION_HOME, jobDir: dir, jobId: readJob(dir).id, me, request }); + } + for (const ctx of [...waiting]) { + const result = admit(ctx, s.deps); ctx.admission = result.record; ctx.waitingSince = result.record.since; + if (result.admit) { waiting.splice(waiting.indexOf(ctx), 1); running.push(ctx); admitted++; } + } + s.tick(7000); + // Fake remote work uses little resident memory; release startup holds naturally. + assert.ok(admitted + waiting.length === submitted); + } + assert.equal(submitted, 33); assert.ok(held > 0); assert.equal(running.length, 33); + for (const ctx of running) writeJob(ctx.jobDir, { ...readJob(ctx.jobDir), state: 'completed' }); +}); + +test('foreground task uses a detached worker rather than the waiting caller', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; + const script = path.join(ROOT, 'plugins/relay/scripts/companion.mjs'); + const caller = spawn(process.execPath, [script, 'task', '[quick] Dummy.'], { cwd: s.ctx.work, env: s.ctx.env, stdio: 'ignore' }); + const exited = once(caller, 'exit'); + t.after(() => { try { caller.kill('SIGKILL'); } catch {} }); + const dir = await waitFor(() => jobDirs(s.ctx).find(dir => readJob(dir).worker?.pid)); + assert.notEqual(readJob(dir).worker.pid, caller.pid); assert.equal(hasLiveWorker(dir), true); + await exited; +}); + +test('a continuation interrupted at handoff is named as owed on the finished job and can retry', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; + const started = companion(s.ctx, ['start', '--prompt-file', promptFile(s.ctx), '--json']); + assert.equal(started.status, 0, started.stdout + started.stderr); + const id = JSON.parse(started.stdout).jobId; + assert.equal(companion(s.ctx, ['wait', id, '--timeout-seconds', '15']).status, 0); + const dir = jobDirs(s.ctx)[0]; + await waitFor(() => !hasLiveWorker(dir)); + const msg = postInboxMessage(dir, { kind: 'continue', text: '[quick] Interrupted handoff.' }); + fs.writeFileSync(path.join(dir, 'launch-interrupted.json'), JSON.stringify({ state: 'handoff-failed', requestedAt: new Date(Date.now() + 1).toISOString(), worker: { pid: 2147483647, startTime: 'gone' } })); + const view = loadJobView(dir); assert.ok(view.undelivered.includes(msg.id)); + const block = companion(s.ctx, ['result', id, '--state-only']); assert.match(block.stdout, new RegExp(msg.id)); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, wait: false, timeoutMs: 5000 }); + assert.equal(result.consumer, true); + assert.equal(companion(s.ctx, ['wait', id, '--timeout-seconds', '15']).status, 0); + assert.equal(readJob(dir).attempts.filter(a => a.messageId === msg.id).length, 1); +}); + +test('a message racing warm-worker exit redispatches promptly after the lease disappears', async t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION = 'off'; s.ctx.env.RELAY_JOBS_ACK_TIMEOUT_MS = '5000'; s.ctx.env.RELAY_ADMISSION_POLL_MS = '3000'; + const dir = job(s, 'job-warm-exit', 'completed'); + const leaseModule = new URL('../plugins/relay/scripts/lib/jobs/lease.mjs', import.meta.url).href; + const processModule = new URL('../plugins/relay/scripts/lib/util/process.mjs', import.meta.url).href; + const warm = spawn(process.execPath, ['--input-type=module', '-e', `import {acquireWorkerLease,releaseWorkerLease} from ${JSON.stringify(leaseModule)}; +import {readProcessStartTime} from ${JSON.stringify(processModule)}; +const dir=${JSON.stringify(dir)}, me={pid:process.pid,startTime:readProcessStartTime(process.pid)}; +acquireWorkerLease(dir,me); console.log(JSON.stringify(me)); +setTimeout(()=>releaseWorkerLease(dir,me),100); setTimeout(()=>process.exit(0),300);`], { cwd: s.ctx.work, env: s.ctx.env, stdio: ['ignore', 'pipe', 'inherit'] }); + t.after(() => { try { warm.kill('SIGKILL'); } catch {} }); + const identity = JSON.parse(String((await once(warm.stdout, 'data'))[0])); + fs.writeFileSync(path.join(dir, 'launch-warm.json'), JSON.stringify({ state: 'acknowledged', worker: identity, requestedAt: new Date().toISOString() })); + writeJob(dir, { ...readJob(dir), worker: identity }); + const msg = postInboxMessage(dir, { kind: 'continue', text: '[quick] Warm-exit message.' }); + const begin = Date.now(); + const result = await awaitOutcome(dir, msg.id, { env: s.ctx.env, timeoutMs: 5000 }); + assert.equal(result.consumer, true); assert.match(result.outcome?.action, /^(queued-next-turn|turn-started)$/); assert.notEqual(readJob(dir).worker?.pid, identity.pid); assert.ok(Date.now() - begin < 2800, 'redispatch must not wait for the old five-second timer'); +}); + +test('a dispatch arriving after refusal cannot acquire the worker lease or run the brief', async t => { + const s = domain(t, 0), dir = job(s); + await startWithinWindow(dir, s.ctx.work, s.ctx.env, { windowMs: 0 }); + assert.equal(readJob(dir).admission.state, 'not-started'); + const me = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + assert.equal(acquireWorkerLease(dir, me), false); + assert.equal(hasLiveWorker(dir), false); + const explicit = postInboxMessage(dir, { kind: 'resume', text: '[quick] Submit again explicitly.' }); + assert.equal(acquireWorkerLease(dir, me), true); releaseWorkerLease(dir, me); + // The assertion fixture has no actual worker; close the explicit message for teardown. + claimInboxMessages(dir); writeOutcome(dir, explicit.id, { action: 'skipped' }); +}); + +test('recovery converts an adopting relay reservation rather than blocking behind it', async t => { + const s = domain(t, 1); s.ctx.env.RELAY_ADMISSION_POLL_MS = '1000'; + const dir = job(s, 'job-reserved-recovery', 'running'); + writeJob(dir, { ...readJob(dir), worker: { pid: 2147483647, startTime: 'gone' }, relay: { agentId: 'a-adopted' } }); + const place = reserveWaitingPlace(s.ctx.env, s.deps); + assert.equal(bindReservation(place.reservation, 'a-adopted', s.ctx.env, s.deps), true); + assert.equal(reserveWaitingPlace(s.ctx.env, s.deps).pass, false); + assert.equal(await resumeDeadJobOnce(dir, s.ctx.env), true); + assert.equal(fs.existsSync(path.join(s.ctx.env.RELAY_ADMISSION_HOME, 'slots', `r-${place.reservation}.json`)), false); + await waitFor(() => readJob(dir).state === 'completed' && !hasLiveWorker(dir)); +}); + +test('a guard error is an error from the gate so hooks can fall back to bounded start', t => { + const s = domain(t); s.ctx.env.RELAY_ADMISSION_HOME = '/dev/null/unwritable'; + const result = companion(s.ctx, ['gate', '--wait-seconds', '30']); + assert.equal(result.status, 1); assert.match(result.stderr, /guard unavailable/); assert.equal(result.stdout, ''); +}); + +test('a plain resume with no job bypasses the retry gate even if no waiting place is free', t => { + const s = domain(t, 0); + const transcript = path.join(s.ctx.root, 'subagents', 'agent-a-refused.jsonl'); + fs.mkdirSync(path.dirname(transcript)); fs.writeFileSync(transcript, ''); + const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-refused', '--transcript', transcript, '--wait-seconds', '30']); + assert.equal(result.status, 0); assert.equal(result.stdout, 'pass\n'); +}); + + +test('an unclassified retry with no transcript waits for capacity instead of bypassing the gate', t => { + const s = domain(t, 0); + const transcript = path.join(s.ctx.root, 'subagents', 'workflows', 'wf-test', 'agent-a-unknown.jsonl'); + const result = companion(s.ctx, ['gate', '--retry', '--agent', 'a-unknown', '--transcript', transcript]); + assert.equal(result.status, 0); assert.match(result.stdout, /^hold waiting worker limit: 0 of 0/); +}); diff --git a/tests/grok-delivery.test.mjs b/tests/grok-delivery.test.mjs index 84f21254..9628f26f 100644 --- a/tests/grok-delivery.test.mjs +++ b/tests/grok-delivery.test.mjs @@ -430,7 +430,8 @@ import { buildLaunchRequest } from "../plugins/relay/scripts/lib/jobs/request.mj } else { await waitFor(() => companion(ctx, ['status', id]).stdout.includes('state: dead')); } - assert.equal(companion(ctx, ['resume', id, '--message', '[quick] Recover.']).status, 0); + const resumed = companion(ctx, ['resume', id, '--message', '[quick] Recover.']); + assert.equal(resumed.status, 0, resumed.stdout + resumed.stderr); await finishJob(ctx, id); const after = readJob(ctx, id); assert.equal(after.sessionId, before.sessionId); diff --git a/tests/harness-workflow.test.mjs b/tests/harness-workflow.test.mjs index 2f34d3ab..12e478cd 100644 --- a/tests/harness-workflow.test.mjs +++ b/tests/harness-workflow.test.mjs @@ -84,3 +84,115 @@ test("readWorkflowRunId returns null for absent or unreadable transcripts and ab } assert.equal(readWorkflowRunId(transcript([WORKFLOW_RESULT]), undefined), null); }); + +// Exercise the module's admission hooks with the host API faked, including retries +// that enter the first step without another agent.spawn event. +import { register as registerHooks } from 'node:module'; +const modStub = 'data:text/javascript,' + encodeURIComponent('export const atom = (key, initial) => ({key, initial}); export const read = async () => null; export const update = async () => {};'); +registerHooks('data:text/javascript,' + encodeURIComponent(`export async function resolve(s, c, next) { return s === 'claude-code' ? {url: ${JSON.stringify(modStub)}, shortCircuit: true} : next(s, c); }`)); +const { register } = await import('../plugins/relay/hooks/mod/register.js'); +function hooks(answers) { + const handlers = {}, catches = {}, runs = [], calls = []; + register((event, filter, handler) => { handlers[event] = handler ?? filter; return { catch(fn) { catches[event] = fn; return this; } }; }); + const $ = { plugin: { root: '/synthetic/plugin' }, session: { cwd: async () => '/synthetic/work' }, + process: { run: async (argv, init) => { + runs.push({ argv, init }); + if (argv[2] === 'check-header') return { exitCode: 0, stdout: 'ok\n' }; + if (argv.includes('--bind')) return { exitCode: 0, stdout: 'bound\n' }; + const answer = answers.shift(); if (answer instanceof Error) throw answer; + return { exitCode: 0, stdout: answer ?? 'pass\n' }; + } } }; + const next = async e => { calls.push(e); return { agentId: 'a-new' }; }; + return { handlers, catches, runs, calls, $, next }; +} +const spawnEvent = { subagentType: 'relay:job', prompt: 'model: synthetic\neffort: low\n---\nDummy.' }; + +test('Workflow spawn holds through host waits and binds its reservation before the wrapper runs', async () => { + const id = '00000000-0000-4000-8000-000000000001'; + const h = hooks(['hold waiting worker limit: 2 of 2', 'hold waiting worker limit: 2 of 2', `pass ${id}`]); + assert.deepEqual(await h.handlers['agent.spawn'](h.$, { ...spawnEvent, workflow: { runId: 'wf-one', agentIndex: 1 } }, h.next), { agentId: 'a-new' }); + assert.equal(h.calls.length, 1); + const gate = h.runs.filter(r => r.argv[2] === 'gate'); assert.equal(gate.length, 4); + assert.equal(gate[0].init.timeoutMs, 600000); assert.ok(gate[0].argv.includes('540')); + assert.deepEqual(gate.at(-1).argv.slice(3), ['--bind', id, '--agent', 'a-new']); + assert.equal(typeof h.catches['agent.spawn'], 'function'); +}); +test('plain spawns wait at most 30 seconds then deny with the capacity reason', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2']); + const result = await h.handlers['agent.spawn'](h.$, spawnEvent, h.next); + assert.match(result.deny, /relay is at capacity: waiting worker limit: 2 of 2/); assert.match(result.deny, /Workflow/); + assert.equal(h.calls.length, 0); assert.ok(h.runs.at(-1).argv.includes('30')); +}); +test('a gate crash falls back to the bounded start, while cancellation is denied by catch', async () => { + const h = hooks([new Error('synthetic gate crash')]); + await h.handlers['agent.spawn'](h.$, spawnEvent, h.next); assert.equal(h.calls.length, 1); + const cancelled = hooks([new Error('process.run aborted')]); + await assert.rejects(cancelled.handlers['agent.spawn'](cancelled.$, spawnEvent, cancelled.next), /aborted/); + assert.ok(cancelled.catches['agent.spawn'](cancelled.$, spawnEvent, { called: false }).deny); assert.equal(cancelled.calls.length, 0); +}); +test('resume spawns do not take another waiting place', async () => { + const h = hooks([]); await h.handlers['agent.spawn'](h.$, { ...spawnEvent, resume: { agentId: 'a-old' } }, h.next); + assert.equal(h.runs.length, 0); assert.equal(h.calls.length, 1); +}); +test('a Workflow stall retry is gated at its first step even without agent.spawn', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2', 'pass']); + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-retry', transcript_path: '/synthetic/session.jsonl' }, h.next); + h.calls.length = 0; + const step = h.handlers['turn.step'](h.$, { index: 0, agentId: 'a-retry', turnId: 'turn-retry' }, async function* (e) { return await h.next(e); }); + await step.next(); + assert.equal(h.calls.length, 1); assert.equal(h.runs.length, 2); assert.ok(h.runs.every(r => r.argv.includes('--retry') && r.argv.includes('a-retry'))); +}); + +test('admission catches preserve a delegated result and never restart an already delegated step', async () => { + const h = hooks([]), result = { agentId: 'a-delegated' }; + let calls = 0; + const next = async () => { calls++; return result; }; next.called = true; + assert.equal(await h.catches['agent.spawn'](h.$, spawnEvent, next), result); assert.equal(calls, 1); + const step = async function* () { throw new Error('synthetic model API error'); }; step.called = true; + const caught = h.catches['turn.step'](h.$, { index: 0, turnId: 't-error' }, step); + await assert.rejects(caught.next(), /synthetic model API error/); +}); +test('a SendMessage resume skips the first-step gate even when its initial start had no job', async () => { + const h = hooks(['hold waiting worker limit: 2 of 2']); + await h.handlers['agent.spawn'](h.$, { ...spawnEvent, resume: { agentId: 'a-resumed' } }, h.next); + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-resumed' }, h.next); + const step = h.handlers['turn.step'](h.$, { index: 0, agentId: 'a-resumed', turnId: 't-resumed' }, async function* (e) { return await h.next(e); }); + await step.next(); assert.equal(h.runs.length, 0); +}); + +for (const event of ['agent.spawn', 'turn.step']) test(`${event} gate storage uncertainty reaches bounded start and cannot admit a worker`, async t => { + const { setup, companion, promptFile, jobDirs } = await import('./support/job-helpers.mjs'); + const { readJob } = await import('../plugins/relay/scripts/lib/jobs/store.mjs'); + const { hasLiveWorker } = await import('../plugins/relay/scripts/lib/worker/delivery.mjs'); + const ctx = setup(t, { env: { RELAY_ADMISSION: 'on', RELAY_ADMISSION_HOME: '/dev/null/unwritable', RELAY_JOBS_ACK_TIMEOUT_MS: '100', RELAY_ADMISSION_POLL_MS: '20' } }); + const h = hooks([]); + const gateResults = []; + h.$.process.run = async argv => { + if (argv[2] === 'check-header') return { exitCode: 0, stdout: 'ok\n' }; + const result = companion(ctx, argv.slice(2)); gateResults.push(result); + return { exitCode: result.status, stdout: result.stdout, stderr: result.stderr }; + }; + let starts = 0; + const boundedStart = () => { + starts++; + const result = companion(ctx, ['start', '--prompt-file', promptFile(ctx)]); + assert.equal(result.status, 6, result.stdout + result.stderr); + assert.match(result.stdout, /state: not started/); + return { agentId: 'a-refused' }; + }; + if (event === 'agent.spawn') await h.handlers[event](h.$, { ...spawnEvent, workflow: { runId: 'wf-storage' } }, boundedStart); + else { + await h.handlers['classic.SubagentStart'](h.$, { agent_type: 'relay:job', agent_id: 'a-unknown', transcript_path: '/missing/workflows/wf-storage/agent-a-unknown.jsonl' }, h.next); + const step = h.handlers[event](h.$, { index: 0, agentId: 'a-unknown', turnId: 't-storage' }, async function* () { return boundedStart(); }); + await step.next(); + } + assert.equal(starts, 1); + assert.equal(gateResults.length, 1); + assert.equal(gateResults[0].status, 1); + assert.match(gateResults[0].stderr, /guard unavailable/); + for (const dir of jobDirs(ctx)) { + assert.equal(readJob(dir).admission.state, 'not-started'); + assert.equal(hasLiveWorker(dir), false); + assert.equal(readJob(dir).attempts.length, 0); + } +}); diff --git a/tests/message-account.test.mjs b/tests/message-account.test.mjs index 7e128223..7540080b 100644 --- a/tests/message-account.test.mjs +++ b/tests/message-account.test.mjs @@ -85,7 +85,7 @@ test("cancelled with a pending continuation exits five", () => { test("worker and ownership share identity decisions and keep private helpers private", async () => { const modules = makeTempDir("relay-identity-check-"); - for (const file of ["jobs/store.mjs", "jobs/lease.mjs", "util/ownership.mjs", "util/errors.mjs", "util/fs.mjs"]) { + for (const file of ["jobs/store.mjs", "jobs/lease.mjs", "jobs/messages.mjs", "util/ownership.mjs", "util/errors.mjs", "util/fs.mjs"]) { fs.mkdirSync(path.dirname(path.join(modules, file)), { recursive: true }); fs.copyFileSync(new URL(`../plugins/relay/scripts/lib/${file}`, import.meta.url), path.join(modules, file)); } diff --git a/tests/message-storage.test.mjs b/tests/message-storage.test.mjs new file mode 100644 index 00000000..5d4cfe88 --- /dev/null +++ b/tests/message-storage.test.mjs @@ -0,0 +1,76 @@ +import './support/isolate-env.mjs'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { setup } from './support/job-helpers.mjs'; +import { createJobDir, readJob, writeJob, jobPaths } from '../plugins/relay/scripts/lib/jobs/store.mjs'; +import { readLaunchPrompt } from '../plugins/relay/scripts/lib/jobs/request.mjs'; +import { messageAccount, postInboxMessage } from '../plugins/relay/scripts/lib/jobs/messages.mjs'; +import { readLease } from '../plugins/relay/scripts/lib/jobs/lease.mjs'; +import { isProcessIdentityAlive } from '../plugins/relay/scripts/lib/util/process.mjs'; +import { runWorker } from '../plugins/relay/scripts/lib/worker/worker.mjs'; + +function fixture(t) { + const ctx = setup(t, { turnMs: 50, env: { RELAY_ADMISSION: 'off', RELAY_WORKER_SCOPE: 'off' } }); + const { jobDir } = createJobDir(ctx.work, 'job-storage', ctx.env); + const { request, brief } = readLaunchPrompt('model: gpt-test\neffort: low\n---\n[quick] Finish.', { baseDir: ctx.work, env: ctx.env }); + writeJob(jobDir, { id: 'job-storage', kind: 'task', state: 'starting', pendingFirstTurn: true, attempts: [], request }); + fs.writeFileSync(jobPaths(jobDir).brief, brief); + return { ctx, jobDir, paths: jobPaths(jobDir) }; +} + +for (const code of ['EACCES', 'EIO']) for (const stage of ['inbox', 'claimed', 'handoff', 'released']) { + test(`${code} reading ${stage} during worker finalization fails explicitly and preserves owed work`, async t => { + const { ctx, jobDir, paths } = fixture(t); + const readdir = fs.readdirSync; + let armed = false, injected = false, message, completionReads = 0; + const mock = t.mock.method(fs, 'readdirSync', function (directory, ...args) { + const job = readJob(jobDir); + if (String(directory) === paths.inbox && job.state === 'completed') completionReads++; + const finalizing = stage === 'released' ? completionReads >= 2 : stage === 'handoff' ? completionReads >= 1 : job.attempts?.at(-1)?.state === 'completed'; + if (!armed && finalizing) { + message = postInboxMessage(jobDir, { kind: 'continue', text: 'Owed follow-up.' }); + if (stage === 'claimed') { + fs.mkdirSync(paths.claimed, { recursive: true }); + fs.renameSync(path.join(paths.inbox, `${message.id}.json`), path.join(paths.claimed, `${message.id}.json`)); + } + armed = true; + } + if (armed && String(directory) === (stage === 'claimed' ? paths.claimed : paths.inbox)) { + injected = true; + throw Object.assign(new Error('synthetic message storage failure'), { code }); + } + return readdir.call(this, directory, ...args); + }); + let rejected; + try { await runWorker(jobDir, { env: ctx.env }); } catch (error) { rejected = error; } + finally { mock.mock.restore(); } + assert.equal(injected, true); + const job = readJob(jobDir); + assert.equal(job.state, 'failed'); + assert.match(job.errorMessage, new RegExp(`storage uncertain:.*${code}`)); + if (rejected) assert.match(rejected.message, /storage uncertain/); + assert.deepEqual(messageAccount(jobDir).undelivered, [message.id]); + assert.equal(readLease(jobDir), null); + assert.equal(ctx.fakeState().appServerStarts, 1); + assert.equal(isProcessIdentityAlive(job.server), false); + }); +} + +test('absent message directories allow an empty worker to complete', async t => { + const { ctx, jobDir, paths } = fixture(t); + const readdir = fs.readdirSync; + let injected = false; + const mock = t.mock.method(fs, 'readdirSync', function (directory, ...args) { + if ([paths.inbox, paths.claimed].includes(String(directory)) && readJob(jobDir).attempts?.at(-1)?.state === 'completed') { + injected = true; + throw Object.assign(new Error('absent directory'), { code: 'ENOENT' }); + } + return readdir.call(this, directory, ...args); + }); + try { await runWorker(jobDir, { env: ctx.env }); } finally { mock.mock.restore(); } + assert.equal(injected, true); + assert.equal(readJob(jobDir).state, 'completed'); + assert.equal(readLease(jobDir), null); +}); diff --git a/tests/process.test.mjs b/tests/process.test.mjs index cd350f71..089c2d14 100644 --- a/tests/process.test.mjs +++ b/tests/process.test.mjs @@ -304,3 +304,15 @@ test("Darwin cleanup of an otherwise empty worker group finds no phantom sampler const result = await stopExecution({ execution: { leader: identity, pgid } }, { listProcesses: list, graceMs: 0 }); assert.deepEqual(result.survivors, []); }); + +test('a busy process-cache lock never supplies stale rows to a fresh execution read', async t => { + const { readProcessTableCached } = await import('../plugins/relay/scripts/lib/util/process.mjs'); + const { acquireOwnership, releaseOwnership } = await import('../plugins/relay/scripts/lib/util/ownership.mjs'); + const home = makeTempDir('relay-fresh-cache-'), file = path.join(home, '.ps-table.json'); + fs.writeFileSync(file, JSON.stringify({ at: Date.now() - 60000, rows: [] })); + assert.equal(Boolean(acquireOwnership(`${file}.lock`)), true); + t.after(() => releaseOwnership(`${file}.lock`)); + const rows = [{ pid: 123, startTime: 'synthetic-live' }]; let reads = 0; + for (const maxAgeMs of [0, 5000]) assert.deepEqual(readProcessTableCached({ home, maxAgeMs, list: () => { reads++; return rows; } }), rows); + assert.equal(reads, 2); +}); diff --git a/tests/queued.test.mjs b/tests/queued.test.mjs index 3e1bffdc..22e37522 100644 --- a/tests/queued.test.mjs +++ b/tests/queued.test.mjs @@ -4,8 +4,8 @@ import test from "node:test"; import { computeJobView, exitCodeFor, isTerminalView, jobViewJson, renderJobBlock } from "../plugins/relay/scripts/lib/jobs/view.mjs"; import { readProcessStartTime } from "../plugins/relay/scripts/lib/util/process.mjs"; -test("durable queued jobs without a worker stay pending, with reason and since", () => { - const job = { id: "job-queued", state: "queued", createdAt: "2025-01-01T00:00:00.000Z", admission: { state: "waiting", waitKind: "worker", reason: "memory low", since: "2025-01-01T00:00:00.000Z" } }; +test("workerless queued jobs stay pending only while their launcher lives", () => { + const job = { id: "job-queued", state: "queued", createdAt: "2025-01-01T00:00:00.000Z", admission: { launcher: { pid: process.pid, startTime: readProcessStartTime(process.pid) }, state: "waiting", waitKind: "worker", reason: "memory low", since: "2025-01-01T00:00:00.000Z" } }; const view = { ...computeJobView(job), job, hasReport: false, paths: { brief: "/synthetic/absent" }, jobDir: "/synthetic/job-queued" }; assert.equal(view.state, "queued"); assert.equal(view.stalled, false); @@ -20,3 +20,8 @@ test("resident queued jobs never stall; dead queued workers remain recoverable", assert.equal(computeJobView(job).stalled, false); assert.equal(computeJobView({ ...job, worker: { pid: 2147483647 } }).state, "dead"); }); + +test("legacy workerless queues are dead with an explicit no-dispatcher reason", () => { + const view = computeJobView({ state: "queued", admission: { waitKind: "worker" } }); + assert.equal(view.state, "dead"); assert.match(view.deadReason, /older relay with nobody to start it/); +}); diff --git a/tests/recovery.test.mjs b/tests/recovery.test.mjs index f02b8ebe..842debf7 100644 --- a/tests/recovery.test.mjs +++ b/tests/recovery.test.mjs @@ -189,9 +189,49 @@ for (const persisted of [false, true]) { // Delivering a message into the automatic turn earns a fresh retry too. assert.equal(companion(ctx, ["steer", id, "--message", "User correction"]).status, 0); killWorker(ctx, id); - await waitFor(() => readJobJson(ctx, id).automaticResumes.length === 3 && readJobJson(ctx, id).turnId); + await waitFor(() => turnRecorded(ctx, id, 5)); + assert.equal(readJobJson(ctx, id).automaticResumes.length, 3); companion(ctx, ["cancel", id]); - assert.equal((await waitingAgain).status, 5); + const cancelled = await waitingAgain; + assert.equal(cancelled.status, 5, cancelled.stdout + cancelled.stderr); + }); + + test("a delivered steer rearms recovery before its receipt can be observed", async t => { + const ctx = relayContext("job", { turnMs: 30000 }); + const marker = path.join(ctx.home, "receipt-published"), release = path.join(ctx.home, "release-receipt"); + const preload = path.join(ctx.home, "freeze-after-receipt.mjs"); + fs.writeFileSync(preload, `import fs from "node:fs"; +const rename = fs.renameSync; +fs.renameSync = function (from, to, ...args) { + const result = rename.call(this, from, to, ...args); + if (process.argv[2] === "worker" && String(to).includes("/inbox/outcomes/") && fs.readFileSync(to, "utf8").includes('"action": "steered"')) { + fs.writeFileSync(${JSON.stringify(marker)}, ""); + while (!fs.existsSync(${JSON.stringify(release)})) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5); + } + return result; +};`); + ctx.env.NODE_OPTIONS = `${ctx.env.NODE_OPTIONS ?? ""} --import ${preload}`; + t.after(() => fs.writeFileSync(release, "")); + const id = await start(ctx); + killWorker(ctx, id); + const waiting = waitAsync(ctx, await waitInput(ctx, id)); + t.after(async () => { + fs.writeFileSync(release, ""); + companion(ctx, ["cancel", id]); + await waiting; + }); + await waitFor(() => turnRecorded(ctx, id, 2)); + const sent = companion(ctx, ["steer", id, "--message", "Durable user correction"]); + assert.equal(sent.status, 0, sent.stdout + sent.stderr); + await waitFor(() => fs.existsSync(marker)); + const messageId = /^message: steer (.+)$/m.exec(sent.stdout)[1]; + assert.equal(readJobJson(ctx, id).recoveryEpoch, messageId, "the receipt cannot precede its retry allowance"); + killWorker(ctx, id); + await waitFor(() => turnRecorded(ctx, id, 3)); + assert.equal(readJobJson(ctx, id).automaticResumes.length, 2); + companion(ctx, ["cancel", id]); + const cancelled = await waiting; + assert.equal(cancelled.status, 5, cancelled.stdout + cancelled.stderr); }); test("two concurrent relay waits reserve only one automatic resume", async () => { @@ -698,7 +738,7 @@ for (const persisted of [false, true]) { assert.equal(fs.existsSync(path.join(ctx.binDir, "fake-codex-state.json")), false); }); - test("automatic recovery's lease is no wait-false consumer", async t => { + test("automatic recovery's lease cannot accept a message before its worker owns it", async t => { const marker = path.join(makeTempDir("relay-recovery-marker-"), "term"); const { identity } = await oldServer(t, { ignoreTerm: true, marker }); const ctx = sweepJob({ server: identity, servers: [identity] }); @@ -726,11 +766,11 @@ for (const persisted of [false, true]) { const message = postInboxMessage(ctx.jobDir, { kind: "continue", text: "[quick] Arrived during recovery" }); const delivered = await awaitOutcome(ctx.jobDir, message.id, { env, wait: false, timeoutMs: 50 }); assert.equal(delivered.consumer, false); - assert.equal(delivered.outcome, null); + assert.equal(delivered.outcome.action, "not-started"); assert.equal(readOutcome(ctx.jobDir, message.id), null); await recovering; - await waitFor(() => readOutcome(ctx.jobDir, message.id)?.action === "turn-started" && !readLease(ctx.jobDir)); - assert.equal(ctx.fakeState().turnStarts.filter(turn => turn.prompt === message.text).length, 1); + await waitFor(() => !readLease(ctx.jobDir)); + assert.equal(ctx.fakeState().turnStarts.filter(turn => turn.prompt === message.text).length, 0); } finally { await recovering; } @@ -870,3 +910,40 @@ for (const gone of ["removed", "dead", "reused pid"]) { assert.equal(readJob(ctx, id).automaticResumes.length, 1); }); } + + +for (const winner of ["cancel", "successor"]) test(`recovery refusal preserves a ${winner} published before its final lease`, async t => { + const ctx = setup(t, { env: { RELAY_ADMISSION: "on", RELAY_MAX_WAITING_WORKERS: "0", RELAY_ADMISSION_POLL_MS: "20" } }); + ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root, "admission"); + const { jobDir } = createJobDir(ctx.work, "job-recovery-fence", ctx.env); + const original = { id: "job-recovery-fence", kind: "task", state: "running", pendingFirstTurn: true, attempts: [], + worker: { pid: 2147483647, startTime: "gone" }, + request: buildLaunchRequest({ model: "gpt-test", effort: "low", cd: ctx.work }, { env: ctx.env, baseDir: ctx.work }) }; + writeJob(jobDir, original); + fs.writeFileSync(path.join(jobDir, "brief.md"), "Never start under full capacity."); + let successor; + if (winner === "successor") { + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + await once(child, "spawn"); + successor = { pid: child.pid, startTime: readProcessStartTime(child.pid) }; + t.after(() => child.kill("SIGKILL")); + } + const directory = ownershipDir(path.join(jobDir, "worker.lease")); + const link = fs.linkSync; + let recoveryLeases = 0, published = false; + const mock = t.mock.method(fs, "linkSync", function (from, to, ...args) { + if (path.dirname(String(to)) === directory && JSON.parse(fs.readFileSync(from, "utf8")).role === "recovery" && ++recoveryLeases === 2) { + const job = JSON.parse(fs.readFileSync(path.join(jobDir, "job.json"), "utf8")); + writeJob(jobDir, { ...job, ...(winner === "cancel" ? { state: "cancelled", completedAt: new Date().toISOString() } : { worker: successor }) }); + published = true; + } + return link.call(this, from, to, ...args); + }); + try { assert.equal(await resumeDeadJobOnce(jobDir, ctx.env), false); } + finally { mock.mock.restore(); } + assert.equal(published, true); + const job = JSON.parse(fs.readFileSync(path.join(jobDir, "job.json"), "utf8")); + if (winner === "cancel") assert.equal(job.state, "cancelled"); + else assert.deepEqual(job.worker, successor); + assert.equal(job.attempts.length, 0); +}); diff --git a/tests/relay-delivery.test.mjs b/tests/relay-delivery.test.mjs index 0329ed43..7b76a2b9 100644 --- a/tests/relay-delivery.test.mjs +++ b/tests/relay-delivery.test.mjs @@ -192,7 +192,7 @@ test("completed Workflow adoption consumes an early message with wait false", as assert.equal(runs(first, text), 1); }); -test("an acknowledgement timeout retries the original post exactly once and in order", async () => { +test("an acknowledgement timeout under a live worker retries the original post exactly once and in order", async () => { const ctx = relayContext("job", { turnMs: 100 }); const { jobId, dir } = await completed(ctx); const release = holdLease(dir); @@ -497,14 +497,16 @@ test("setup before posting on the adoption start path records one violation", as fs.writeFileSync(inbox, "obstruct inbox setup"); try { const { started } = await relayStart(retry); - assert.notEqual(started.status, 0); + assert.notEqual(started.status, 0, started.stdout + started.stderr); + const { listJobDirs } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + assert.deepEqual(listJobDirs({ env: retry.env }), [dir], "uncertain adoption must never submit a duplicate job"); assert.equal(readRelayViolations(dir).length, 1); - assert.match(readRelayViolations(dir)[0].reason, /messages could not be delivered: EEXIST/); + assert.match(readRelayViolations(dir)[0].reason, /messages could not be delivered: Message storage uncertain:.*ENOTDIR/); } finally { fs.unlinkSync(inbox); fs.renameSync(saved, inbox); } - assert.match(companion(first, ["result", jobId]).stdout, /^relay-violation: delivery: .*EEXIST/m); + assert.match(companion(first, ["result", jobId]).stdout, /^relay-violation: delivery: .*storage uncertain:.*ENOTDIR/m); }); test("wait false reports no consumer on timeout", async () => { @@ -513,7 +515,7 @@ test("wait false reports no consumer on timeout", async () => { const message = postInboxMessage(dir, { kind: "continue", text: "[quick] No consumer yet" }); const result = await awaitOutcome(dir, message.id, { env: ctx.env, wait: false, timeoutMs: 0 }); assert.equal(result.consumer, false); - assert.equal(result.outcome, null); + assert.equal(result.outcome.action, "not-started"); assert.equal(runs(ctx, message.text), 0); }); @@ -623,12 +625,12 @@ fs.linkSync = function (from, to, ...rest) { fs.writeFileSync(go, ""); await ended; } - assert.notEqual(receipt.status, "done", "a cancel holding the lease cannot acknowledge consumption"); - // Released, the cancel read the inbox again and started a worker for the follow-up. + assert.equal(receipt, null, "a cancel lease cannot accept work; the refused mapping was removed"); + assert.equal(runs(ctx, text), 0); + // The next hook posts the withdrawn message again and gets a real consumer. + await deliverRelayMessages(dir, [{ uuid: entry.uuid, text }], { env: ctx.env, wait: false }); await waitFor(() => runs(ctx, text) === 1); await waitFor(() => readJobJson(ctx, jobId).state === "completed" && !readLease(dir)); - // The next hook closes the receipt without running the message again. - await deliverRelayMessages(dir, [{ uuid: entry.uuid, text }], { env: ctx.env, wait: false }); assert.equal(readLedger(dir, entry.uuid).status, "done"); assert.equal(runs(ctx, text), 1); }); diff --git a/tests/relay-guard.test.mjs b/tests/relay-guard.test.mjs index 44c0d7e2..db6264ed 100644 --- a/tests/relay-guard.test.mjs +++ b/tests/relay-guard.test.mjs @@ -461,13 +461,20 @@ test("without SubagentHandback, SubagentStop still checks the final text", async }); test("late unresolved cleanup preserves the copied result and stays visible in status", async () => { - const preload = path.join(makeTempDir(), "fail-launch.mjs"); - fs.writeFileSync(preload, "if (process.argv[1]?.endsWith('/launcher.mjs')) process.exit(1);\n"); - const ctx = relayContext("job", { turnMs: 50, env: { NODE_OPTIONS: `--import=${preload}` } }); - const jobId = await startedRelay(ctx, QUICK); - assert.equal(readJobJson(ctx, jobId).state, "queued"); + const ctx = relayContext("job", { turnMs: 50 }); + const { createJobDir, writeJob, jobPaths } = await import("../plugins/relay/scripts/lib/jobs/store.mjs"); + const { readLaunchPrompt } = await import("../plugins/relay/scripts/lib/jobs/request.mjs"); + const { writeRelayPointer } = await import("../plugins/relay/scripts/lib/harness/relay.mjs"); const { runWorker } = await import("../plugins/relay/scripts/lib/worker/worker.mjs"); const { listProcesses } = await import("../plugins/relay/scripts/lib/util/process.mjs"); + ctx.append(ctx.prompt(QUICK), ctx.systemReminder()); + const jobId = "job-late-cleanup", { jobDir } = createJobDir(ctx.work, jobId, ctx.env); + const launch = readLaunchPrompt(QUICK, { baseDir: ctx.work, env: ctx.env }); + const relay = { agentId: ctx.agentId, transcriptPath: ctx.transcript }; + writeJob(jobDir, { id: jobId, kind: "task", state: "starting", createdAt: new Date().toISOString(), + pendingFirstTurn: true, attempts: [], request: launch.request, relay }); + fs.writeFileSync(jobPaths(jobDir).brief, launch.brief); + writeRelayPointer(ctx.env, { ...relay, jobId, jobDir, startError: null }); let copied; await runWorker(jobDirOf(ctx, jobId), { env: ctx.env, @@ -1546,7 +1553,7 @@ test("a Workflow relay that never runs start gets the Workflow correction with i assert.equal(waitJob(ctx, jobId).status, 0); }); -test("a queued Workflow stays pending through repeated hand-back and stop attempts beyond the escape budget", async () => { +test("a queued Workflow stays pending through repeated hand-back and stop attempts beyond the escape budget", async t => { const ctx = relayContext("job", { workflow: WORKFLOW, turnMs: 100 }); ctx.append(ctx.userRequest(), ctx.environment(), ctx.workflowPrompt("computedTask", QUICK), ctx.snapshot("string")); const { jobId } = await relayStart(ctx); @@ -1556,11 +1563,17 @@ test("a queued Workflow stays pending through repeated hand-back and stop attemp await waitFor(() => !readLease(dir)); const file = path.join(dir, "job.json"); const job = JSON.parse(fs.readFileSync(file, "utf8")); + const { acquireWorkerLease, releaseWorkerLease, isWorkerAlive } = await import("../plugins/relay/scripts/lib/jobs/lease.mjs"); + const { readProcessStartTime } = await import("../plugins/relay/scripts/lib/util/process.mjs"); + await waitFor(() => !isWorkerAlive(job.worker)); + const worker = { pid: process.pid, startTime: readProcessStartTime(process.pid) }; + assert.equal(acquireWorkerLease(dir, worker), true); + t.after(() => { releaseWorkerLease(dir, worker); fs.writeFileSync(file, JSON.stringify(job)); }); ctx.env.RELAY_ADMISSION = "on"; ctx.env.RELAY_ADMISSION_HOME = path.join(ctx.root ?? path.dirname(dir), "synthetic-admission"); ctx.env.RELAY_TEST_ADMISSION_READINGS = path.join(dir, "synthetic-readings.json"); fs.writeFileSync(ctx.env.RELAY_TEST_ADMISSION_READINGS, JSON.stringify({ availableBytes: 0, totalBytes: 40 * 2 ** 30, cores: 8, load5: 0 })); - fs.writeFileSync(file, JSON.stringify({ ...job, state: "queued", worker: null, admission: { state: "waiting", waitKind: "worker", reason: "memory low", since: new Date().toISOString() } })); + fs.writeFileSync(file, JSON.stringify({ ...job, state: "queued", worker, admission: { state: "waiting", waitKind: "memory", reason: "memory low", since: new Date().toISOString() } })); for (let i = 0; i < RELAY_BLOCK_BUDGET + 2; i++) { assert.equal(decision(await pre(ctx, "SubagentHandback", { message: "done" })), "deny"); const blocked = await stop(ctx, "done");