backends: deadlines, liveness and provider-recovery evidence (C08, C05, M11)

.claude-plugin/marketplace.json

@@ -5,13 +5,13 @@
   },
   "metadata": {
     "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.",
-    "version": "3.0.0-dev.24"
+    "version": "3.0.0-dev.26"
   },
   "plugins": [
     {
       "name": "relay",
       "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.",
-      "version": "3.0.0-dev.24",
+      "version": "3.0.0-dev.26",
       "author": {
         "name": "Libar"
       },

README.md

@@ -372,18 +372,35 @@ lock stays queued and cancellable; a stuck-lock diagnostic identifies the holder
 
 Recognized usage refusals pause Codex, Grok, Kimi or ZCode for 30 minutes, or a definite reset within
 six hours. Rate pauses start at 60 seconds and double to 15 minutes; Kimi concurrency refusals
-pause for 10 minutes. After a pause, one generation-fenced probe runs; its success clears the
-pause. Refused jobs remain failed for explicit resume. Pauses follow the selected backend account home; default homes retain the existing pause files.
+pause for 10 minutes. After a pause, one job probes the provider; the pause ends only when that job gets an actual answer from the
+provider, not when it merely starts. Refused jobs remain failed for explicit resume. Pauses follow the selected backend account home; default homes retain the existing pause files.
 Grok recognizes explicit weekly/usage/credit exhaustion, rate-limit text and provider 429 errors;
 its usage classifier has not been verified against a live exhausted account. A live probe is never replaced by age.
-These pauses gate new starts, not requests or retries inside running turns.
+These pauses gate new starts and later turns on other jobs while a running turn keeps its own retries.
+
+| Backend | Paused on | When |
+|---|---|---|
+| Codex | usage or rate | at first retry, including subagent signals |
+| Grok | weekly credits or rate | at first retry, unverified with a live exhausted account |
+| Kimi | usage, rate or concurrency | at first retry |
+| ZCode | usage or rate | when the turn fails; runtime retries are not exposed |
+
+Grok's web search cannot be turned off.
 With the guard effective, Kimi's internal retry default is 3 unless explicitly configured.
 
 Turn it off for an invocation with `RELAY_ADMISSION=off`, or domain-wide with
 `/relay:setup --admission off`. `/relay:setup --admission on` restores it. Settings live in
 `~/.relay-jobs/admission/config.json` (or `RELAY_ADMISSION_HOME/config.json`), shared across jobs
 homes. A custom jobs home cannot disable the guard. Old weight/class controls are ignored.
 
+Backend startup has a 2-minute deadline; session setup has 5 minutes, and a steer or cancel request has
+30 seconds. ACP cancellation is a notification and has no reply to wait for. Turns have no time limit.
+After 10 minutes without notifications, relay checks the backend every 5 minutes. Two unanswered
+30-second pings restart it once, using the normal server recovery path. A backend answering pings can
+keep a silent build or sleep running indefinitely. Relay tests ping support on each connection; an
+unanswered initial ping disables that watchdog and records a note, while control deadlines still apply.
+Job JSON and events record whether the backend is answering or relay is checking it.
+
 After a job starts, relay samples memory used by the job and its tools. Job status shows `memory:`,
 and guard status shows `relay jobs use`. On Linux, when a user systemd manager is available, every
 worker runs in its own scope under `relay-jobs.slice`; its tools inherit `oom_score_adj=500`, making

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@libar-dev/relay-plugin-cc",
-  "version": "3.0.0-dev.24",
+  "version": "3.0.0-dev.26",
   "private": true,
   "type": "module",
   "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.",

plugins/relay/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "relay",
-  "version": "3.0.0-dev.24",
+  "version": "3.0.0-dev.26",
   "description": "Run jobs on Codex, Grok Build, ZCode and Kimi Code from Claude Code like Claude agents.",
   "author": {
     "name": "Libar"

plugins/relay/CHANGELOG.md

@@ -8,6 +8,12 @@ its entries here.
 
 Changed:
 
+- Backend controls have request deadlines; silent turns use a capability-tested ping watchdog.
+  Responsive long-running tools have no turn time limit. Unresponsive backends follow the existing
+  one-restart recovery path.
+- Provider probes require model-response evidence and slow startup grants are checked again before
+  sending. Codex, Grok and Kimi retry signals pause new work immediately for the selected account.
+
 - Foreground `task` workers run independently of the caller; Ctrl-C leaves a dispatched job running.
 - Jobs own their execution groups and stop their tools at backend loss, cancellation and completion.
   Linux workers use per-job scopes whenever a user manager is available, and set OOM score 500.
@@ -21,6 +27,14 @@ Changed:
 
 Fixed:
 
+- Provider retry refusals preserve a live probe’s claim and generation, including deferred refusals
+  replayed before a healthy answer clears the pause.
+
+- RPC input coalesces tiny fragments so per-fragment object overhead cannot escape the line-memory bound.
+
+- ZCode duplicate waits cancel promptly and report uncertain starts accurately. Duplicate steer
+  acknowledgements stay owed until inclusion is proved, and cancellation prevents discarded-input resends.
+
 - Repeated cleanup preserves a predecessor's first observed death, keeping the recorded
   ordering of predecessor exit and replacement startup intact.
 - Worker heartbeats keep process accounting active with the load guard off without creating

plugins/relay/scripts/lib/backends/codex/connection.mjs

@@ -1,3 +1,4 @@
+import { CONTROL_MS } from "../transport/stdio-rpc.mjs";
 /** @typedef {import("../types").Connection} Connection */
 import { CodexAppServerClient } from "./client.mjs";
 import { clientIdsInThread, openThread, runTurnOnThread } from "./turn.mjs";
@@ -64,7 +65,7 @@ class CodexConnection {
     };
     this.steerTracker.turnSteerHandler?.("requested", params);
     try {
-      const response = await this.client.request("turn/steer", params);
+      const response = await this.client.request("turn/steer", params, { timeoutMs: CONTROL_MS });
       this.steerTracker.turnSteerHandler?.("settled", params, true);
       return response;
     } catch (error) {
@@ -73,7 +74,9 @@ class CodexConnection {
     }
   }
 
-  async interrupt(turnId) { await this.client.request("turn/interrupt", { threadId: this.sessionId, turnId }); }
+  async interrupt(turnId) { await this.client.request("turn/interrupt", { threadId: this.sessionId, turnId }, { timeoutMs: CONTROL_MS }); }
+  get pingMethod() { return "account/read"; }
+  async ping() { await this.client.request("account/read", { refreshToken: false }, { timeoutMs: CONTROL_MS, wedgeOnTimeout: false }); }
   async close() { await this.client.close(); }
 }
 

plugins/relay/scripts/lib/backends/codex/turn.mjs

@@ -1,3 +1,4 @@
+import { SESSION_MS } from "../transport/stdio-rpc.mjs";
 /**
  * @typedef {import("./protocol").AppServerNotification} AppServerNotification
  * @typedef {import("./protocol").ThreadResumeParams} ThreadResumeParams
@@ -31,6 +32,8 @@
  *   reasoningSummary: string[],
  *   error: unknown,
  *   onTurnId: ((turnId: string) => void) | null,
+ *   onProviderResponse?: () => void,
+ *   onProviderLimit?: (error: unknown) => void,
  *   responseReceived: boolean,
  *   workSeen: boolean
  * }} TurnCaptureState
@@ -200,6 +203,8 @@ function createTurnCaptureState(threadId, options = {}) {
     reasoningSummary: [],
     error: null,
     onTurnId: options.onTurnId ?? null,
+    onProviderResponse: options.onProviderResponse,
+    onProviderLimit: options.onProviderLimit,
     responseReceived: false,
     // Any item beyond the user's own message: a failed turn that did work is never resent.
     workSeen: false
@@ -383,6 +388,8 @@ function applyTurnNotification(state, message) {
       (message.method.startsWith("item/") || message.method === "turn/started")) {
     invalidateInference(state);
   }
+  if (["item/agentMessage/delta", "item/reasoning/summaryTextDelta", "item/reasoning/textDelta"].includes(message.method) ||
+      (["item/started", "item/completed"].includes(message.method) && ["agentMessage", "reasoning", "commandExecution", "fileChange", "mcpToolCall", "webSearch", "collabAgentToolCall"].includes(message.params.item?.type))) state.onProviderResponse?.();
   switch (message.method) {
     case "thread/started":
       trackThread(state, message.params.thread.id);
@@ -414,6 +421,7 @@ function applyTurnNotification(state, message) {
       break;
     case "error":
       state.error = message.params.error;
+      if (message.params.willRetry) state.onProviderLimit?.(message.params.error);
       break;
     case "turn/completed":
       if ((message.params.threadId ?? null) !== state.threadId) {
@@ -541,7 +549,7 @@ export function connectOptions(options) {
 
 
 export async function startThread(client, cwd, options = {}) {
-  const response = await client.request("thread/start", buildThreadParams(cwd, options));
+  const response = await client.request("thread/start", buildThreadParams(cwd, options), { timeoutMs: SESSION_MS });
   const threadId = response.thread.id;
   if (options.threadName) {
     try {
@@ -564,7 +572,7 @@ export async function startThread(client, cwd, options = {}) {
 
 
 async function resumeThread(client, threadId, cwd, options = {}) {
-  return client.request("thread/resume", buildResumeParams(threadId, cwd, options));
+  return client.request("thread/resume", buildResumeParams(threadId, cwd, options), { timeoutMs: SESSION_MS });
 }
 
 
@@ -608,7 +616,7 @@ export async function runTurnOnThread(client, threadId, options = {}) {
         ...(options.approvalPolicy ? { approvalPolicy: options.approvalPolicy } : {}),
         outputSchema: options.outputSchema ?? null
       }),
-    { onTurnId: options.onTurnId, steerTracker: options.steerTracker }
+    { onTurnId: options.onTurnId, steerTracker: options.steerTracker, onProviderResponse: options.onProviderResponse, onProviderLimit: options.onProviderLimit }
   );
 
   return {

plugins/relay/scripts/lib/backends/grok/connection.mjs

@@ -1,3 +1,4 @@
+import { CONTROL_MS, SESSION_MS } from "../transport/stdio-rpc.mjs";
 /** @typedef {import("../types").Connection} Connection */
 import { ACP_INITIALIZE_PARAMS, ACP_CHUNKS, acpUpdate, openSession, setConfigOption, promptSession, cancelSession, promptCancelled, promptResult, answerPermission } from "../transport/acp.mjs";
 import { heldMessage, evidenceKey, countHeld, grokSessionMessages } from "./history.mjs";
@@ -96,6 +97,11 @@ class GrokConnection {
   }
 
   get server() { return this.client; }
+  get pingMethod() { return "relay/ping"; }
+  async ping() {
+    try { await this.client.request("relay/ping", {}, { timeoutMs: CONTROL_MS, wedgeOnTimeout: false }); }
+    catch (error) { if (error.rpcCode !== -32601) throw error; }
+  }
   async close() { await this.client.close(); }
 
   saveState(patch) {
@@ -235,14 +241,14 @@ class GrokConnection {
     const promptId = this.running;
     this.steers = (this.steers ?? 0) + 1;
     try {
-      const response = await client.request("_x.ai/interject", { sessionId, text: message.text });
+      const response = await client.request("_x.ai/interject", { sessionId, text: message.text }, { timeoutMs: CONTROL_MS });
       const status = response.result?.status ?? response.status;
       if (status !== "queued") throw new Error(`Grok did not queue the interjection: ${status ?? "unknown"}`);
       // Queued is not durable: Grok can lose the interjection before it persists it.
       if (promptId) this.saveState({ promptIds: { ...this.state.promptIds, [message.id]: promptId } });
       return { turnId, promptId, unconfirmed: true };
     } catch (error) {
-      if (!client.exitSeen) this.saveDeliveries(this.settle(delivery, false));
+      if (!client.exitSeen && !client.wedged) this.saveDeliveries(this.settle(delivery, false));
       throw error;
     } finally {
       this.steers -= 1;
@@ -287,6 +293,8 @@ class GrokConnection {
         check();
       }
       const update = params.update;
+      if (notification.method === "session/update" && ["agent_message_chunk", "agent_thought_chunk", "tool_call"].includes(update?.sessionUpdate)) options.onProviderResponse?.();
+      if (notification.method === "_x.ai/session/update" && update?.sessionUpdate === "retry_state") options.onProviderLimit?.({ message: update.reason ?? update.retryState?.reason, data: { retryState: update.retryState ?? update } });
       const standard = acpUpdate(notification);
       if (standard.didWork || update?.sessionUpdate === "tool_call_delta_chunk") didWork = true;
       if (standard.startsTool) text = "";
@@ -354,6 +362,6 @@ export async function connect(request, hooks) {
       }
     });
     try { await client.initialize(); }
-    catch (error) { await client.close(); throw new Error(client.exitInfo ? grokEndMessage(client.exitInfo) : error.message); }
+    catch (error) { await client.close(); if (error.name === "TransportTimeout") throw error; throw new Error(client.exitInfo ? grokEndMessage(client.exitInfo) : error.message); }
     return new GrokConnection(client, request, { ...hooks, state: { ...hooks.state, binary } });
 }

plugins/relay/scripts/lib/backends/kimi/connection.mjs

@@ -1,3 +1,4 @@
+import { CONTROL_MS, SESSION_MS } from "../transport/stdio-rpc.mjs";
 import { admissionEnabled, DEFAULTS } from "../../worker/admission.mjs";
 import { shellCommandMarkdown } from "../../util/shell.mjs";
 /** @typedef {import("../types").Connection} Connection */
@@ -137,6 +138,11 @@ export class KimiConnection {
 
   get server() { return this.client; }
 
+  get pingMethod() { return "relay/ping"; }
+  async ping() {
+    try { await this.client.request("relay/ping", {}, { timeoutMs: CONTROL_MS, wedgeOnTimeout: false }); }
+    catch (error) { if (error.rpcCode !== -32601) throw error; }
+  }
   async close() {
     await this.stopObserver("the connection closed");
     await this.reads?.close();
@@ -173,7 +179,7 @@ export class KimiConnection {
       sessionId = sessionId ?? response.sessionId;
       this.sessionId = sessionId;
       // Kimi keeps model and effort across a load but not the mode, so all three go every time.
-      await client.request("session/set_mode", { sessionId, modeId: KIMI_MODES[request.mode] });
+      await client.request("session/set_mode", { sessionId, modeId: KIMI_MODES[request.mode] }, { timeoutMs: SESSION_MS });
       for (const [configId, value] of [["model", request.model], ["thinking", request.effort]]) {
         if (value != null) await setConfigOption(client, sessionId, configId, value);
       }
@@ -215,9 +221,13 @@ export class KimiConnection {
     const wire = this.wire.read();
     // Where this prompt's records can start: a record at or after it came after the send.
     const from = wire.consumed;
+    wire.onRecord = record => {
+      if (record.type === "turn.step.retrying") options.onProviderLimit?.(record.error ?? { message: record.errorMessage });
+    };
     const run = { turnId: null, didWork: false, from };
     client.setNotificationHandler(notification => {
       if (notification.params?.sessionId !== sessionId) return;
+      if (notification.method === "session/update" && ["agent_message_chunk", "agent_thought_chunk", "tool_call"].includes(notification.params?.update?.sessionUpdate)) options.onProviderResponse?.();
       if (acpUpdate(notification).didWork) run.didWork = true;
     });
     const ended = client.exitPromise.then(() => { throw new Error(kimiEndMessage(client.exitInfo)); });
@@ -238,6 +248,7 @@ export class KimiConnection {
       return { turnId: run.turnId, turnStatus: "failed", finalMessage: "", didWork: run.didWork, retryable: false, error: { message: error.message } };
     } finally {
       await this.stopObserver(client.exitSeen || client.exitInfo ? "Kimi's server ended" : "the run ended");
+      wire.onRecord = null;
       client.setNotificationHandler(null);
     }
   }
@@ -377,6 +388,6 @@ export async function connect(request, hooks) {
     }
   });
   try { await client.initialize(); }
-  catch (error) { await reads?.close(); await client.close(); throw new Error(client.exitInfo ? kimiEndMessage(client.exitInfo) : error.message); }
+  catch (error) { await reads?.close(); await client.close(); if (error.name === "TransportTimeout") throw error; throw new Error(client.exitInfo ? kimiEndMessage(client.exitInfo) : error.message); }
   return new KimiConnection(client, request, { ...hooks, state: { ...hooks.state, binary } }, reads);
 }

plugins/relay/scripts/lib/backends/kimi/wire.mjs

@@ -65,6 +65,7 @@ export class KimiWire {
    */
   constructor(locate, { io = fs, retainPrompt = () => false } = {}) {
     this.locate = locate;
+    this.onRecord = null;
     this.io = io;
     this.retainPrompt = retainPrompt;
     /** @type {string | null} */
@@ -123,6 +124,7 @@ export class KimiWire {
           if (!line.trim()) continue;
           let record;
           try { record = JSON.parse(line); } catch { continue; }
+          this.onRecord?.(record);
           this.apply(record, at);
           this.prune();
         }

plugins/relay/scripts/lib/backends/transport/acp.mjs

@@ -1,3 +1,4 @@
+import { SESSION_MS } from "./stdio-rpc.mjs";
 // The ACP messages Grok and Kimi use. Grok's queue and interjections, and Kimi's session record,
 // stay in their connections.
 export const ACP_INITIALIZE_PARAMS = {
@@ -26,11 +27,11 @@ export function acpUpdate(notification) {
 export function openSession(client, cwd, sessionId) {
   return client.request(sessionId ? "session/load" : "session/new", {
     ...(sessionId ? { sessionId } : {}), cwd, mcpServers: []
-  });
+  }, { timeoutMs: SESSION_MS });
 }
 
 export function setConfigOption(client, sessionId, configId, value) {
-  return client.request("session/set_config_option", { sessionId, configId, value });
+  return client.request("session/set_config_option", { sessionId, configId, value }, { timeoutMs: SESSION_MS });
 }
 
 // A client that serves file writes and terminals itself declares them, and the agent sends them.