From 2e83cece7c5726344b90a007ba21e15dbfbed795 Mon Sep 17 00:00:00 2001 From: Erik <262919414+try-works@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:34:22 +0800 Subject: [PATCH 1/5] fix(replay): stop the replay tick starving on an unclassified pending capture The tick builds its capture set from the sidecar's cheap pending view and then adds the rich corpus read only for captures the pending view does not already contain. Because only the corpus read carried the route classification, deduping by capture ref discarded the classified row and kept the unclassified one, so the ownership filter matched nothing and the tick threw NoReplayableRequest on every tick for ever. Whenever the focus task's captures all fit inside the pending scan window - a task that has just started being routed, or any small state root - no replay job is ever created, no disposition is written and the pending row is never retired. On a mature root the corpus read usually supplies captures outside the window, which is why the defect stayed hidden on stage and in production. A capture that is already present is now enriched from the corpus read instead of dropped, so an unclassified row can never mask the identity the corpus read proved it has; a capture that is already classified keeps the cheap row because then the two views agree. Adds the regression test that models the real interaction - the same capture ref in both views, with identity only in the corpus read - which the run-105 suite never exercised because it stubbed the two readers independently and almost always left the pending view empty. Also names every gate that rejects a capture in readRouteReplayableCaptures and logs the failing gates under ROLE_MODEL_FOCUS_DIAG, and logs the cause when the corpus read is unavailable instead of collapsing to null. That corpus read had been blamed for the starvation because a rejected capture surfaced only as an unattributable NoReplayableRequest; the transcript was never the problem, so the trial artifact readback added for it is removed. --- .../src/route-challenge-evidence.ts | 96 ++++++++++++------- .../src/track-b-auto-replay-runtime.ts | 51 +++++++--- .../test/run105-review-dispatch.test.ts | 29 ++++++ 3 files changed, 129 insertions(+), 47 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts b/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts index 865b1656..7482a434 100644 --- a/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts +++ b/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts @@ -515,6 +515,7 @@ export async function readRouteReplayableCaptures(input: { readonly scopeId: string; readonly channel: string; readonly request: { readonly roleId: string; readonly taskTypeId: string }; + /** The capsule read resolves the transcript out of the encrypted artifact store itself. */ readonly readCapture: (requestId: string) => Promise; }): Promise { if (!text(input.request.roleId) || !text(input.request.taskTypeId) || !text(input.scopeId)) @@ -568,48 +569,75 @@ export async function readRouteReplayableCaptures(input: { for (const requestId of ids) { const capture = row(await input.readCapture(requestId)); if (!capture) continue; // Exact read knows eviction/absence; telemetry alone is not a capsule. + const messages = capture.messages; const classification = row(capture.classification); const trace = row(capture.trace); const source = row(capture.replaySource); const roleId = text(capture.roleId) ?? text(classification?.roleId); const taskTypeId = text(capture.taskTypeId) ?? text(classification?.taskTypeId); const sourceClass = text(row(capture.replayEvidenceClass)?.class); - if ( - capture.requestId !== requestId || - capture.scope !== scope || - roleId !== input.request.roleId || - taskTypeId !== input.request.taskTypeId || - (classification?.roleId != null && classification.roleId !== roleId) || - (classification?.taskTypeId != null && classification.taskTypeId !== taskTypeId) || - !text(capture.endpointId) || - !text(capture.rootArtifactId) || - !source || - !trace || - trace.scopeId !== scope || - trace.channel !== input.channel || - trace.readiness !== "ready" || - trace.completeness !== "complete" || - !Number.isSafeInteger(trace.generation) || - Number(trace.generation) < 0 || - !text(trace.rootOccurrenceId) || - !text(trace.closedAt) || - ![ + /** + * Every gate is named once, in one ordered table, so the corpus read has exactly one + * expression of its admission rule. A rejected capture used to leave the tick with an + * unattributable "no replayable request", which is what made a corpus that persisted and + * hydrated correctly look like it had never been captured. The names are operator-facing: + * with ROLE_MODEL_FOCUS_DIAG set, one tick prints the gate that dropped each capture. + */ + const rejections: string[] = []; + const gate = (name: string, admitted: boolean): void => { + if (!admitted) rejections.push(name); + }; + gate("request_id", capture.requestId === requestId); + gate("scope", capture.scope === scope); + gate("role", roleId === input.request.roleId); + gate("task", taskTypeId === input.request.taskTypeId); + gate( + "classification_role", + classification?.roleId == null || classification.roleId === roleId, + ); + gate( + "classification_task", + classification?.taskTypeId == null || classification.taskTypeId === taskTypeId, + ); + gate("endpoint", text(capture.endpointId) != null); + gate("root_artifact", text(capture.rootArtifactId) != null); + gate("replay_source", source != null); + gate("trace", trace != null); + gate("trace_scope", trace?.scopeId === scope); + gate("trace_channel", trace?.channel === input.channel); + gate("trace_readiness", trace?.readiness === "ready"); + gate("trace_completeness", trace?.completeness === "complete"); + gate( + "trace_generation", + Number.isSafeInteger(trace?.generation) && Number(trace?.generation) >= 0, + ); + gate("trace_root_occurrence", text(trace?.rootOccurrenceId) != null); + gate("trace_closed_at", text(trace?.closedAt) != null); + gate( + "replay_source_refs", + [ "normalizedRequestRef", "sharedPrefixRef", "forkOccurrenceId", "policySnapshotRef", "capturePolicyRef", - ].every((key) => text(source[key])) || - !Array.isArray(capture.messages) || - capture.messages.length === 0 || - capture.branchKind != null || - capture.branchPhase != null || - capture.replayProduced === true || - /^replay-/.test(requestId) || - isBenchmarkReplaySourceRef(requestId) || - isSyntheticProbeSourceClass(sourceClass) - ) + ].every((key) => text(source?.[key]) != null), + ); + gate("messages", Array.isArray(messages) && messages.length > 0); + gate("not_branch", capture.branchKind == null && capture.branchPhase == null); + gate("not_replay_produced", capture.replayProduced !== true); + gate("not_replay_request_id", !/^replay-/.test(requestId)); + gate("not_benchmark_source", !isBenchmarkReplaySourceRef(requestId)); + gate("not_synthetic_probe", !isSyntheticProbeSourceClass(sourceClass)); + if (rejections.length > 0) { + if (process.env.ROLE_MODEL_FOCUS_DIAG) + console.error( + `[replay-projection] capture=${requestId} rejected=${rejections.join(",")} trace=${trace ? `${String(trace.scopeId) === scope ? "scope-ok" : "scope-mismatch"}/${String(trace.channel)}/${String(trace.readiness)}/${String(trace.completeness)}` : "absent"} messages=${Array.isArray(messages) ? messages.length : "unreadable"}`, + ); continue; + } + // Narrowing only: the messages gate above already rejected every non-array transcript. + if (!Array.isArray(messages)) continue; captures.push({ captureRef: requestId, sourceEndpointId: capture.endpointId as string, @@ -618,13 +646,17 @@ export async function readRouteReplayableCaptures(input: { hasRecordedToolResults: hasRecordedToolResults(capture), replayProduced: false, sourceClass, - messages: capture.messages, + messages, requirements: readReplayRequestRequirements(capture), replayDispositionSettled: settledRefs.has(requestId), }); } return captures; - } catch { + } catch (error) { + if (process.env.ROLE_MODEL_FOCUS_DIAG) + console.error( + `[replay-projection] unavailable: ${error instanceof Error ? error.message : String(error)}`, + ); return null; } } diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts index 36aee8a7..7338d678 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts @@ -1591,21 +1591,42 @@ export function startAutoReplayLoop(input: { `[replay-tick] after readRouteReplayableCaptures history=${history === null ? "null" : history.length}`, ); if (history === null) throw new Error("route replayable corpus unavailable"); - const existing = new Set(captures.map((item) => item.captureRef)); - captures = [ - ...captures, - ...history.filter( - (item) => - item.roleId === plannedFocus?.roleId && - item.taskTypeId === plannedFocus?.taskTypeId && - !existing.has(item.captureRef) && - // Run 105: the corpus read stays a faithful projection and still ENUMERATES a capture whose - // replay disposition is terminal. The dispatch planner is the layer that acts on that - // annotation, so the tick advances to a fresh capture instead of re-picking one whose - // dispatch the lane would only refuse as duplicate_already_processed. - item.replayDispositionSettled !== true, - ), - ]; + /** + * Run 106 cold-start repair: the pending view and the corpus read are two projections of the + * SAME captures, and only the corpus read carries the route classification. Deduping by + * capture ref alone therefore discarded the classified row and kept the unclassified one, so + * the ownership filter below matched nothing. Whenever every capture of the focus task also + * fit inside the pending scan window - which is exactly the state of a task that has just + * started being routed, and of any small state root - the tick livelocked on + * `NoReplayableRequest` for ever: no dispatch, no disposition, so the pending row was never + * retired. A capture that is already present is now ENRICHED instead of dropped, so an + * unclassified row can never mask the identity the corpus read proved it has. A capture that + * is already classified keeps the cheap row, because then the two views agree. + */ + const byCaptureRef = new Map(captures.map((item) => [item.captureRef, item])); + for (const item of history) { + if (item.roleId !== plannedFocus?.roleId) continue; + if (item.taskTypeId !== plannedFocus?.taskTypeId) continue; + // Run 105: the corpus read stays a faithful projection and still ENUMERATES a capture whose + // replay disposition is terminal. The dispatch planner is the layer that acts on that + // annotation, so the tick advances to a fresh capture instead of re-picking one whose + // dispatch the lane would only refuse as duplicate_already_processed. + if (item.replayDispositionSettled === true) continue; + const present = byCaptureRef.get(item.captureRef); + if (!present) { + captures = [...captures, item]; + byCaptureRef.set(item.captureRef, item); + continue; + } + if (present.roleId != null && present.taskTypeId != null) continue; + const enriched = { + ...present, + roleId: item.roleId ?? present.roleId, + taskTypeId: item.taskTypeId ?? present.taskTypeId, + }; + captures = captures.map((entry) => (entry === present ? enriched : entry)); + byCaptureRef.set(item.captureRef, enriched); + } } const owned = captures.filter( (item) => diff --git a/role-model-router/apps/runtime-host-bridge/test/run105-review-dispatch.test.ts b/role-model-router/apps/runtime-host-bridge/test/run105-review-dispatch.test.ts index d32a4668..bfa4417d 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run105-review-dispatch.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run105-review-dispatch.test.ts @@ -517,6 +517,35 @@ test("historical replayable source seam retains exact task classification and ac await loop.tick(); expect(h.calls).toEqual([{ captureRef: "idle-a-history", source: "a", candidates: ["new"] }]); }); +/** + * Run 106 cold-start regression: the cheap pending view and the rich corpus read are two projections of + * the SAME captures, and only the corpus read carries the route classification. Deduping by capture ref + * alone discarded the classified row and kept the unclassified one, so a task whose captures all fit in + * the pending scan window - a task that has just started being routed, or any small state root - threw + * NoReplayableRequest on every tick for ever. The pending fixture below is the sidecar's real shape: the + * same capture ref, without identity. + */ +test("cold-start task owns an unclassified pending row once the corpus read proves its classification", async () => { + const h = harness(); + h.configure(["a", "b", "c", "new"]); + h.census([task("idle", 10, 3, 4)]); + h.pending([ + { + captureRef: "idle-a", + sourceEndpointId: "a", + hasRecordedToolResults: false, + roleId: null, + taskTypeId: null, + }, + ]); + const loop = startAutoReplayLoop({ + ...h.input, + readRouteReplayableCaptures: async () => [capture("idle", "a")], + }); + cleanups.push(() => loop.stop()); + await loop.tick(); + expect(h.calls).toEqual([{ captureRef: "idle-a", source: "a", candidates: ["new"] }]); +}); test("restart preserves recorded challenge cutoff rather than rewriting now and hiding durable groups", async () => { const h = harness(); h.configure(["a", "b", "c", "new"]); From b5a11efcd058e5b9b2d4f33ca9ec2797e5dd23f1 Mon Sep 17 00:00:00 2001 From: Erik <262919414+try-works@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:59:43 +0800 Subject: [PATCH 2/5] fix(replay): carry the task family through the classification fallback The supervised-replay comparability read the task family straight off the capture record, while the role and the taxonomy revision next to it have always been read through a fallback to classification. A capture that records the family only where the capture contract puts it therefore produced a comparison whose comparability block named the role and not the task. The knowledge worker scopes a comparison by (comparability.roleId, comparability.taskTypeId) and excludes an incomplete scope fail-closed as incomplete_scope. So the group was finalized, the admission floor never counted it, and no RouteLadderPackV1 could ever be written for the task - the learner saw a task that never accumulated evidence while its comparisons piled up durably. Measured on the dev replay queue: two of eight finalized comparison groups carried taskTypeId null while their source captures' durable classification named writer.summarize. The fixtures that covered the reader always declared a flat taskTypeId, so the gap was invisible to the suite; the new cases pin the shape production actually has. --- .../apps/runtime-host-bridge/src/cli.ts | 35 ++++++++++++--- .../run104-r22-learning-scope-travel.test.ts | 45 ++++++++++++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index 55b11319..fb790585 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -748,6 +748,29 @@ export function readCaptureRoleId(capture: DurableReplayCapture): string | undef return typeof roleId === "string" && roleId.trim() ? roleId.trim() : undefined; } +/** + * Run 106 (live finding on the dev replay queue, 2026-10-07): the task family the capture was classified + * under, read from the same two places as the role and the revision. The learner's floor and its pack + * scope are keyed by (roleId, taskTypeId) and a comparison whose comparability block names only the role + * is excluded fail-closed as `incomplete_scope` - the comparison group is finalized, the floor never + * counts it, and no pack can ever be written for the task. + * + * `roleId` and `taxonomyVersion` have always been read through a classification fallback. `taskTypeId` + * was read straight off the capture record, so it worked only while the capture writer also stamped the + * redundant top-level copy; a capture that records the family ONLY where the contract puts it - the + * classification - produced an unscoped comparison. Read it exactly like its two siblings. + */ +export function readCaptureTaskTypeId(capture: DurableReplayCapture): string | undefined { + const direct = capture.taskTypeId; + if (typeof direct === "string" && direct.trim()) return direct.trim(); + const classification = capture.classification; + if (!classification || typeof classification !== "object" || Array.isArray(classification)) { + return undefined; + } + const taskTypeId = (classification as Readonly>).taskTypeId; + return typeof taskTypeId === "string" && taskTypeId.trim() ? taskTypeId.trim() : undefined; +} + /** * Run 98 addendum 58 §23 (live v316, 08:33Z): Replay Core re-presents an `append_recovery` request when a * previous attempt already burned the nonce and persisted the provider receipt but never appended the @@ -2430,11 +2453,13 @@ export function createSupervisedReplayEvaluationCompleter(input: { // Run 99 R33 (S34 live finding): every comparison in the live store carried no task family // because this supervised-replay path never passed the capture's family, so the learner's // family-scoped floor could never be met from real traffic. The capture records the family - // (`addendum 19 S33`), so it travels with the replay. - ...(typeof input.sourceCapture.taskTypeId === "string" && - input.sourceCapture.taskTypeId.trim() - ? { taskTypeId: input.sourceCapture.taskTypeId.trim() } - : {}), + // (`addendum 19 S33`), so it travels with the replay - read through the same classification + // fallback the role and the revision already use (Run 106), because a capture that records the + // family only on its classification produced a comparison the floor excludes as incomplete_scope. + ...(() => { + const taskTypeId = readCaptureTaskTypeId(input.sourceCapture); + return taskTypeId ? { taskTypeId } : {}; + })(), // Run 98 addendum 58 §22.2.3: the revision the capture was classified under travels with the // family, so the comparability key — and the pack scope the learner derives from it — is // version-stamped and the advisory's version gate can match it instead of failing closed. diff --git a/role-model-router/apps/runtime-host-bridge/test/run104-r22-learning-scope-travel.test.ts b/role-model-router/apps/runtime-host-bridge/test/run104-r22-learning-scope-travel.test.ts index 89113ee0..a251a783 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run104-r22-learning-scope-travel.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run104-r22-learning-scope-travel.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "vitest"; -import { readCaptureRoleId } from "../src/cli.js"; +import { readCaptureRoleId, readCaptureTaskTypeId } from "../src/cli.js"; /** * Run 104 R22 (R22-B), TDD strict - the replay path's declared taxonomy must reach the comparison. @@ -52,3 +52,46 @@ test("R22-B: a capture that classified no role reports absence rather than inven expect(readCaptureRoleId({} as never)).toBeUndefined(); expect(readCaptureRoleId({ classification: { roleId: " " } } as never)).toBeUndefined(); }); + +/** + * Run 106 (live finding on the dev replay queue, 2026-10-07): the family travels with the role, and it + * needs the SAME classification fallback. Every fixture above declares a flat \`taskTypeId\`, which is why + * the gap was invisible: the reader used for the family read only that flat copy, so a capture that + * records the family where the contract actually puts it - \`classification.taskTypeId\` - produced a + * comparison whose comparability block named the role and not the task. The knowledge worker scopes a + * comparison by \`(comparability.roleId, comparability.taskTypeId)\` and excludes anything incomplete + * fail-closed (\`incomplete_scope\`), so those comparisons were finalized, never counted by the admission + * floor, and no pack could ever be written for the task. + * + * Measured: two of eight finalized groups on the dev root carried \`taskTypeId: null\` while their captures' + * durable classification named \`writer.summarize\`. + */ +const captureWithClassifiedFamily = () => ({ + classification: { + taskTypeId: "writer.summarize", + roleId: "writer", + taxonomyVersion: "1.0.0-alpha.1", + toolClassIds: [], + }, +}); + +test("R22-B follow-up: a family declared only inside the capture's classification is returned", () => { + expect(readCaptureTaskTypeId(captureWithClassifiedFamily() as never)).toBe("writer.summarize"); +}); + +test("R22-B follow-up: a family declared on the capture itself is still preferred", () => { + expect( + readCaptureTaskTypeId({ + taskTypeId: " coder.review ", + ...captureWithClassifiedFamily(), + } as never), + ).toBe("coder.review"); +}); + +test("R22-B follow-up: a capture that classified no family reports absence rather than inventing one", () => { + expect(readCaptureTaskTypeId({} as never)).toBeUndefined(); + expect(readCaptureTaskTypeId({ taskTypeId: " " } as never)).toBeUndefined(); + expect( + readCaptureTaskTypeId({ classification: { roleId: "writer", taskTypeId: " " } } as never), + ).toBeUndefined(); +}); From ecc432ae0829a9a0cd94a829494a009050901b91 Mon Sep 17 00:00:00 2001 From: Erik <262919414+try-works@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:26:27 +0800 Subject: [PATCH 3/5] fix(replay): choose the counterfactual arm at the source capture's effort The focus-dispatch walk returns the first configured endpoint that is not admitted, not an unavailable rung, not already a rung and not excluded - effort is not an input to it at all. The downstream same-model repair (preferEffortMatchedReplayArms) cannot rescue the result when the arm's model IS the source's model, because the only configured endpoint satisfying 'this model at the source effort' is then the source endpoint itself, which the run-105 bug-3 distinct-source clause excludes. The arm keeps the effort it was handed, the comparison is finalized with validityIssues ['arm_effort_mismatch'], and the learner's admission floor discards it fail-closed. Measured on the dev replay queue: 3 of 8 finalized comparison groups, and a whole source endpoint - the max-effort variant of the source's model - could never contribute a single eligible comparison however much traffic it served. The walk now prefers a survivor that runs at the source capture's effort, and the tick resolves that effort from the endpoint registry it already has (the capture names the endpoint it came from; the registry names the effort that endpoint runs at), so no extra capture plumbing is needed. The comparison is what the run-104 comparability dimension already grades, and effort is compared trimmed and case-insensitively exactly as that grading does, so the chosen arm and the record that grades it agree. It never refuses: with no effort-matched survivor, or with no effort view supplied at all, the first survivor is returned exactly as before, so nothing that used to dispatch stops dispatching and the run-104 same-model repair contract is left untouched. --- .../apps/runtime-host-bridge/src/cli.ts | 9 +++ .../src/track-b-auto-replay-runtime.ts | 49 +++++++++++++ .../src/track-b-replay-policy.ts | 7 ++ .../core/src/route-ladder-dispatch.ts | 59 ++++++++++++--- .../run105-d-route-ladder-dispatch.test.ts | 72 +++++++++++++++++++ 5 files changed, 188 insertions(+), 8 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index fb790585..60043764 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -7434,6 +7434,15 @@ export async function main(): Promise { ledger, policySet, configuredEndpointIds: endpoints, + /** + * Run 106: the same descriptors the supervised-replay path already builds from the registry + * (`configuredReplayArms`), handed to the loop so the focus-dispatch walk can prefer a + * counterfactual arm that runs at the source capture's reasoning effort. Without it the arm is + * effort-mismatched whenever the source's own model is the arm's model, the comparison is finalized + * `arm_effort_mismatch`, and the admission floor discards it - a replay and a paid provider call + * spent on evidence nothing can use. + */ + configuredEndpointDescriptors: endpointDescriptors, healthyEndpointIds: healthyEndpoints, /** * Run 105 R8/R9/R11: the ladder providers the depth-first dispatcher consumes. Each is a diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts index 7338d678..b8c78eb1 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-auto-replay-runtime.ts @@ -500,6 +500,25 @@ export function startAutoReplayLoop(input: { * every tick instead of a snapshot. */ readonly configuredEndpointIds: readonly string[] | (() => readonly string[]); + /** + * Run 106 (live finding on the dev replay queue, 2026-10-07): the configured endpoints' declared model and + * reasoning effort. The focus-dispatch walk is otherwise effort-blind, and an arm whose effort differs from + * the source capture's is finalized `arm_effort_mismatch` and discarded fail-closed by the learner's + * admission floor - it spends a replay and a paid provider call to produce evidence nothing can use. A + * provider (not a snapshot) so an endpoint added or reconfigured while the runtime is up takes effect on + * the next tick, exactly like `configuredEndpointIds` above. Absent keeps the previous behaviour. + */ + readonly configuredEndpointDescriptors?: + | readonly { + readonly endpointId: string; + readonly modelId: string; + readonly reasoningEffort: string | null; + }[] + | (() => readonly { + readonly endpointId: string; + readonly modelId: string; + readonly reasoningEffort: string | null; + }[]); /** * Endpoints the runtime can actually dispatch to. A provider may be supplied so a * credential-less or degraded endpoint is re-evaluated every tick instead of being @@ -1219,6 +1238,28 @@ export function startAutoReplayLoop(input: { typeof input.configuredEndpointIds === "function" ? input.configuredEndpointIds() : input.configuredEndpointIds; + /** + * Run 106: endpointId -> declared reasoning effort, read from the same provider the CLI wires from the + * endpoint registry. Empty when the caller supplies none, which leaves the focus-dispatch walk exactly + * as it was. + */ + const endpointEffortsByEndpointId: Record = {}; + try { + const descriptors = + typeof input.configuredEndpointDescriptors === "function" + ? input.configuredEndpointDescriptors() + : (input.configuredEndpointDescriptors ?? []); + for (const descriptor of descriptors) { + if (typeof descriptor?.endpointId !== "string" || !descriptor.endpointId) continue; + endpointEffortsByEndpointId[descriptor.endpointId] = + typeof descriptor.reasoningEffort === "string" && descriptor.reasoningEffort.trim() + ? descriptor.reasoningEffort.trim() + : null; + } + } catch { + // An unreadable registry leaves the map empty: the walk keeps its previous behaviour rather than + // refusing to dispatch. + } const pending = await input.operations.listPendingReplayCaptures({ policySetDigest: input.policySet.policySetDigest, limit: maxCapturesPerTick * 4, @@ -1737,6 +1778,14 @@ export function startAutoReplayLoop(input: { .filter((rung) => rung.status === "available") .map((rung) => rung.endpointId), rungs: routeLadder?.rungs, + // Run 106: the arm must run at the source capture's effort, or the comparison is voided. + // The capture records where it came from, and the endpoint registry records the effort + // that endpoint runs at, so no extra capture plumbing is needed. + endpointEfforts: endpointEffortsByEndpointId, + sourceReasoningEffort: + (source?.sourceEndpointId + ? endpointEffortsByEndpointId[source.sourceEndpointId] + : null) ?? null, }); if (process.env.ROLE_MODEL_FOCUS_DIAG) { console.error( diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-replay-policy.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-replay-policy.ts index eb82d8af..97372f6c 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-replay-policy.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-replay-policy.ts @@ -930,6 +930,13 @@ export function classifyReplayArmEffort(input: { * repoint never crosses models and never invents an endpoint - it only chooses among endpoints the registry * already holds for the arm's own model. An arm without a matching variant is kept exactly as requested; the * caller records the resulting comparability with `classifyReplayArmEffort`. + * + * Run 106 note: this repair is a same-model patch on a decision taken upstream, and it can only act when the + * requested arm's model has an effort variant that is not the source endpoint. Where the arm's model IS the + * source's model, that is impossible by construction, so the arm keeps the effort it was handed and the + * comparison is finalized `arm_effort_mismatch` - which the learner's admission floor discards. The + * selection layer (`route-ladder-dispatch.planFocusDispatch`) is the layer that owns the choice, and it is + * where the effort view now lives; this function is left exactly as the run-104 contract states it. */ export function preferEffortMatchedReplayArms(input: { readonly arms: readonly ReplayArmDescriptor[]; diff --git a/role-model-router/packages/core/src/route-ladder-dispatch.ts b/role-model-router/packages/core/src/route-ladder-dispatch.ts index e964b299..e1b80d45 100644 --- a/role-model-router/packages/core/src/route-ladder-dispatch.ts +++ b/role-model-router/packages/core/src/route-ladder-dispatch.ts @@ -317,8 +317,28 @@ export interface RouteFocusDispatchInput { */ readonly excludedEndpointIds?: readonly string[]; readonly rungs?: readonly RouteLadderRung[] | null; + /** + * Run 106 (live finding on the dev replay queue, 2026-10-07): endpointId -> the reasoning effort that + * endpoint is configured to run at. A counterfactual arm whose effort differs from the source capture's + * is finalized `arm_effort_mismatch` and then discarded fail-closed by the learner's admission floor, so + * the arm is chosen effort-matched when the configuration offers one. Absent keeps the previous + * first-survivor behaviour exactly, so a caller with no effort view is unaffected. + */ + readonly endpointEfforts?: Readonly> | null; + /** The effort the source capture actually ran at, resolved from its own endpoint. */ + readonly sourceReasoningEffort?: string | null; } +/** + * Run 106: effort identity is compared the way the host compares it when it records the comparability + * dimension - trimmed and case-insensitive - so the arm this chooses and the record that grades it agree. + */ +const normalizeDispatchEffort = (value: unknown): string | null => { + if (typeof value !== "string") return null; + const trimmed = value.trim().toLowerCase(); + return trimmed.length > 0 ? trimmed : null; +}; + export interface PlannedFocusDispatch { readonly roleId: string; readonly taskTypeId: string; @@ -373,21 +393,44 @@ export function planFocusDispatch( .map((rung) => rung.endpointId) .filter((value): value is string => typeof value === "string" && value.length > 0), ); + const survivors: string[] = []; for (const endpointId of input.configuredEndpointIds ?? []) { if (typeof endpointId !== "string" || endpointId.length === 0) continue; if (admitted.has(endpointId)) continue; if (unavailable.has(endpointId)) continue; if (ranked.has(endpointId)) continue; if (excluded.has(endpointId)) continue; - return { - roleId: focus.roleId, - taskTypeId: focus.taskTypeId, - scopeKey: scopeKeyFor(focus.roleId, focus.taskTypeId), - captureRef: input.replayableCapture.captureRef, - endpointId, - }; + survivors.push(endpointId); + } + const captureRef = input.replayableCapture.captureRef; + const plan = (endpointId: string): PlannedFocusDispatch => ({ + roleId: focus.roleId, + taskTypeId: focus.taskTypeId, + scopeKey: scopeKeyFor(focus.roleId, focus.taskTypeId), + captureRef, + endpointId, + }); + /** + * Run 106: prefer an arm that runs at the source capture's effort. The walk above is otherwise + * effort-blind, and the downstream same-model repair cannot rescue it when the arm's model IS the + * source's model - the only same-model endpoint at the source effort is then the source itself, which + * the distinct-source rule excludes. Measured on the dev root: every comparison sourced from the + * max-effort variant of the source's model was finalized `arm_effort_mismatch` and discarded, and that + * endpoint could never contribute a single eligible comparison. + * + * This never refuses: when no configured survivor runs at the source effort - or the caller supplied no + * effort view at all - the first survivor is returned exactly as before, so nothing that used to + * dispatch stops dispatching. + */ + const sourceEffort = normalizeDispatchEffort(input.sourceReasoningEffort); + const efforts = input.endpointEfforts ?? null; + if (sourceEffort !== null && efforts !== null) { + const matched = survivors.find( + (endpointId) => normalizeDispatchEffort(efforts[endpointId]) === sourceEffort, + ); + if (matched) return plan(matched); } - return null; + return survivors.length > 0 ? plan(survivors[0] as string) : null; } /* --------------------------- derived activation (R9/D7) --------------------------- */ diff --git a/role-model-router/packages/core/test/run105-d-route-ladder-dispatch.test.ts b/role-model-router/packages/core/test/run105-d-route-ladder-dispatch.test.ts index e60c8485..25dfddd1 100644 --- a/role-model-router/packages/core/test/run105-d-route-ladder-dispatch.test.ts +++ b/role-model-router/packages/core/test/run105-d-route-ladder-dispatch.test.ts @@ -245,6 +245,78 @@ describe("run105 D planFocusDispatch (R8 depth-first fill)", () => { expect(planned).toBeNull(); }); + /** + * Run 106 (live finding on the dev replay queue, 2026-10-07): the walk was effort-blind, and the + * downstream same-model repair cannot rescue it when the arm's model is the source's model - the only + * same-model endpoint at the source effort is then the source itself, which the distinct-source rule + * excludes. The arm kept its own effort, the comparison was finalized \`arm_effort_mismatch\`, and the + * learner's admission floor discarded it fail-closed: a replay and a paid provider call spent on evidence + * nothing can use. Measured: 3 of 8 finalized groups on the dev root, and a whole source endpoint could + * never contribute a single eligible comparison. + */ + test("prefers a survivor that runs at the source capture's effort", () => { + const planned = planFocusDispatch({ + focus, + replayableCapture: { captureRef: "capture:1" }, + configuredEndpointIds: ["endpoint:low", "endpoint:max", "endpoint:other"], + admittedEndpointIds: [], + rungs: [], + endpointEfforts: { "endpoint:low": "low", "endpoint:max": "max", "endpoint:other": "max" }, + sourceReasoningEffort: "max", + }); + expect(planned).toMatchObject({ endpointId: "endpoint:max" }); + }); + + test("falls back to the first survivor when no survivor matches the source effort", () => { + const planned = planFocusDispatch({ + focus, + replayableCapture: { captureRef: "capture:1" }, + configuredEndpointIds: ["endpoint:low", "endpoint:max"], + admittedEndpointIds: [], + rungs: [], + endpointEfforts: { "endpoint:low": "low", "endpoint:max": "max" }, + sourceReasoningEffort: "minimal", + }); + expect(planned).toMatchObject({ endpointId: "endpoint:low" }); + }); + + test("keeps configured order when the caller supplies no effort view", () => { + const planned = planFocusDispatch({ + focus, + replayableCapture: { captureRef: "capture:1" }, + configuredEndpointIds: ["endpoint:low", "endpoint:max"], + admittedEndpointIds: [], + rungs: [], + }); + expect(planned).toMatchObject({ endpointId: "endpoint:low" }); + }); + + test("compares effort trimmed and case-insensitively, exactly as the comparability record does", () => { + const planned = planFocusDispatch({ + focus, + replayableCapture: { captureRef: "capture:1" }, + configuredEndpointIds: ["endpoint:low", "endpoint:max"], + admittedEndpointIds: [], + rungs: [], + endpointEfforts: { "endpoint:low": "low", "endpoint:max": " MAX " }, + sourceReasoningEffort: "max", + }); + expect(planned).toMatchObject({ endpointId: "endpoint:max" }); + }); + + test("an admitted or ranked effort-matched endpoint is still skipped", () => { + const planned = planFocusDispatch({ + focus, + replayableCapture: { captureRef: "capture:1" }, + configuredEndpointIds: ["endpoint:max", "endpoint:other"], + admittedEndpointIds: ["endpoint:max"], + rungs: [], + endpointEfforts: { "endpoint:max": "max", "endpoint:other": "max" }, + sourceReasoningEffort: "max", + }); + expect(planned).toMatchObject({ endpointId: "endpoint:other" }); + }); + test("a task with no replayable capture is skipped as NoReplayableRequest", () => { const result = planFocusDispatch({ focus, From 45dbc0b84891a1b7e47a3ef636c494501e3dbfa1 Mon Sep 17 00:00:00 2001 From: Erik <262919414+try-works@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:34:46 +0800 Subject: [PATCH 4/5] fix(replay): read the endpoint identity the registry actually publishes The loop's endpoint descriptor provider read modelId and reasoningEffort off the endpoint candidate's TOP LEVEL, where they do not exist: an EndpointCandidate carries them at identity.model_id and identity.reasoning_effort. Every descriptor therefore answered an empty model and a null effort, which made the focus-dispatch effort view empty and made the queue handoff's arm comparability always arm_effort_unspecified. Read the identity fields, keeping the top-level shape as a fallback. --- .../apps/runtime-host-bridge/src/cli.ts | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index 60043764..0ea25190 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -10738,18 +10738,33 @@ export async function main(): Promise { return null; } }, + /** + * Run 106 (live finding on the dev replay queue, 2026-10-07): the descriptor read the model and the + * effort off the endpoint's TOP LEVEL, where a registry endpoint candidate does not carry them - the + * identity lives at `identity.model_id` and `identity.reasoning_effort` (see + * `EndpointCandidate` in packages/endpoint-registry). Every descriptor therefore answered + * `modelId: ""` and `reasoningEffort: null`, so the arm comparability the queue handoff records + * was always `arm_effort_unspecified` and the focus-dispatch effort view was empty - which is + * exactly why an effort-mismatched arm could be chosen and then voided downstream. Read the identity + * fields, keeping the top-level shape as a fallback for an endpoint object that carries them. + */ () => - created.effectiveRegistry.endpoints.map((endpoint) => ({ - endpointId: endpoint.identity.endpoint_id, - modelId: - typeof (endpoint as { modelId?: unknown }).modelId === "string" - ? String((endpoint as { modelId?: unknown }).modelId) - : "", - reasoningEffort: - typeof (endpoint as { reasoningEffort?: unknown }).reasoningEffort === "string" - ? String((endpoint as { reasoningEffort?: unknown }).reasoningEffort) - : null, - })), + created.effectiveRegistry.endpoints.map((endpoint) => { + const declared = endpoint as { + readonly modelId?: unknown; + readonly reasoningEffort?: unknown; + }; + const modelId = endpoint.identity.model_id ?? declared.modelId; + const reasoningEffort = endpoint.identity.reasoning_effort ?? declared.reasoningEffort; + return { + endpointId: endpoint.identity.endpoint_id, + modelId: typeof modelId === "string" ? modelId : "", + reasoningEffort: + typeof reasoningEffort === "string" && reasoningEffort.trim() + ? reasoningEffort + : null, + }; + }), // Run 105 bug 3: pass the wired judge resolution down so the dispatcher's focus planner excludes the // SAME controller judge the supervised replay endpoint will reject (deepseek-flash), instead of // returning null from the empty durable table and planning the judge as a counterfactual arm. From aa33133719837cd9dd901aadfe1e8c01675733ba Mon Sep 17 00:00:00 2001 From: Erik <262919414+try-works@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:54:59 +0800 Subject: [PATCH 5/5] fix(learning): record the arm-effort dimension instead of excluding on it Operator decision (2026-10-07): reasoning effort is not a requirement for comparing two endpoints. Any configured endpoint is a legitimate counterfactual for any other one no matter what effort each runs at, so an effort difference must not cost a comparison its place in the learner's evidence. Run 104 R9 recorded the dimension but also named a mismatch as a validity issue. validityIssues means 'this comparison is not valid evidence at all' - a judge grading its own arm, or two arms that produced one byte-identical outcome - and every consumer treats a non-empty list as disqualifying: the knowledge worker's projection drops the group out of the admission floor, the ranking and the pack; the learning pass counts it as incomparable:arm_effort_mismatch and skips it; and this repository's finalized-route-challenge read refused to accept it as ladder progress. On a configuration whose source model has no sibling at the source effort, that removed every comparison the endpoint could ever produce, and 24 of 30 finalized groups were excluded on this name alone while the eligible count sat at 4 against a floor of 5. The dimension itself is unchanged and still published in the group's comparability block (matched / mismatched / unspecified on either side), so a receipt can still answer 'was this comparison effort-confounded?' and the learner can still report it. Only the disqualification is removed: the finalized-challenge read now accepts a mismatched arm and still requires the dimension to be present and well formed, and every other validity issue keeps its fail-closed meaning. Groups finalized before the decision still carry the name durably, so the knowledge worker and the learning pass filter that one name out rather than trusting the record to be clean. --- .../src/route-challenge-evidence.ts | 31 ++++++++++++++++--- .../src/track-b-learning-pass.ts | 9 ++++++ .../src/track-b-runtime.ts | 5 +-- ...-r9-effort-comparability-threading.test.ts | 6 ++-- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts b/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts index 7482a434..3f6c3c70 100644 --- a/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts +++ b/role-model-router/apps/runtime-host-bridge/src/route-challenge-evidence.ts @@ -88,6 +88,18 @@ function replayIdForGroup(group: Row): string | null { return replayIds.every((id) => id !== null && id === replayIds[0]) ? replayIds[0] : null; } +/** + * Run 106: the comparability spellings the durable record may carry for one arm's reasoning effort. The + * dimension is recorded, never gating (see the guard below), but an unknown spelling still fails closed so + * this readback stays a faithful projection of what Evaluation Core wrote. + */ +const ARM_EFFORT_COMPARABILITY_VALUES = new Set([ + "matched", + "mismatched", + "source_effort_unspecified", + "arm_effort_unspecified", +]); + async function project( listed: Row, request: RouteDispatchEvidenceReadRequest, @@ -130,9 +142,20 @@ async function project( return null; if (actual.outcome !== "source" && actual.outcome !== "candidate" && actual.outcome !== "tie") return null; + /** + * Run 106 (operator decision, 2026-10-07): the effort dimension is recorded, never gating. A comparison + * whose arms run at different reasoning efforts is valid evidence - a judge grading its own arm or two + * byte-identical outcomes are not, and those keep their fail-closed meaning here. The dimension is still + * required to be present and well formed, so the readback stays a faithful projection of the durable + * record rather than accepting whatever shape arrives. + */ + const COMPARABILITY_ONLY_VALIDITY_ISSUES = new Set(["arm_effort_mismatch"]); if ( actual.validityIssues !== undefined && - (!Array.isArray(actual.validityIssues) || actual.validityIssues.length > 0) + (!Array.isArray(actual.validityIssues) || + actual.validityIssues.some( + (issue) => !COMPARABILITY_ONLY_VALIDITY_ISSUES.has(text(issue) ?? ""), + )) ) return null; const effort = proof.effortComparability; @@ -141,11 +164,9 @@ async function project( if ( !arm || arm.endpointId !== candidate || - arm.comparability !== "matched" || + !ARM_EFFORT_COMPARABILITY_VALUES.has(text(arm.comparability) ?? "") || !text(arm.modelId) || - !text(arm.sourceModelId) || - !text(arm.reasoningEffort) || - arm.reasoningEffort !== arm.sourceReasoningEffort + !text(arm.sourceModelId) ) return null; if ( diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-learning-pass.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-learning-pass.ts index 413f6211..9a73805d 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-learning-pass.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-learning-pass.ts @@ -1112,10 +1112,19 @@ export function buildTrackBLearningEvidenceSummary(input: { // Run 99 R33 (addendum 21 D10, `guidance/11` "incomparable ... groups are ineligible for // promotion evidence"): a comparison that reports validity issues — including the judge's // own position-order disagreement — is excluded and counted by code, never averaged in. + /** + * Run 106 (operator decision, 2026-10-07): `arm_effort_mismatch` is a recorded comparability + * dimension, not a validity issue - see the durable writer in Evaluation Core. Groups finalized + * before that decision still carry the name in their durable `validityIssues`, so it is filtered + * here too and the comparison is learned from like any other. Every other issue keeps its meaning: + * a comparison reporting one is excluded and counted by code, never averaged in. + */ + const COMPARABILITY_ONLY_VALIDITY_ISSUES = new Set(["arm_effort_mismatch"]); const validityIssues = Array.isArray(result.validityIssues) ? result.validityIssues .map((issue) => boundedText(issue)) .filter((issue): issue is string => issue !== null) + .filter((issue) => !COMPARABILITY_ONLY_VALIDITY_ISSUES.has(issue)) : []; if ( result.orderDisagreement === true || diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 64628f5b..410ef07a 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -8889,8 +8889,9 @@ export async function runTrackBShadowPipeline( firstCounterfactual.endpointId, "counterfactual candidate", ), - // Run 104 R9 (post-closeout): the arm-effort dimension travels with the comparison identity, so - // Evaluation Core's arm_effort_mismatch exclusion can fire on a live comparison. + // Run 104 R9 (post-closeout): the arm-effort dimension travels with the comparison identity, so a + // receipt can answer "was this comparison effort-confounded?". Run 106: it is recorded, not gating - + // an arm at a different effort than the source no longer costs the comparison its eligibility. effortComparability: classifyReplayArmEffort({ arms: counterfactualRollouts.map((arm) => ({ endpointId: typeof arm.endpointId === "string" ? arm.endpointId : "", diff --git a/role-model-router/apps/runtime-host-bridge/test/run104-r9-effort-comparability-threading.test.ts b/role-model-router/apps/runtime-host-bridge/test/run104-r9-effort-comparability-threading.test.ts index c2cff1f3..f05928d8 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run104-r9-effort-comparability-threading.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run104-r9-effort-comparability-threading.test.ts @@ -16,8 +16,10 @@ import { classifyReplayArmEffort } from "../src/track-b-replay-policy.js"; * sourceReasoningEffort: ..., * }) * - * This locks in the record shape the consumer (evaluation-core normalizeArmEffortComparability) expects, - * so the arm_effort_mismatch exclusion can fire on a live comparison. + * This locks in the record shape the consumer (evaluation-core normalizeArmEffortComparability) expects. + * Run 106 (operator decision, 2026-10-07): the record is a comparability DIMENSION, not a validity issue - + * an arm at a different effort than the source no longer costs the comparison its place in the learner's + * admission floor, so the shape matters for the audit trail and for reporting, not for eligibility. */ // Mirrors the builder's type-narrowing glue over Record rollouts.