diff --git a/docs/guides/MODULE_MAP.md b/docs/guides/MODULE_MAP.md index 7e5bf58a..e9798879 100644 --- a/docs/guides/MODULE_MAP.md +++ b/docs/guides/MODULE_MAP.md @@ -228,7 +228,7 @@ Bootstraps UI, initializes store, wires remaining Tier 3 modules. All business l | Module | Concern | |--------|---------| -| `app-connection.js` + `websocket-lifecycle.js` | WebSocket creation, epoch-guarded handshake/heartbeat/probes, lifecycle-owned bounded jittered reconnect/auth timers, connection status UI, disconnect/restore notifications | +| `app-connection.js` + `websocket-lifecycle.js` + `websocket-watchdog.js` | WebSocket creation, epoch-guarded handshake and bounded jittered reconnect/auth timers, suspendable generation-guarded heartbeat/probe watchdogs, connection status UI, disconnect/restore notifications | | `app-messages.js` | WebSocket message router (`processMessage`). Dispatches all incoming message types to appropriate handlers | | `pair-result-status.js` | Per-session pending/blocked/uncertain Split Worker result notice, retained-result view, and explicit duplicate-risk retry confirmation; state is cleared on session switch and never stored locally | | `message-delivery.js` + `message-delivery-ui.js` | Bounded client message acknowledgement/retry state and context-scoped receipt recovery notices | diff --git a/lib/driver-continuation-pair.js b/lib/driver-continuation-pair.js index 8f36cfdf..c8711c6d 100644 --- a/lib/driver-continuation-pair.js +++ b/lib/driver-continuation-pair.js @@ -1,4 +1,5 @@ var sessionProvenance = require("./session-provenance"); +var splitRoles = require("./session-split-group-roles"); var ACTIVE_RUN_STATES = ["armed", "running", "reviewing", "waiting-worker", "waiting-user", "paused"]; @@ -37,7 +38,10 @@ function inspect(source, ctx) { if (!group.pair || group.pair.driverId !== source.localId) { return { ok: false, error: "Only the configured Driver can continue an attached split pair." }; } - var worker = ctx.sm && ctx.sm.sessions.get(group.pair.workerId); + var roles = splitRoles.normalizePair(group.pair, group.members); + if (!roles.ok || roles.kind === "adhoc") return { ok: false, error: "The attached split pair roles are invalid." }; + if (roles.kind === "versioned") return { ok: false, error: "Driver continuation does not yet support a multi-Worker group." }; + var worker = ctx.sm && ctx.sm.sessions.get(roles.workerIds[0]); if (!worker || group.members.indexOf(worker.localId) === -1) return { ok: false, error: "The attached Split Worker is unavailable." }; if ((source.ownerId || null) !== (worker.ownerId || null)) return { ok: false, error: "The attached Split Worker owner changed." }; if (!sessionProvenance.isWorker(worker)) return { ok: false, error: "The attached split partner is not a verified Worker." }; diff --git a/lib/multi-worker-feature.js b/lib/multi-worker-feature.js new file mode 100644 index 00000000..9c2cae93 --- /dev/null +++ b/lib/multi-worker-feature.js @@ -0,0 +1,11 @@ +var ENABLED_GATE = Object.freeze({ feature: "multi-worker-runtime" }); + +function fromServerConfig(config) { + return config && config.multiWorkerRuntimeEnabled === true ? ENABLED_GATE : null; +} + +function isEnabled(gate) { + return gate === ENABLED_GATE; +} + +module.exports = { fromServerConfig: fromServerConfig, isEnabled: isEnabled }; diff --git a/lib/project-pair-autonomous-stop.js b/lib/project-pair-autonomous-stop.js index 8fa0e5c6..ff97595b 100644 --- a/lib/project-pair-autonomous-stop.js +++ b/lib/project-pair-autonomous-stop.js @@ -1,15 +1,21 @@ function stopAutonomousWork(session, roles, turnControl, taskControl, workerPermission) { if (!roles) return false; var runId = session.autonomousRun && session.autonomousRun.id; - var cancelled = taskControl.cancelAll(roles.worker, runId); - var owned = roles.worker._pairDelegation && roles.worker._pairDelegation.autonomousRunId === runId; - if (!owned && !cancelled) return true; + var workers = roles.workers || [roles.worker]; + var cancelled = 0, owned = []; + for (var wi = 0; wi < workers.length; wi++) { + cancelled += taskControl.cancelAll(workers[wi], runId); + if (workers[wi]._pairDelegation && workers[wi]._pairDelegation.autonomousRunId === runId) owned.push(workers[wi]); + } + if (!owned.length && !cancelled) return true; turnControl.markHumanStop(session); - if (!owned) return true; - taskControl.markInterruption(roles.worker, "user", "The Until complete run was stopped."); - workerPermission.cancelForSession(roles.worker, "The Until complete run was stopped."); - roles.worker.taskStopRequested = true; - if (roles.worker.abortController) roles.worker.abortController.abort(); + if (!owned.length) return true; + for (var oi = 0; oi < owned.length; oi++) { + taskControl.markInterruption(owned[oi], "user", "The Until complete run was stopped."); + workerPermission.cancelForSession(owned[oi], "The Until complete run was stopped."); + owned[oi].taskStopRequested = true; + if (owned[oi].abortController) owned[oi].abortController.abort(); + } return true; } diff --git a/lib/project-pair-close-transaction.js b/lib/project-pair-close-transaction.js new file mode 100644 index 00000000..40b2f7ec --- /dev/null +++ b/lib/project-pair-close-transaction.js @@ -0,0 +1,59 @@ +var structuralClose = require("./project-pair-structural-close"); + +function attachPairCloseTransaction(ctx) { + function close(args, caller) { + var resolved = ctx.resolvePair(caller, args); + var partner = resolved.partner; + var token = partner._pairDelegation || null; + var interrupted = !!(partner.isProcessing || partner._queryStarting); + var interruption = token && interrupted ? { + source: "driver", + reason: "The Driver closed the Split Worker pair.", + targetTaskId: token.taskId, + requestedAt: Date.now(), + } : null; + var response = token ? ctx.responseText(partner.history || [], token.startIndex) : ""; + var failure = token ? ctx.errorSince(partner.history || [], token.startIndex) : null; + var outcome = token ? ctx.taskControl.prepareCompletion(partner, caller, token, + interrupted ? "interrupted" : "completed", response, failure, interruption) : null; + var prepared = token ? ctx.resultCapture.prepare(caller, partner, token, outcome) : { ok: true, prepared: false }; + if (!prepared.ok) throw new Error(prepared.error || "could not durably capture the Worker outcome"); + var postCommitError = null; + var result = structuralClose.removeSelectedWorker(ctx.store, resolved.group, caller, partner, + ctx.multiWorkerFeature, { afterPersist: function () { + try { + if (token) { + var finalized = ctx.resultCapture.finalizePrepared(token, outcome); + if (!finalized.ok) throw new Error(finalized.error || "prepared close outcome could not be committed"); + partner._pairClosing = true; + token.response = response; + token.failure = failure; + token.interrupted = interrupted && !failure; + ctx.taskControl.commitCompletion(partner, caller, token, outcome); + ctx.resultCapture.acceptPrepared(caller, partner, token); + } + } catch (error) { postCommitError = error; } + finally { delete partner._pairClosing; } + return { ok: true }; + } }); + if (!result.ok) { + if (token && prepared.prepared && !prepared.duplicate) { + var rolledBack = ctx.resultCapture.rollbackPrepared(token, outcome); + if (!rolledBack.ok) throw new Error((result.error || "could not close the Worker pair") + + "; prepared result rollback failed: " + (rolledBack.error || "unknown error")); + } + throw new Error(result.error || "could not close the Worker pair"); + } + if (interrupted) { + partner.taskStopRequested = true; + if (partner.abortController) partner.abortController.abort(); + } + ctx.workerPermission.cancelForSession(partner, "The Driver closed the Split Worker pair."); + if (postCommitError) partner._pairCloseFinalizationError = postCommitError.message || String(postCommitError); + return { status: "closed", partnerId: partner.localId, interrupted: interrupted, + historyPreserved: true }; + } + return { close: close }; +} + +module.exports = { attachPairCloseTransaction: attachPairCloseTransaction }; diff --git a/lib/project-pair-global-stop.js b/lib/project-pair-global-stop.js new file mode 100644 index 00000000..bb93a077 --- /dev/null +++ b/lib/project-pair-global-stop.js @@ -0,0 +1,18 @@ +function stopAllWorkers(session, roles, ctx) { + if (!roles) return false; + for (var i = 0; i < roles.workers.length; i++) { + var worker = roles.workers[i]; + if (worker._pairDelegation && worker._pairDelegation.outboxKey && ctx.resultOutbox) { + ctx.resultOutbox.markBlocked(worker._pairDelegation.outboxKey, "The human stopped this Split Worker turn.", "human_stop"); + } + ctx.taskControl.markInterruption(worker, "user", "The human stopped this Split Worker turn."); + ctx.workerPermission.cancelForSession(worker, "The human stopped this Split Worker turn."); + worker.taskStopRequested = true; + if (worker.abortController) { + try { worker.abortController.abort(); } catch (e) {} + } + } + return true; +} + +module.exports = { stopAllWorkers: stopAllWorkers }; diff --git a/lib/project-pair-lifecycle-status.js b/lib/project-pair-lifecycle-status.js new file mode 100644 index 00000000..a4fc2838 --- /dev/null +++ b/lib/project-pair-lifecycle-status.js @@ -0,0 +1,34 @@ +var pairUsage = require("./project-pair-usage"); + +var MAX_TASK_PREVIEW_CHARS = 200; + +function clampText(value, max) { + var text = typeof value === "string" ? value.replace(/\s+/g, " ").trim() : ""; + if (text.length > max) text = text.slice(0, max - 1) + "…"; + return text; +} + +function continuityStatus(session) { + var history = session.history || []; + var userTurns = 0; + var errors = 0; + for (var i = 0; i < history.length; i++) { + if (!history[i]) continue; + if (history[i].type === "user_message") userTurns++; + else if (history[i].type === "error") errors++; + } + var lastActivity = typeof session.lastActivity === "number" ? session.lastActivity : null; + return { historyEntries: history.length, userTurns: userTurns, errorEntries: errors, + idleSeconds: lastActivity ? Math.max(0, Math.round((Date.now() - lastActivity) / 1000)) : null }; +} + +function activityStatus(session) { + var token = session._pairDelegation || null; + return { isProcessing: !!(session.isProcessing || session._queryStarting), delegated: !!token, + currentTask: token ? clampText(token.message, MAX_TASK_PREVIEW_CHARS) : "", + currentTaskId: token && token.taskId || null, interruption: session._pairInterruption || null, + lastTurnInterrupted: !!session._lastTurnInterrupted }; +} + +module.exports = { activityStatus: activityStatus, clampText: clampText, continuityStatus: continuityStatus, + contextStatus: pairUsage.contextStatus, firstNumber: pairUsage.firstNumber }; diff --git a/lib/project-pair-lifecycle.js b/lib/project-pair-lifecycle.js index 2a8279b9..9ed10f31 100644 --- a/lib/project-pair-lifecycle.js +++ b/lib/project-pair-lifecycle.js @@ -1,12 +1,16 @@ var eligibility = require("./session-driver-eligibility"); var replacementState = require("./project-pair-replacement-state"); -var pairUsage = require("./project-pair-usage"); -var contextStatus = pairUsage.contextStatus; -var firstNumber = pairUsage.firstNumber; +var lifecycleStatus = require("./project-pair-lifecycle-status"); +var contextStatus = lifecycleStatus.contextStatus; +var firstNumber = lifecycleStatus.firstNumber; +var continuityStatus = lifecycleStatus.continuityStatus; +var activityStatus = lifecycleStatus.activityStatus; +var clampText = lifecycleStatus.clampText; +var pairTarget = require("./session-pair-target"); +var splitRoles = require("./session-split-group-roles"); var MAX_GENERATIONS = 5; var MAX_NOTE_CHARS = 400; -var MAX_TASK_PREVIEW_CHARS = 200; var EVALUATION_OUTCOMES = ["succeeded", "partial", "failed", "abandoned"]; @@ -21,57 +25,23 @@ function toolError(message) { }); } -function clampText(value, max) { - var text = typeof value === "string" ? value.replace(/\s+/g, " ").trim() : ""; - if (text.length > max) text = text.slice(0, max - 1) + "…"; - return text; -} - function replacementFingerprint(args, worker) { return JSON.stringify({ sourceWorkerId: worker.localId, interrupt: args.interrupt === true, vendor: args.workerVendor || "", model: args.workerModel || "", effort: args.workerEffort || "", message: args.message || "", evaluation: args.evaluation || null }); } -function continuityStatus(session) { - var history = session.history || []; - var userTurns = 0; - var errors = 0; - for (var i = 0; i < history.length; i++) { - if (!history[i]) continue; - if (history[i].type === "user_message") userTurns++; - else if (history[i].type === "error") errors++; - } - var lastActivity = typeof session.lastActivity === "number" ? session.lastActivity : null; - return { - historyEntries: history.length, - userTurns: userTurns, - errorEntries: errors, - idleSeconds: lastActivity ? Math.max(0, Math.round((Date.now() - lastActivity) / 1000)) : null, - }; -} - -function activityStatus(session) { - var token = session._pairDelegation || null; - return { - isProcessing: !!(session.isProcessing || session._queryStarting), - delegated: !!token, - currentTask: token ? clampText(token.message, MAX_TASK_PREVIEW_CHARS) : "", - currentTaskId: token && token.taskId || null, - interruption: session._pairInterruption || null, - lastTurnInterrupted: !!session._lastTurnInterrupted, - }; -} - function attachPairLifecycle(ctx) { var sm = ctx.sm; var store = ctx.splitStore; var turnControl = ctx.turnControl; - function ledgerFor(driver) { + function ledgerFor(driver, selectedWorker) { if (!Array.isArray(driver._workerGenerations)) driver._workerGenerations = []; var group = store.groupForMember(driver.localId); - var worker = group && group.pair && sm.sessions.get(group.pair.workerId); + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + var fallbackId = normalized.ok && normalized.kind !== "adhoc" && normalized.workerIds.length === 1 ? normalized.workerIds[0] : null; + var worker = selectedWorker || fallbackId != null && sm.sessions.get(fallbackId); var generation = worker && (worker._pairGeneration || worker.sessionProvenance && worker.sessionProvenance.generation); if (worker && Number.isInteger(generation) && generation > 0 && !findGeneration(driver, worker)) { driver._workerGenerations.push({ @@ -91,7 +61,7 @@ function attachPairLifecycle(ctx) { return driver._workerGenerations; } function recordGenerationStart(driver, worker) { - var ledger = ledgerFor(driver); + var ledger = ledgerFor(driver, worker); var existing = findGeneration(driver, worker); if (existing && !existing.endedAt) { worker._pairGeneration = existing.generation; @@ -166,6 +136,8 @@ function attachPairLifecycle(ctx) { return record; } function resolveDriverPair(caller) { + var args = arguments[1]; + var options = arguments[2]; if (!caller) throw new Error("pair lifecycle tools require a session-bound tool server"); if (sm.sessions.get(caller.localId) !== caller) { throw new Error("this session is no longer live; the pair tools are bound to an exact session"); @@ -176,24 +148,25 @@ function attachPairLifecycle(ctx) { if (!group) throw new Error("this session is not in a split group"); if (!group.pair) throw new Error("this split group has no Driver/Split Worker roles"); if (group.pair.driverId !== caller.localId) throw new Error("only the configured Driver can direct this pair"); - var worker = sm.sessions.get(group.pair.workerId); - if (!worker) throw new Error("split partner session was not found"); + var targetOptions = Object.assign({}, options || { mutation: true }, { + multiWorkerFeature: ctx.multiWorkerFeature, + }); + var resolved = pairTarget.resolveGroupTarget(sm, group, caller, args, targetOptions); + var worker = resolved.partner; if ((caller.ownerId || null) !== (worker.ownerId || null)) throw new Error("split partner access denied"); - return { group: group, worker: worker }; + return { group: resolved.group, worker: resolved.partner, role: resolved.role, workerId: resolved.workerId }; } - function replaceBlockedReason(worker) { if (worker.isProcessing || worker._queryStarting) return "the Split Worker is mid-turn"; if (worker._pairDelegation) return "a delegated task is still open"; return null; } - - function partnerStatus(caller) { - var resolved = resolveDriverPair(caller); + function partnerStatus(caller, args) { + var resolved = resolveDriverPair(caller, args, { readOnly: true }); var worker = resolved.worker; var blocked = turnControl.blockedReason(caller) || replaceBlockedReason(worker); var resumeBlocked = turnControl.blockedReason(caller) || (worker.isProcessing || worker._queryStarting ? "the Split Worker is mid-turn" : (!worker._pairInterruption ? "the Split Worker has no interrupted task" : null)); - var ledger = ledgerFor(caller).map(function (record) { + var ledger = ledgerFor(caller, worker).map(function (record) { return { generation: record.generation, vendor: record.vendor, @@ -245,18 +218,16 @@ function attachPairLifecycle(ctx) { generations: ledger, }; } - function sourcePairRegistered(caller, worker) { try { if (sm.sessions.get(caller.localId) !== caller || sm.sessions.get(worker.localId) !== worker) return false; if ((caller.ownerId || null) !== (worker.ownerId || null)) return false; var group = store.groupForMember(caller.localId); - return !!(group && group.pair && group.pair.driverId === caller.localId && - group.pair.workerId === worker.localId && group.members.indexOf(caller.localId) !== -1 && - group.members.indexOf(worker.localId) !== -1); + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + return !!(normalized.ok && normalized.kind !== "adhoc" && normalized.driverId === caller.localId && + normalized.workerIds.indexOf(worker.localId) !== -1); } catch (e) { return false; } } - function restoreSourcePair(ws, caller, worker) { try { return store.create(ws, { @@ -265,23 +236,25 @@ function attachPairLifecycle(ctx) { }); } catch (e) { return { ok: false, error: e.message || String(e) }; } } - - function replacementTargetRegistered(caller, oldWorker) { + function replacementTargetRegistered(caller, oldWorker, sourceWorkerIds) { try { if (sm.sessions.get(caller.localId) !== caller || sm.sessions.get(oldWorker.localId) !== oldWorker) return null; var group = store.groupForMember(caller.localId); - if (!group || !group.pair || group.pair.driverId !== caller.localId || - group.pair.workerId === oldWorker.localId) return null; - var target = sm.sessions.get(group.pair.workerId); + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok || normalized.kind === "adhoc" || normalized.driverId !== caller.localId) return null; + var candidates = normalized.workerIds.filter(function (id) { return sourceWorkerIds.indexOf(id) === -1; }); + if (candidates.length !== 1) return null; + var target = sm.sessions.get(candidates[0]); if (!target || sm.sessions.get(target.localId) !== target) return null; return (caller.ownerId || null) === (target.ownerId || null) ? target : null; } catch (e) { return null; } } - function replacePartner(args, caller) { - var resolved = resolveDriverPair(caller); + var resolved = resolveDriverPair(caller, args, { mutation: true }); var group = resolved.group; var oldWorker = resolved.worker; + var versioned = resolved.role.kind === "versioned"; + var sourceWorkerIds = resolved.role.workerIds.slice(); ledgerFor(caller); turnControl.assertWorkerAction(caller); var replacementEntry = replacementState.begin(caller, args.transactionId, oldWorker, replacementFingerprint(args, oldWorker)); @@ -336,9 +309,11 @@ function attachPairLifecycle(ctx) { ctx.cancelWorkerPermissions(oldWorker, "The Driver replaced this Split Worker."); } - replacementState.stage(replacementEntry.transaction, "dissolving_source_pair"); - var dissolved = store.dissolve(ws, { id: group.id }); - if (!dissolved.ok) throw new Error(dissolved.error || "could not dissolve the existing pair"); + if (!versioned) { + replacementState.stage(replacementEntry.transaction, "dissolving_source_pair"); + var dissolved = store.dissolve(ws, { id: group.id }); + if (!dissolved.ok) throw new Error(dissolved.error || "could not dissolve the existing pair"); + } } catch (precreateError) { var sourceIntact = sourcePairRegistered(caller, oldWorker); var precreateRestored = sourceIntact ? { ok: true } : restoreSourcePair(ws, caller, oldWorker); @@ -351,13 +326,20 @@ function attachPairLifecycle(ctx) { var created; try { replacementState.stage(replacementEntry.transaction, "creating_target"); - created = ctx.createWorkerForDriver(caller, { + var createReplacement = versioned ? ctx.replaceWorkerForDriver : ctx.createWorkerForDriver; + created = createReplacement(caller, versioned ? oldWorker : { workerVendor: typeof args.workerVendor === "string" ? args.workerVendor : "", workerModel: typeof args.workerModel === "string" ? args.workerModel : "", workerEffort: typeof args.workerEffort === "string" ? args.workerEffort : "", - }); + }, versioned ? { + workerVendor: typeof args.workerVendor === "string" ? args.workerVendor : "", + workerModel: typeof args.workerModel === "string" ? args.workerModel : "", + workerEffort: typeof args.workerEffort === "string" ? args.workerEffort : "", + expectedGroup: group, + expectedWorkerIds: sourceWorkerIds, + } : undefined); } catch (e) { - var target = replacementTargetRegistered(caller, oldWorker); + var target = replacementTargetRegistered(caller, oldWorker, sourceWorkerIds); if (target) { var targetClosed = closeGeneration(caller, oldWorker); var targetGeneration = recordGenerationStart(caller, target); @@ -404,7 +386,7 @@ function attachPairLifecycle(ctx) { var message = typeof args.message === "string" ? args.message.trim() : ""; if (!message) return Promise.resolve(result); - return Promise.resolve(ctx.sendToPartner({ message: message, wait: args.wait, timeoutSeconds: args.timeoutSeconds }, caller)) + return Promise.resolve(ctx.sendToPartner({ message: message, wait: args.wait, timeoutSeconds: args.timeoutSeconds, workerId: created.worker.localId }, caller)) .then(function (delivered) { result.delivery = delivered; return result; @@ -430,8 +412,8 @@ function attachPairLifecycle(ctx) { } function recordEvaluation(args, caller) { - var resolved = resolveDriverPair(caller); - var ledger = ledgerFor(caller); + var resolved = resolveDriverPair(caller, args, { mutation: true }); + var ledger = ledgerFor(caller, resolved.worker); var target = Number.isInteger(args.generation) ? (function () { for (var i = 0; i < ledger.length; i++) { @@ -453,14 +435,14 @@ function attachPairLifecycle(ctx) { }; } - function optionalStatus(boundSession) { - try { return partnerStatus(boundSession); } catch (e) { return null; } + function optionalStatus(boundSession, args) { + try { return partnerStatus(boundSession, args); } catch (e) { return null; } } function toolHandlers(boundSession) { return { - status: function () { - try { return toolResult(partnerStatus(boundSession)); } + status: function (args) { + try { return toolResult(partnerStatus(boundSession, args || {})); } catch (e) { return toolError(e.message || String(e)); } }, replace: function (args) { diff --git a/lib/project-pair-message.js b/lib/project-pair-message.js index 125be800..a1e0564f 100644 --- a/lib/project-pair-message.js +++ b/lib/project-pair-message.js @@ -17,7 +17,7 @@ function attachPairMessage(ctx) { if (!id) return toolResult({ status: "rejected", requestId: null, reason: "requestId is invalid" }); try { ctx.turnControl.assertWorkerAction(caller); - var resolved = ctx.resolvePair(caller); + var resolved = ctx.resolvePair(caller, args, { mutation: true }); var worker = resolved.partner; var message = typeof args.message === "string" ? args.message.trim() : ""; if (!message) return toolResult({ status: "rejected", requestId: id, partnerId: worker.localId, reason: "message is required" }); diff --git a/lib/project-pair-owned-stop.js b/lib/project-pair-owned-stop.js index 92fc3b4b..950ddb88 100644 --- a/lib/project-pair-owned-stop.js +++ b/lib/project-pair-owned-stop.js @@ -1,10 +1,13 @@ function stopOwnedPair(session, roles, turnControl, taskControl, workerPermission, reason) { if (!roles) return false; turnControl.markHumanStop(session); - taskControl.cancelAll(roles.worker); - taskControl.markInterruption(roles.worker, "user", reason || "The human stopped this scheduled run."); - workerPermission.cancelForSession(roles.worker, reason || "The human stopped this scheduled run."); - var members = [roles.driver, roles.worker]; + var workers = roles.workers || [roles.worker]; + for (var wi = 0; wi < workers.length; wi++) { + taskControl.cancelAll(workers[wi]); + taskControl.markInterruption(workers[wi], "user", reason || "The human stopped this scheduled run."); + workerPermission.cancelForSession(workers[wi], reason || "The human stopped this scheduled run."); + } + var members = [roles.driver].concat(workers); for (var i = 0; i < members.length; i++) { members[i].taskStopRequested = true; if (members[i].queryInstance && typeof members[i].queryInstance.close === "function") { diff --git a/lib/project-pair-result-capture.js b/lib/project-pair-result-capture.js index df096626..b1379599 100644 --- a/lib/project-pair-result-capture.js +++ b/lib/project-pair-result-capture.js @@ -1,4 +1,10 @@ function attachPairResultCapture(ctx) { + var splitRoles = require("./session-split-group-roles"); + function current(caller, partner, token) { + var group = ctx.store.groupForMember(partner.localId); + return !!(group && group.id === token.groupId && partner._pairDelegation === token && + splitRoles.matchesLiveTarget(ctx.sm, group, caller, partner) && splitRoles.tokenMatchesWorker(partner, token)); + } function begin(caller, partner, token, message, projectSlug) { if (!ctx.resultOutbox) return { ok: true }; var driverOriginId = ctx.resultOutbox.stableOrigin(caller), workerOriginId = ctx.resultOutbox.stableOrigin(partner); @@ -6,14 +12,15 @@ function attachPairResultCapture(ctx) { var started = ctx.resultOutbox.begin({ ownerId: caller.ownerId, projectSlug: projectSlug, driverOriginId: driverOriginId, workerOriginId: workerOriginId, taskId: token.taskId, generation: token.generation, driverSessionId: caller.localId, - workerSessionId: partner.localId, historyStartIndex: token.startIndex, message: message, + workerSessionId: partner.localId, groupId: token.groupId, + historyStartIndex: token.startIndex, message: message, deliveryRoute: token.wait === true ? "blocking" : "callback" }); if (started.ok) token.outboxKey = started.key; return started; } function finish(caller, partner, token, delivered) { var group = ctx.store.groupForMember(partner.localId); - if (token._captureFinished || partner._pairDelegation !== token || !group || group.id !== token.groupId || group.members.indexOf(caller.localId) === -1 || group.members.indexOf(partner.localId) === -1 || (group.pair && (group.pair.driverId !== caller.localId || group.pair.workerId !== partner.localId)) || ctx.sm.sessions.get(caller.localId) !== caller || ctx.sm.sessions.get(partner.localId) !== partner || caller.ownerId !== partner.ownerId) return false; + if (token._captureFinished || !current(caller, partner, token)) return false; token._captureFinished = true; try { if (typeof ctx.onPartnerResult === "function") ctx.onPartnerResult(caller, partner, token); @@ -37,7 +44,7 @@ function attachPairResultCapture(ctx) { } delete token._capturePending; var resumed = delivered ? true : ctx.resumeDriverWithResult(caller, partner, token); - ctx.drain(caller, partner); + if (!partner._pairClosing) ctx.drain(caller, partner); return resumed; } function capture(caller, partner, token, outcome) { @@ -50,12 +57,12 @@ function attachPairResultCapture(ctx) { token._captureInProgress = false; if (retryResult && retryResult.ok) { token._captureComplete = true; - var record = ctx.resultOutbox.get(token.outboxKey); - if (typeof ctx.onStateChange === "function") ctx.onStateChange(caller, record); - if (record && record.deliveryRoute === "callback") deliver(caller, partner, token); + var capturedRecord = ctx.resultOutbox.get(token.outboxKey); + if (typeof ctx.onStateChange === "function") ctx.onStateChange(caller, capturedRecord); + if (capturedRecord && capturedRecord.deliveryRoute === "callback") deliver(caller, partner, token); else finish(caller, partner, token); } - }); + }, function () { return current(caller, partner, token); }); } catch (error) { token._captureInProgress = false; token._capturePending = true; @@ -64,9 +71,9 @@ function attachPairResultCapture(ctx) { } if (saved.ok) { token._captureInProgress = false; token._captureComplete = true; } if (saved.ok) { - var current = ctx.resultOutbox.get(token.outboxKey); - if (typeof ctx.onStateChange === "function") ctx.onStateChange(caller, current); - var delivered = current && current.deliveryRoute === "callback" ? deliver(caller, partner, token) : finish(caller, partner, token); + var capturedRecord = ctx.resultOutbox.get(token.outboxKey); + if (typeof ctx.onStateChange === "function") ctx.onStateChange(caller, capturedRecord); + var delivered = capturedRecord && capturedRecord.deliveryRoute === "callback" ? deliver(caller, partner, token) : finish(caller, partner, token); return delivered === false && token._capturePending ? false : delivered; } token._capturePending = true; @@ -76,11 +83,34 @@ function attachPairResultCapture(ctx) { } return false; } + function prepare(caller, partner, token, outcome) { + if (!ctx.resultOutbox || !token.outboxKey) return { ok: true, prepared: false }; + var saved = ctx.resultOutbox.prepareCapture(token.outboxKey, outcome); + return saved.ok ? { ok: true, prepared: true, duplicate: !!saved.duplicate } : saved; + } + function rollbackPrepared(token, outcome) { + if (!ctx.resultOutbox || !token.outboxKey) return { ok: true }; + return ctx.resultOutbox.rollbackPreparedCapture(token.outboxKey, outcome); + } + function finalizePrepared(token, outcome) { + if (!ctx.resultOutbox || !token.outboxKey) return { ok: true, record: null }; + return ctx.resultOutbox.finalizePreparedCapture(token.outboxKey, outcome); + } + function acceptPrepared(caller, partner, token) { + if (ctx.resultOutbox && token.outboxKey) { + token._captureComplete = true; + var record = ctx.resultOutbox.get(token.outboxKey); + if (typeof ctx.onStateChange === "function") ctx.onStateChange(caller, record); + return record && record.deliveryRoute === "callback" ? deliver(caller, partner, token) : finish(caller, partner, token); + } + return finish(caller, partner, token); + } function deliver(caller, partner, token) { if (!ctx.delivery) return finish(caller, partner, token); return ctx.delivery.wake(caller, partner, token); } - return { begin: begin, capture: capture, deliver: deliver, finish: finish }; + return { begin: begin, capture: capture, prepare: prepare, finalizePrepared: finalizePrepared, rollbackPrepared: rollbackPrepared, + acceptPrepared: acceptPrepared, deliver: deliver, finish: finish }; } module.exports = { attachPairResultCapture: attachPairResultCapture }; diff --git a/lib/project-pair-result-delivery.js b/lib/project-pair-result-delivery.js index 863a7dc8..ccd061c7 100644 --- a/lib/project-pair-result-delivery.js +++ b/lib/project-pair-result-delivery.js @@ -1,5 +1,6 @@ var RETRY_DELAY_MS = 50; var DEFAULT_ACCEPTANCE_TIMEOUT_MS = 15000; +var splitRoles = require("./session-split-group-roles"); function attachPairResultDelivery(ctx) { function recordMatches(caller, partner, token, record) { @@ -9,9 +10,9 @@ function attachPairResultDelivery(ctx) { } function valid(caller, partner, token) { var group = ctx.store.groupForMember(partner.localId), record = ctx.outbox.get(token.outboxKey); - return !!(group && group.id === token.groupId && group.members.indexOf(caller.localId) !== -1 && - group.members.indexOf(partner.localId) !== -1 && group.pair && group.pair.driverId === caller.localId && - group.pair.workerId === partner.localId && ctx.sm.sessions.get(caller.localId) === caller && + return !!(group && group.id === token.groupId && splitRoles.matchesLiveTarget(ctx.sm, group, caller, partner) && + splitRoles.tokenMatchesWorker(partner, token) && + ctx.sm.sessions.get(caller.localId) === caller && ctx.sm.sessions.get(partner.localId) === partner && caller.ownerId === partner.ownerId && !caller.destroying && !partner.destroying && !ctx.blockedReason(caller) && partner._pairDelegation === token && recordMatches(caller, partner, token, record)); diff --git a/lib/project-pair-result-outbox.js b/lib/project-pair-result-outbox.js index 0fd54b11..d4b34885 100644 --- a/lib/project-pair-result-outbox.js +++ b/lib/project-pair-result-outbox.js @@ -70,6 +70,8 @@ function attachPairResultOutbox(options) { } var nextRecords = Object.assign({}, records), next = Object.assign({}, record, { outcome: copy(outcome), state: record.state === "blocked" ? "blocked" : "captured", capturedAt: Date.now(), + preparedCloseOutcome: null, + closePreparationState: record.preparedCloseOutcome ? "superseded_by_terminal_evidence" : record.closePreparationState || null, }); nextRecords[record.key] = next; var saved = persist(nextRecords); @@ -87,6 +89,15 @@ function attachPairResultOutbox(options) { function retryCapture(key, operation, attempt) { attempt = attempt || 1; if (captureOperations[key] !== operation || !operation.active) return { ok: false, stale: true }; + var isCurrent = true; + try { if (operation.isCurrent) isCurrent = operation.isCurrent() === true; } catch (error) { isCurrent = false; } + if (!isCurrent) { + operation.active = false; + var stale = { ok: false, stale: true, error: "pair result capture no longer matches the live Worker task" }; + operation.lastResult = stale; + notifyCapture(operation, stale); + return stale; + } var result = captureRecord(key, operation.outcome); operation.lastResult = result; if (result.ok) { @@ -118,6 +129,7 @@ function attachPairResultOutbox(options) { generation: args.generation == null ? null : args.generation, driverSessionId: args.driverSessionId == null ? null : args.driverSessionId, workerSessionId: args.workerSessionId == null ? null : args.workerSessionId, + groupId: typeof args.groupId === "string" ? args.groupId : null, workerSessionOriginId: args.workerOriginId, historyStartIndex: args.historyStartIndex == null ? null : args.historyStartIndex, message: args.message || "", @@ -143,7 +155,7 @@ function attachPairResultOutbox(options) { records = nextRecords; return { ok: true, key: key, record: copy(record), duplicate: false }; } - function capture(args, outcome, callback) { + function capture(args, outcome, callback, isCurrent) { var key = typeof args === "string" ? args : resultKey(args); var record = records[key]; if (!record) return { ok: false, error: "pair result outbox record was not found" }; @@ -158,11 +170,65 @@ function attachPairResultOutbox(options) { addCaptureCallback(operation, callback); return retryCapture(key, operation, 1); } - operation = { outcome: copy(outcome), callbacks: [], active: true, complete: false, lastResult: null, timer: null }; + operation = { outcome: copy(outcome), callbacks: [], active: true, complete: false, lastResult: null, timer: null, + isCurrent: typeof isCurrent === "function" ? isCurrent : null }; captureOperations[key] = operation; addCaptureCallback(operation, callback); return retryCapture(key, operation, 1); } + function prepareCapture(key, outcome) { + var record = records[key]; + if (!record) return { ok: false, error: "pair result outbox record was not found" }; + if (record.outcome) { + if (!sameOutcome(record.outcome, outcome)) return { ok: false, conflict: true, error: "pair result outcome is immutable", record: copy(record) }; + return { ok: true, record: copy(record), duplicate: true, committed: true }; + } + if (record.preparedCloseOutcome) { + if (!sameOutcome(record.preparedCloseOutcome, outcome)) return { ok: false, conflict: true, error: "prepared close outcome changed", record: copy(record) }; + return { ok: true, record: copy(record), duplicate: true, prepared: true }; + } + var nextRecords = Object.assign({}, records), next = Object.assign({}, record, { + state: "close_prepared", preparedCloseOutcome: copy(outcome), + closePreparationState: "prepared", closePreparedAt: Date.now(), + }); + nextRecords[key] = next; + var saved = persist(nextRecords); + if (!saved.ok) return { ok: false, error: saved.error, record: copy(record) }; + records = nextRecords; + return { ok: true, record: copy(next), duplicate: false, prepared: true }; + } + function finalizePreparedCapture(key, outcome) { + var record = records[key]; + if (!record) return { ok: false, error: "pair result outbox record was not found" }; + if (record.outcome) return captureRecord(key, outcome); + if (!record.preparedCloseOutcome || !sameOutcome(record.preparedCloseOutcome, outcome)) { + return { ok: false, error: "prepared close outcome is unavailable or changed", record: copy(record) }; + } + var nextRecords = Object.assign({}, records), next = Object.assign({}, record, { + outcome: copy(outcome), state: "captured", capturedAt: Date.now(), + preparedCloseOutcome: null, closePreparationState: "committed", closeCommittedAt: Date.now(), + }); + nextRecords[key] = next; + var saved = persist(nextRecords); + if (!saved.ok) return { ok: false, error: saved.error, record: copy(record) }; + records = nextRecords; + return { ok: true, record: copy(next), duplicate: false }; + } + function rollbackPreparedCapture(key, outcome) { + var record = records[key]; + if (!record || !sameOutcome(record.preparedCloseOutcome, outcome) || record.outcome || record.finalized || record.transcriptPersisted || + record.deliveryAttemptCount !== 0 || record.deliveryState !== "pending") { + return { ok: false, error: "prepared pair result can no longer be rolled back" }; + } + var nextRecords = Object.assign({}, records), next = Object.assign({}, record, { + state: "dispatching", preparedCloseOutcome: null, closePreparationState: null, closePreparedAt: null, + }); + nextRecords[key] = next; + var saved = persist(nextRecords); + if (!saved.ok) return { ok: false, error: saved.error, record: copy(record) }; + records = nextRecords; + return { ok: true, record: copy(next) }; + } function transition(key, changes) { var record = records[key]; if (!record) return { ok: false, error: "pair result outbox record was not found" }; @@ -229,7 +295,7 @@ function attachPairResultOutbox(options) { function recoverPending(sessionMap) { var results = [], sessionsList = sessionMap && typeof sessionMap.values === "function" ? Array.from(sessionMap.values()) : [], keys = Object.keys(records); for (var i = 0; i < keys.length; i++) { - if (records[keys[i]].state !== "dispatching") continue; + if (records[keys[i]].state !== "dispatching" && records[keys[i]].state !== "close_prepared") continue; var session = sessionsList.find(function (candidate) { return stableOrigin(candidate) === records[keys[i]].workerOriginId && candidate.ownerId === records[keys[i]].ownerId; }); if (session) results.push({ key: keys[i], result: recover(keys[i], session) }); } @@ -294,7 +360,7 @@ function attachPairResultOutbox(options) { return Object.keys(source).map(function (key) { var record = copy(source[key]); record.deliveryState = record.deliveryState || "pending"; - record.recoveryState = record.state === "dispatching" || record.deliveryState === "attempting" || record.deliveryState === "uncertain" ? "uncertain" : "known"; + record.recoveryState = record.state === "dispatching" || record.state === "close_prepared" || record.deliveryState === "attempting" || record.deliveryState === "uncertain" ? "uncertain" : "known"; record.blockedReason = record.blockedReason || null; return record; }); @@ -306,6 +372,9 @@ function attachPairResultOutbox(options) { return { begin: begin, capture: capture, + prepareCapture: prepareCapture, + finalizePreparedCapture: finalizePreparedCapture, + rollbackPreparedCapture: rollbackPreparedCapture, beginDelivery: beginDelivery, markTranscript: markTranscript, markDelivery: markDelivery, diff --git a/lib/project-pair-result-recovery.js b/lib/project-pair-result-recovery.js index 948c2cbf..bd873535 100644 --- a/lib/project-pair-result-recovery.js +++ b/lib/project-pair-result-recovery.js @@ -1,4 +1,5 @@ var crypto = require("crypto"); +var splitRoles = require("./session-split-group-roles"); function attachPairResultRecovery(ctx) { var scheduled = Object.create(null); @@ -23,8 +24,9 @@ function attachPairResultRecovery(ctx) { if (!driver || !worker) return { ok: false, code: "session", reason: "The original Driver or Split Worker session is unavailable." }; if (ownerId(driver) !== (record.ownerId || null) || ownerId(worker) !== (record.ownerId || null)) return { ok: false, code: "owner", reason: "The original session owner no longer matches." }; var group = ctx.store.groupForMember(driver.localId); - if (!group || ctx.store.groupForMember(worker.localId) !== group || !group.pair || - group.pair.driverId !== driver.localId || group.pair.workerId !== worker.localId) { + var roles = splitRoles.resolveLiveRoles(ctx.sm, group); + if (!group || ctx.store.groupForMember(worker.localId) !== group || !roles || + roles.driver !== driver || roles.workers.indexOf(worker) === -1) { return { ok: false, code: "pair", reason: "The original Driver and Split Worker pair is no longer active." }; } var provenance = worker.sessionProvenance; @@ -49,6 +51,7 @@ function attachPairResultRecovery(ctx) { } token = { outboxKey: record.key, taskId: record.taskId, generation: record.generation, + workerSessionId: resolved.worker.localId, workerId: resolved.worker.localId, from: resolved.driver.localId, groupId: resolved.group.id, startIndex: Number.isInteger(record.historyStartIndex) ? record.historyStartIndex : resolved.worker.history.length, message: record.message || "", wait: record.deliveryRoute === "blocking", @@ -118,6 +121,7 @@ function attachPairResultRecovery(ctx) { return runtimeStatus[record.key] ? Object.assign({}, record, runtimeStatus[record.key]) : record; }).filter(function (record) { if (record.projectSlug !== ctx.projectSlug || record.driverOriginId !== originId || (record.ownerId || null) !== currentOwner) return false; + if (record.state === "close_prepared") return false; if (healthyLiveDispatch(record)) return false; if (record.finalized) return false; if (record.deliveryState === "accepted") return record.state === "blocked" || record.finalizationState === "failed"; diff --git a/lib/project-pair-structural-close.js b/lib/project-pair-structural-close.js new file mode 100644 index 00000000..c7314519 --- /dev/null +++ b/lib/project-pair-structural-close.js @@ -0,0 +1,20 @@ +var splitRoles = require("./session-split-group-roles"); + +function removeSelectedWorker(store, group, driver, worker, capability, hooks) { + var normalized = splitRoles.normalizePair(group.pair, group.members); + if (!normalized.ok || normalized.kind !== "versioned") { + var ws = { _clayUser: driver.ownerId ? { id: driver.ownerId } : null }; + return store.dissolve(ws, { id: group.id }, hooks); + } + return store.removeWorker(driver.ownerId || null, { + groupId: group.id, + driverId: driver.localId, + driverOriginId: driver.sessionOriginId || null, + workerId: worker.localId, + workerOriginId: worker.sessionOriginId || null, + generation: worker._pairGeneration || + worker.sessionProvenance && worker.sessionProvenance.generation || null, + }, capability, hooks); +} + +module.exports = { removeSelectedWorker: removeSelectedWorker }; diff --git a/lib/project-pair-task-control.js b/lib/project-pair-task-control.js index 39879ca4..3dfecd5e 100644 --- a/lib/project-pair-task-control.js +++ b/lib/project-pair-task-control.js @@ -43,6 +43,9 @@ function attachPairTaskControl(ctx) { function taskGeneration(worker) { return worker._pairGeneration || worker.sessionProvenance && worker.sessionProvenance.generation || null; } + function matchesGeneration(worker, task) { + return !!(worker && task && task.workerSessionId === worker.localId && task.generation === taskGeneration(worker)); + } function findTask(worker, taskId) { var active = current(worker); if (active && active.taskId === taskId) return active; @@ -65,6 +68,7 @@ function attachPairTaskControl(ctx) { taskId: taskId, generation: taskGeneration(worker), workerSessionId: worker.localId, + workerId: worker.localId, driverSessionId: caller.localId, message: message, autonomousRunId: caller.autonomousRun && caller.autonomousRun.id || null, @@ -89,7 +93,7 @@ function attachPairTaskControl(ctx) { function queue(args, caller) { try { ctx.turnControl.assertWorkerAction(caller); - var resolved = ctx.resolvePair(caller); + var resolved = ctx.resolvePair(caller, args, { mutation: true }); var worker = resolved.partner; var message = cleanText(args.message, 30000); if (!message) throw new Error("message is required"); @@ -104,14 +108,14 @@ function attachPairTaskControl(ctx) { } function inspect(args, caller) { try { - var worker = ctx.resolvePair(caller).partner; + var worker = ctx.resolvePair(caller, args, { readOnly: true }).partner; return result(status(worker)); } catch (err) { return result({ status: "rejected", reason: err.message || String(err) }); } } function cancel(args, caller) { try { ctx.turnControl.assertWorkerAction(caller); - var worker = ctx.resolvePair(caller).partner; + var worker = ctx.resolvePair(caller, args, { mutation: true }).partner; var task = findTask(worker, args.taskId); if (!task || (task.status !== "queued" && task.status !== "waiting_for_interrupt")) throw new Error("the exact queued task is not cancellable"); task.status = "cancelled"; @@ -150,7 +154,7 @@ function attachPairTaskControl(ctx) { function replaceTask(args, caller) { try { ctx.turnControl.assertWorkerAction(caller); - var resolved = ctx.resolvePair(caller); + var resolved = ctx.resolvePair(caller, args, { mutation: true }); var worker = resolved.partner; var active = current(worker); if (!active || !worker.isProcessing) throw new Error("there is no active delegated task to replace"); @@ -170,12 +174,12 @@ function attachPairTaskControl(ctx) { function resumeTask(args, caller) { try { ctx.turnControl.assertWorkerAction(caller); - var worker = ctx.resolvePair(caller).partner; + var worker = ctx.resolvePair(caller, args, { mutation: true }).partner; if (worker.isProcessing || worker._queryStarting || current(worker)) throw new Error("the Split Worker is not idle"); var prior = worker._lastInterruptedPairTask; if (!prior) throw new Error("the Split Worker has no resumable interrupted task"); var message = cleanText(args.message, 30000) || prior.message; - return ctx.sendFollowup({ message: message, wait: args.wait, timeoutSeconds: args.timeoutSeconds, taskId: args.taskId }, caller); + return ctx.sendFollowup({ message: message, wait: args.wait, timeoutSeconds: args.timeoutSeconds, taskId: args.taskId, workerId: worker.localId }, caller); } catch (err) { return result({ status: "rejected", reason: err.message || String(err) }); } } function report(args, worker) { @@ -185,6 +189,7 @@ function attachPairTaskControl(ctx) { if (!resolved || resolved.worker !== worker) throw new Error("only the exact configured Split Worker can report an outcome"); var active = current(worker); if (!active || args.taskId !== active.taskId) throw new Error("taskId must match the exact active delegated task"); + if (!matchesGeneration(worker, active)) throw new Error("the active task no longer matches this Worker generation"); var outcome = typeof args.outcome === "string" ? args.outcome.trim().toLowerCase() : ""; if (OUTCOMES.indexOf(outcome) === -1) throw new Error("outcome must be completed, partial, failed, or interrupted"); active.reportedOutcome = { @@ -200,11 +205,12 @@ function attachPairTaskControl(ctx) { return result({ status: "recorded", taskId: active.taskId }); } catch (err) { return result({ status: "rejected", reason: err.message || String(err) }); } } - function complete(worker, caller, token, observedStatus, response, error) { + function prepareCompletion(worker, caller, token, observedStatus, response, error, forcedInterruption) { if (!token) return null; if (token.outcomeEnvelope) return token.outcomeEnvelope; - var interruption = worker._pairInterruption && worker._pairInterruption.targetTaskId === token.taskId ? worker._pairInterruption : null; - if (interruption) interruption.completedAt = Date.now(); + var currentInterruption = forcedInterruption || + (worker._pairInterruption && worker._pairInterruption.targetTaskId === token.taskId ? worker._pairInterruption : null); + var interruption = currentInterruption ? Object.assign({}, currentInterruption, { completedAt: Date.now() }) : null; var report = token.reportedOutcome || null; var finalStatus = error ? "failed" : (interruption ? "interrupted" : (report ? report.outcome : observedStatus)); var envelope = { @@ -224,10 +230,16 @@ function attachPairTaskControl(ctx) { response: response || "", error: error || null, }; - token.status = finalStatus; + return envelope; + } + function commitCompletion(worker, caller, token, envelope) { + if (!token || !envelope) return envelope || null; + if (token.outcomeEnvelope) return token.outcomeEnvelope; + token.status = envelope.status; token.outcomeEnvelope = envelope; worker._lastPairOutcome = envelope; - if (finalStatus === "interrupted" || finalStatus === "partial") worker._lastInterruptedPairTask = { taskId: token.taskId, message: token.message, generation: token.generation }; + if (envelope.interruption) worker._pairInterruption = envelope.interruption; + if (envelope.status === "interrupted" || envelope.status === "partial") worker._lastInterruptedPairTask = { taskId: token.taskId, message: token.message, generation: token.generation }; if (ctx.sm.sendAndRecord) { try { ctx.sm.sendAndRecord(worker, { type: "pair_task_completed", _internal: true, ownerId: caller.ownerId || null, driverOriginId: caller.sessionOriginId || null, @@ -237,6 +249,10 @@ function attachPairTaskControl(ctx) { if (ctx.sm.sendToSession) ctx.sm.sendToSession(caller, { type: "partner_task_completed", outcome: envelope }); return envelope; } + function complete(worker, caller, token, observedStatus, response, error) { + return commitCompletion(worker, caller, token, + prepareCompletion(worker, caller, token, observedStatus, response, error)); + } function status(worker) { var active = current(worker); var registry = registryFor(worker); @@ -261,23 +277,28 @@ function attachPairTaskControl(ctx) { next.status = "starting"; next.startedAt = Date.now(); queue.splice(queue.indexOf(next), 1); - Promise.resolve(ctx.sendFollowup({ message: next.message, wait: false, taskId: next.taskId }, caller)).then(function (delivery) { + Promise.resolve(ctx.sendFollowup({ message: next.message, wait: false, taskId: next.taskId, workerId: next.workerId }, caller)).then(function (delivery) { next.status = delivery && delivery.isError ? "failed" : "running"; if (delivery && delivery.isError) next.error = delivery.content && delivery.content[0] && delivery.content[0].text || "Follow-up rejected"; }, function (err) { next.status = "failed"; next.error = err.message || String(err); }); return true; } function driverToolDefs(handlers) { - return [{ name: "queue_partner_followup", description: "Queue a distinct follow-up task behind the active Split Worker task.", inputSchema: buildShape({ message: { type: "string" }, taskId: { type: "string" } }, ["message"]), handler: handlers.queue }, - { name: "inspect_partner_followups", description: "Inspect current, queued, interrupted, and last completed Split Worker task state.", inputSchema: buildShape({}), handler: handlers.inspect }, - { name: "cancel_partner_followup", description: "Cancel one exact queued follow-up without interrupting active work.", inputSchema: buildShape({ taskId: { type: "string" } }, ["taskId"]), handler: handlers.cancel }, - { name: "replace_partner_task", description: "Interrupt one exact active delegated task and queue a replacement task in the same Worker generation.", inputSchema: buildShape({ targetTaskId: { type: "string" }, message: { type: "string" }, taskId: { type: "string" }, reason: { type: "string" } }, ["targetTaskId", "message"]), handler: handlers.replace }, - { name: "resume_partner_task", description: "Resume the last interrupted task as a new correlated task. A human Stop blocks this until a new user turn.", inputSchema: buildShape({ message: { type: "string" }, taskId: { type: "string" }, wait: { type: "boolean" }, timeoutSeconds: { type: "number" } }), handler: handlers.resume }]; + var workerId = { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }; + return [{ name: "queue_partner_followup", description: "Queue a distinct follow-up task behind the active Split Worker task.", inputSchema: buildShape({ message: { type: "string" }, taskId: { type: "string" }, workerId: workerId }, ["message"]), handler: handlers.queue }, + { name: "inspect_partner_followups", description: "Inspect current, queued, interrupted, and last completed Split Worker task state.", inputSchema: buildShape({ workerId: workerId }), handler: handlers.inspect }, + { name: "cancel_partner_followup", description: "Cancel one exact queued follow-up without interrupting active work.", inputSchema: buildShape({ taskId: { type: "string" }, workerId: workerId }, ["taskId"]), handler: handlers.cancel }, + { name: "replace_partner_task", description: "Interrupt one exact active delegated task and queue a replacement task in the same Worker generation.", inputSchema: buildShape({ targetTaskId: { type: "string" }, message: { type: "string" }, taskId: { type: "string" }, reason: { type: "string" }, workerId: workerId }, ["targetTaskId", "message"]), handler: handlers.replace }, + { name: "resume_partner_task", description: "Resume the last interrupted task as a new correlated task. A human Stop blocks this until a new user turn.", inputSchema: buildShape({ message: { type: "string" }, taskId: { type: "string" }, wait: { type: "boolean" }, timeoutSeconds: { type: "number" }, workerId: workerId }), handler: handlers.resume }]; } function workerToolDefs(worker) { return [{ name: "report_partner_outcome", description: "Report structured, Worker-observed task outcome data. This does not claim Driver verification.", inputSchema: buildShape({ taskId: { type: "string" }, outcome: { type: "string" }, summary: { type: "string" }, changedFiles: { type: "array", items: { type: "string" } }, verifications: { type: "array", items: { type: "object" } }, unverified: { type: "array", items: { type: "string" } }, processes: { type: "array", items: { type: "string" } }, nextAction: { type: "string" } }, ["taskId", "outcome"]), handler: function (args) { return report(args || {}, worker); } }]; } - return { begin: begin, cancel: cancel, cancelAll: cancelAll, complete: complete, drain: drain, driverToolDefs: driverToolDefs, ensureSystemInterruption: ensureSystemInterruption, inspect: inspect, markInterruption: markInterruption, queue: queue, replaceTask: replaceTask, resumeTask: resumeTask, status: status, workerToolDefs: workerToolDefs }; + return { begin: begin, cancel: cancel, cancelAll: cancelAll, commitCompletion: commitCompletion, complete: complete, + prepareCompletion: prepareCompletion, drain: drain, driverToolDefs: driverToolDefs, + ensureSystemInterruption: ensureSystemInterruption, inspect: inspect, markInterruption: markInterruption, + matchesGeneration: matchesGeneration, queue: queue, replaceTask: replaceTask, + resumeTask: resumeTask, status: status, workerToolDefs: workerToolDefs }; } module.exports = { attachPairTaskControl: attachPairTaskControl }; diff --git a/lib/project-pair-worker-close.js b/lib/project-pair-worker-close.js new file mode 100644 index 00000000..0b5e5e69 --- /dev/null +++ b/lib/project-pair-worker-close.js @@ -0,0 +1,68 @@ +var multiWorkerFeature = require("./multi-worker-feature"); +var splitRoles = require("./session-split-group-roles"); +var generationOf = require("./session-split-group-v2-store").generationOf; + +function attachPairWorkerClose(ctx) { + function result(ws, msg, ok, error) { + ctx.sendTo(ws, { + type: "split_worker_close_result", + requestId: typeof msg.requestId === "string" ? msg.requestId : null, + ok: ok, + error: error || null, + groupId: typeof msg.groupId === "string" ? msg.groupId : null, + workerId: Number.isInteger(msg.workerId) ? msg.workerId : null, + }); + } + + function validate(ws, msg) { + if (ctx.isMate || !multiWorkerFeature.isEnabled(ctx.multiWorkerFeature)) { + throw new Error("Multi-Worker close is unavailable in this project"); + } + if (!ctx.hasSocket(ws) || ws.readyState !== 1) throw new Error("The project socket is no longer active"); + if (msg.projectSlug !== ctx.projectSlug) throw new Error("The project changed before the Worker was closed"); + if (!Number.isInteger(msg.driverId) || ws._clayActiveSession !== msg.driverId) { + throw new Error("The active session must be the configured Driver"); + } + var driver = ctx.sm.sessions.get(msg.driverId); + if (!driver || ctx.sm.sessions.get(driver.localId) !== driver) throw new Error("The exact Driver session is no longer live"); + var actorId = ws._clayUser && ws._clayUser.id || null; + if ((driver.ownerId || null) !== actorId) throw new Error("The Driver owner does not match this socket"); + if ((msg.driverOriginId || null) !== (driver.sessionOriginId || null)) throw new Error("The exact Driver identity changed"); + var group = ctx.splitStore.groupForMember(driver.localId); + if (!group || group.id !== msg.groupId || (group.ownerId || null) !== actorId) throw new Error("The exact split group changed"); + var roles = splitRoles.normalizePair(group.pair, group.members); + if (!roles.ok || roles.kind !== "versioned" || roles.driverId !== driver.localId) { + throw new Error("The configured Driver or Worker roles changed"); + } + if (!Array.isArray(msg.expectedWorkerIds) || msg.expectedWorkerIds.join(",") !== roles.workerIds.join(",")) { + throw new Error("The configured Worker membership changed"); + } + if (!Number.isInteger(msg.workerId) || roles.workerIds.indexOf(msg.workerId) === -1) { + throw new Error("The selected Worker changed"); + } + var worker = ctx.sm.sessions.get(msg.workerId); + if (!worker || ctx.sm.sessions.get(worker.localId) !== worker || (worker.ownerId || null) !== actorId) { + throw new Error("The exact Worker session is no longer live"); + } + if ((msg.workerOriginId || null) !== (worker.sessionOriginId || null) || msg.generation !== generationOf(worker)) { + throw new Error("The exact Worker identity or generation changed"); + } + return { driver: driver, worker: worker }; + } + + function handleMessage(ws, msg) { + if (!msg || msg.type !== "split_worker_close") return false; + try { + var exact = validate(ws, msg); + ctx.closeWorker({ workerId: exact.worker.localId }, exact.driver); + result(ws, msg, true, null); + } catch (error) { + result(ws, msg, false, error.message || String(error)); + } + return true; + } + + return { handleMessage: handleMessage, validate: validate }; +} + +module.exports = { attachPairWorkerClose: attachPairWorkerClose }; diff --git a/lib/project-session-pair-prompt.js b/lib/project-session-pair-prompt.js new file mode 100644 index 00000000..7c9f8ffd --- /dev/null +++ b/lib/project-session-pair-prompt.js @@ -0,0 +1,26 @@ +var driverEligibility = require("./session-driver-eligibility"); +var driverOrchestration = require("./session-driver-orchestration"); +var prompts = require("./session-pair-prompts"); + +function attachSessionPairPrompt(ctx) { + function getSystemPrompt(session) { + var roles = ctx.rolesFor(session); + if (roles && roles.workers.indexOf(session) !== -1) { + var taskId = session._pairDelegation && session._pairDelegation.taskId || "unknown"; + return prompts.worker(taskId); + } + var group = ctx.store.groupForMember(session.localId); + var result = ""; + if (group && group.pair && group.pair.driverId === session.localId) { + result = prompts.DRIVER_CORE + " " + ctx.workerProposal().getSystemPrompt(session); + if (driverOrchestration.isHighTierDriverSession(session, ctx.sm)) result += " " + prompts.DRIVER_DELEGATION; + } else if (!group && !ctx.isMate && driverEligibility.isEligibleDriverSession(session, ctx.sm) && + driverOrchestration.isHighTierDriverSession(session, ctx.sm)) { + result = prompts.UNPAIRED + " " + ctx.workerProposal().getSystemPrompt(session); + } + return result; + } + return { getSystemPrompt: getSystemPrompt }; +} + +module.exports = { attachSessionPairPrompt: attachSessionPairPrompt }; diff --git a/lib/project-session-pair.js b/lib/project-session-pair.js index 48b17ca4..33dfdf0a 100644 --- a/lib/project-session-pair.js +++ b/lib/project-session-pair.js @@ -1,45 +1,38 @@ -var pairMcp = require("./session-pair-mcp-server"), pairPrompts = require("./session-pair-prompts"); var { attachPairFactory } = require("./session-pair-factory"), { attachPairLifecycle } = require("./project-pair-lifecycle"), { attachPairTurnControl } = require("./session-pair-turn-control"); -var driverEligibility = require("./session-driver-eligibility"), driverOrchestration = require("./session-driver-orchestration"), { attachWorkerPermission } = require("./project-worker-permission"), { attachWorkerProposal } = require("./project-worker-proposal"); +var pairMcp = require("./session-pair-mcp-server"); var { attachPairFactory } = require("./session-pair-factory"), { attachPairLifecycle } = require("./project-pair-lifecycle"), { attachPairTurnControl } = require("./session-pair-turn-control"); +var driverEligibility = require("./session-driver-eligibility"), { attachWorkerPermission } = require("./project-worker-permission"), { attachWorkerProposal } = require("./project-worker-proposal"); +var attachSessionPairPrompt = require("./project-session-pair-prompt").attachSessionPairPrompt; +var attachPairCloseTransaction = require("./project-pair-close-transaction").attachPairCloseTransaction; var { attachPairMessage } = require("./project-pair-message"), { attachPairTaskControl } = require("./project-pair-task-control"), stopAutonomous = require("./project-pair-autonomous-stop").stopAutonomousWork, stopOwned = require("./project-pair-owned-stop").stopOwnedPair; var pairHistory = require("./session-pair-history"), responseText = pairHistory.responseText, errorSince = pairHistory.errorSince, recentTurns = pairHistory.recentTurns, operationFingerprint = require("./session-pair-operation").fingerprint; var beginPairQuery = require("./session-pair-query-start").beginPairQuery, attachPairResultPipeline = require("./project-pair-result-pipeline").attachPairResultPipeline; +var pairTarget = require("./session-pair-target"), splitRoles = require("./session-split-group-roles"), stopAllWorkers = require("./project-pair-global-stop").stopAllWorkers; function toolResult(value) { return Promise.resolve({ content: [{ type: "text", text: JSON.stringify(value) }] }); } function toolError(err) { return Promise.resolve({ content: [{ type: "text", text: "Error: " + (err.message || err) }], isError: true }); } function attachSessionPair(ctx) { - var sm = ctx.sm, store = ctx.splitStore, workerProposal, workerPermission, lifecycle, taskControl; + var sm = ctx.sm, store = ctx.splitStore, workerProposal, workerPermission, lifecycle, taskControl, closeTransaction; var turnControl = attachPairTurnControl({ sm: sm, splitStore: store }), resultCapture; + function targetOptions(options) { return Object.assign({}, options || {}, { multiWorkerFeature: ctx.multiWorkerFeature }); } function groupAndPartner(caller, missingPairMessage) { + var args = arguments[2]; + var options = arguments[3]; if (!caller) throw new Error("partner tools require a session-bound tool server"); if (sm.sessions.get(caller.localId) !== caller) { throw new Error("this session is no longer live; the partner tools are bound to an exact session"); } var group = store.groupForMember(caller.localId); if (!group) throw new Error(missingPairMessage || "this session is not in a split group"); - if (group.pair && group.pair.driverId !== caller.localId) { - throw new Error("only the configured Driver can direct this pair"); - } - var partnerId = group.members[0] === caller.localId ? group.members[1] : group.members[0]; - var partner = sm.sessions.get(partnerId); - if (!partner) throw new Error("split partner session was not found"); - if (caller.ownerId !== partner.ownerId) throw new Error("split partner access denied"); - return { group: group, partner: partner }; + return pairTarget.resolveGroupTarget(sm, group, caller, args, targetOptions(options)); } function rolesFor(session) { - if (!session || sm.sessions.get(session.localId) !== session) return null; - var group = store.groupForMember(session.localId); - if (!group || !group.pair) return null; - var driver = sm.sessions.get(group.pair.driverId); - var worker = sm.sessions.get(group.pair.workerId); - if (!driver || !worker || driver.ownerId !== worker.ownerId) return null; - return { group: group, driver: driver, worker: worker }; + return pairTarget.rolesFor(sm, store, session); } var pairMessage = attachPairMessage({ sm: sm, getSdk: ctx.getSdk, - resolvePair: groupAndPartner, + resolvePair: function (caller, args) { return groupAndPartner(caller, null, args, { mutation: true }); }, turnControl: turnControl, }); taskControl = attachPairTaskControl({ - sm: sm, resolvePair: groupAndPartner, rolesFor: rolesFor, turnControl: turnControl, + sm: sm, resolvePair: function (caller, args, options) { return groupAndPartner(caller, null, args, options); }, rolesFor: rolesFor, turnControl: turnControl, sendFollowup: function (args, caller) { return sendToPartner(args, caller); }, }); function broadcastDelegation(group, caller, partner, active) { @@ -120,11 +113,13 @@ function attachSessionPair(ctx) { var token = partner && partner._pairDelegation; if (!token) return false; if (token._capturePending || token._captureComplete) return false; + if (!taskControl.matchesGeneration(partner, token)) return false; var group = store.groupForMember(partner.localId); if (!group || group.id !== token.groupId || group.members.indexOf(token.from) === -1) return false; var caller = sm.sessions.get(token.from); if (!caller || caller.ownerId !== partner.ownerId) return false; - if (group.pair && (group.pair.driverId !== caller.localId || group.pair.workerId !== partner.localId)) { + var liveRoles = splitRoles.resolveLiveRoles(sm, group); + if (group.pair && (!liveRoles || liveRoles.driver !== caller || liveRoles.workers.indexOf(partner) === -1)) { finishDelegation(group, caller, partner, token); return false; } @@ -140,7 +135,8 @@ function attachSessionPair(ctx) { var deadline = Date.now() + timeoutSeconds * 1000; var timer = setInterval(function () { var currentGroup = store.groupForMember(caller.localId); - if (!currentGroup || currentGroup.id !== group.id || currentGroup.members.indexOf(partner.localId) === -1) { + var exactTarget = splitRoles.matchesLiveTarget(sm, currentGroup, caller, partner); + if (!currentGroup || currentGroup.id !== group.id || !exactTarget) { clearInterval(timer); finishDelegation(group, caller, partner, token); reject(new Error("the split group was dissolved while waiting for the partner")); @@ -148,6 +144,7 @@ function attachSessionPair(ctx) { } if (!partner.isProcessing && !partner._queryStarting) { clearInterval(timer); + if (!taskControl.matchesGeneration(partner, token)) return reject(new Error("the delegated task no longer matches this Worker generation")); var failure = errorSince(partner.history || [], token.startIndex); var interrupted = !failure && !!partner._lastTurnInterrupted; var response = responseText(partner.history || [], token.startIndex); @@ -182,18 +179,20 @@ function attachSessionPair(ctx) { function monitorPartner(group, caller, partner, token) { var timer = setInterval(function () { var currentGroup = store.groupForMember(caller.localId); - if (!currentGroup || currentGroup.id !== group.id || (!partner.isProcessing && !partner._queryStarting)) { + var exactTarget = splitRoles.matchesLiveTarget(sm, currentGroup, caller, partner); + if (!currentGroup || currentGroup.id !== group.id || !exactTarget || (!partner.isProcessing && !partner._queryStarting)) { clearInterval(timer); - if (currentGroup && currentGroup.id === group.id) handleTurnDone(partner); + if (currentGroup && currentGroup.id === group.id && exactTarget) handleTurnDone(partner); else finishDelegation(group, caller, partner, token); } }, 500); + if (timer && typeof timer.unref === "function") timer.unref(); } async function sendToPartner(args, caller) { try { if (caller && caller._delegatedBy) throw new Error("delegated turns cannot delegate to another session"); var resolved = groupAndPartner(caller, - "send_to_partner requires an exact existing Driver/Split Worker pair; call propose_worker while unpaired"); + "send_to_partner requires an exact existing Driver/Split Worker pair; call propose_worker while unpaired", args, { mutation: true }); var message = typeof args.message === "string" ? args.message.trim() : ""; if (!message) throw new Error("message is required"); turnControl.assertWorkerAction(caller); @@ -248,6 +247,7 @@ function attachSessionPair(ctx) { return toolResult({ status: "running", partnerId: partner.localId, taskId: token.taskId, generation: token.generation, hint: "The completed result will be pushed back automatically. Use read_partner only for a requested status, concrete failure or stall diagnosis, or capacity/queue decision; do not poll routinely." }); } if (!partner.isProcessing && !partner._queryStarting) { + if (!taskControl.matchesGeneration(partner, token)) return toolError(new Error("the delegated task no longer matches this Worker generation")); var immediateFailure = errorSince(partner.history || [], token.startIndex); var immediateResponse = responseText(partner.history || [], token.startIndex); var immediateOutcome = taskControl.complete(partner, caller, token, immediateFailure ? "failed" : "completed", immediateResponse, immediateFailure); @@ -268,8 +268,9 @@ function attachSessionPair(ctx) { if (caller) { var group = store.groupForMember(caller.localId); if (group) { - var partnerId = group.members[0] === caller.localId ? group.members[1] : group.members[0]; - var partner = sm.sessions.get(partnerId); + var failedRoles = splitRoles.resolveLiveRoles(sm, group); + var failedTarget = failedRoles && splitRoles.resolveWorkerTarget(failedRoles.role, args && args.workerId); + var partner = failedTarget && failedTarget.ok ? sm.sessions.get(failedTarget.workerId) : null; if (partner && partner._pairDelegation && partner._pairDelegation.from === caller.localId && !partner.isProcessing && !partner._pairDelegation._retainOnError) { finishDelegation(group, caller, partner, partner._pairDelegation); } @@ -280,7 +281,7 @@ function attachSessionPair(ctx) { } function readPartner(args, caller) { try { - var resolved = groupAndPartner(caller); + var resolved = groupAndPartner(caller, null, args, { readOnly: true }); var count = Number.isFinite(args.lastTurns) ? Math.floor(args.lastTurns) : 1; count = Math.max(0, Math.min(5, count)); var payload = { @@ -289,7 +290,7 @@ function attachSessionPair(ctx) { title: resolved.partner.title || "New Session", turns: count > 0 ? recentTurns(resolved.partner, count) : [], }; - payload.capacity = lifecycle.optionalStatus(caller); + payload.capacity = lifecycle.optionalStatus(caller, args); return toolResult(payload); } catch (e) { return toolError(e); @@ -297,7 +298,7 @@ function attachSessionPair(ctx) { } function interruptPartner(args, caller) { try { - var resolved = groupAndPartner(caller); + var resolved = groupAndPartner(caller, null, args, { mutation: true }); var partner = resolved.partner; if (!partner.isProcessing && !partner._queryStarting) { return toolResult({ status: "idle", partnerId: partner.localId, title: partner.title || "New Session" }); @@ -310,43 +311,26 @@ function attachSessionPair(ctx) { return toolError(e); } } + function closeWorker(args, caller) { + return closeTransaction.close(args, caller); + } function closePartner(args, caller) { try { - var resolved = groupAndPartner(caller); - var partner = resolved.partner; - var interrupted = !!(partner.isProcessing || partner._queryStarting); - if (interrupted) { - taskControl.markInterruption(partner, "driver", "The Driver closed the Split Worker pair."); - partner.taskStopRequested = true; - if (partner.abortController) partner.abortController.abort(); - } - if (partner._pairDelegation) { - completeInterruptedTask(partner, caller, partner._pairDelegation); - finishDelegation(resolved.group, caller, partner, partner._pairDelegation); - } - workerPermission.cancelForSession(partner, "The Driver closed the Split Worker pair."); - var ws = { _clayUser: caller.ownerId ? { id: caller.ownerId } : null }; - var result = store.dissolve(ws, { id: resolved.group.id }); - if (!result.ok) throw new Error(result.error || "could not close the Worker pair"); - return toolResult({ status: "closed", partnerId: partner.localId, interrupted: interrupted, historyPreserved: true }); + return toolResult(closeWorker(args, caller)); } catch (e) { return toolError(e); } } function handleHumanStop(session) { var roles = turnControl.markHumanStop(session); - if (!roles) return false; - if (roles.worker._pairDelegation && roles.worker._pairDelegation.outboxKey && ctx.resultOutbox) ctx.resultOutbox.markBlocked(roles.worker._pairDelegation.outboxKey, "The human stopped this Split Worker turn.", "human_stop"); - taskControl.markInterruption(roles.worker, "user", "The human stopped this Split Worker turn."); - workerPermission.cancelForSession(roles.worker, "The human stopped this Split Worker turn."); - return true; + return stopAllWorkers(session, roles, { resultOutbox: ctx.resultOutbox, taskControl: taskControl, workerPermission: workerPermission }); } function stopOwnedPair(session, reason) { return stopOwned(session, rolesFor(session), turnControl, taskControl, workerPermission, reason); } function stopAutonomousWork(session) { return stopAutonomous(session, rolesFor(session), turnControl, taskControl, workerPermission); } function beginHumanTurn(session) { var resumed = turnControl.beginHumanTurn(session); var roles = resumed && rolesFor(session); - if (roles) taskControl.drain(roles.driver, roles.worker); + if (roles) for (var i = 0; i < roles.workers.length; i++) taskControl.drain(roles.driver, roles.workers[i]); return resumed; } function stopBarrier(session) { return turnControl.blockedReason(session); } @@ -362,7 +346,8 @@ function attachSessionPair(ctx) { evaluate: function () { return toolError(new Error("record_partner_evaluation requires a session-bound tool server")); }, }); var group = store.groupForMember(boundSession.localId); - if (group && group.pair && group.pair.driverId !== boundSession.localId) return taskControl.workerToolDefs(boundSession); + var boundRoles = group && group.pair ? splitRoles.resolveLiveRoles(sm, group) : null; + if (boundRoles && boundRoles.workers.indexOf(boundSession) !== -1) return taskControl.workerToolDefs(boundSession); var adHocSplit = !!(group && !group.pair); if (!driverEligibility.isEligibleDriverSession(boundSession, sm)) return []; var persistentCatalog = boundSession.vendor === "codex" && (!group || !!group.pair); @@ -375,7 +360,7 @@ function attachSessionPair(ctx) { send: function (args) { return turnControl.runOperation(boundSession, "send", args && args.operationId, function () { return sendToPartner(args || {}, boundSession); - }, operationFingerprint(args, ["message", "wait", "timeoutSeconds", "taskId"])).catch(toolError); + }, operationFingerprint(args, ["message", "wait", "timeoutSeconds", "taskId", "workerId"])).catch(toolError); }, read: function (args) { return readPartner(args, boundSession); }, message: function (args) { @@ -391,7 +376,7 @@ function attachSessionPair(ctx) { replace: function (args) { return turnControl.runOperation(boundSession, "replace", args && args.operationId, function () { return workerProposal.proposeReplacement(args || {}, boundSession); - }, operationFingerprint(args, ["interrupt", "message", "workerVendor", "workerModel", "workerEffort", "recommendationRationale", "evaluation", "supersedeProposalId"])).catch(toolError); + }, operationFingerprint(args, ["interrupt", "message", "workerVendor", "workerModel", "workerEffort", "recommendationRationale", "evaluation", "supersedeProposalId", "workerId"])).catch(toolError); }, evaluate: lifecycleHandlers.evaluate, }, { lifecycle: persistentCatalog || !adHocSplit }); @@ -412,12 +397,14 @@ function attachSessionPair(ctx) { } var factory = attachPairFactory(ctx), createPairRecord = factory.createPairRecord, createWorkerForDriver = factory.createWorkerForDriver, createPair = factory.createPair, + addWorkerForDriver = factory.addWorkerForDriver, replaceWorkerForDriver = factory.replaceWorkerForDriver, preflightWorkerForDriver = factory.preflightWorkerForDriver, createOwnedPair = factory.createOwnedPair, removeOwnedPair = factory.removeOwnedPair, preflightRuntime = factory.preflightRuntime; lifecycle = attachPairLifecycle({ sm: sm, splitStore: store, createWorkerForDriver: function (driver, args) { return createWorkerForDriver(driver, args); }, + replaceWorkerForDriver: function (driver, worker, args) { return replaceWorkerForDriver(driver, worker, args); }, preflightWorkerForDriver: function (driver, args) { return preflightWorkerForDriver(driver, args); }, sendToPartner: function (args, caller) { return sendToPartner(args, caller); }, finishDelegation: finishDelegation, @@ -427,6 +414,7 @@ function attachSessionPair(ctx) { taskStatus: function (worker) { return taskControl.status(worker); }, proposalStatus: function (driver) { return workerProposal ? workerProposal.proposalStatus(driver) : null; }, turnControl: turnControl, + multiWorkerFeature: ctx.multiWorkerFeature, }); workerPermission = attachWorkerPermission({ sm: sm, @@ -450,7 +438,9 @@ function attachSessionPair(ctx) { recordGenerationStart: function (driver, worker) { return lifecycle.recordGenerationStart(driver, worker); }, sendToPartner: sendToPartner, replacePartner: function (args, session) { return lifecycle.replacePartner(args, session); }, + addWorkerForDriver: function (driver, args) { return addWorkerForDriver(driver, args); }, preflightWorkerForDriver: preflightWorkerForDriver, + multiWorkerFeature: ctx.multiWorkerFeature, }); function handleMessage(ws, msg) { if (workerProposal.handleMessage(ws, msg)) return true; @@ -471,28 +461,15 @@ function attachSessionPair(ctx) { } return false; } - function getSystemPrompt(session) { - var roles = rolesFor(session); - if (roles && roles.worker === session) { - var taskId = session._pairDelegation && session._pairDelegation.taskId || "unknown"; - return pairPrompts.worker(taskId); - } - var group = store.groupForMember(session.localId); - var pairPrompt = ""; - if (group && group.pair && group.pair.driverId === session.localId) { - pairPrompt = pairPrompts.DRIVER_CORE; - if (driverOrchestration.isHighTierDriverSession(session, sm)) { - pairPrompt += " " + pairPrompts.DRIVER_DELEGATION; - } - } else if (!group && !ctx.isMate && driverEligibility.isEligibleDriverSession(session, sm) && - driverOrchestration.isHighTierDriverSession(session, sm)) { - pairPrompt = pairPrompts.UNPAIRED + " " + workerProposal.getSystemPrompt(session); - } - return pairPrompt; - } + var pairPrompt = attachSessionPairPrompt({ sm: sm, store: store, isMate: ctx.isMate, rolesFor: rolesFor, + workerProposal: function () { return workerProposal; } }); var resultPipeline = attachPairResultPipeline({ sm: sm, store: store, resultOutbox: ctx.resultOutbox, getSdk: ctx.getSdk, getLinuxUserForSession: ctx.getLinuxUserForSession, acceptanceTimeoutMs: ctx.acceptanceTimeoutMs, finishDelegation: finishDelegation, resumeDriverWithResult: resumeDriverWithResult, onPartnerResult: ctx.onPartnerResult, onStateChange: ctx.onResultStateChange, blockedReason: turnControl.blockedReason, drain: function (caller, partner) { return taskControl.drain(caller, partner); } }); resultCapture = resultPipeline.capture; var resultDelivery = resultPipeline.delivery; - return { beginHumanTurn: beginHumanTurn, getToolDefs: getToolDefs, handleHumanStop: handleHumanStop, stopOwnedPair: stopOwnedPair, stopAutonomousWork: stopAutonomousWork, stopBarrier: stopBarrier, handleMessage: handleMessage, handleTurnDone: handleTurnDone, getSystemPrompt: getSystemPrompt, - respondToWorkerProposal: workerProposal.respondToProposal, workerPermission: workerPermission, createOwnedPair: createOwnedPair, removeOwnedPair: removeOwnedPair, preflightRuntime: preflightRuntime, - sendToPartner: sendToPartner, rolesFor: rolesFor, resultCapture: resultCapture, resultOutbox: ctx.resultOutbox, wakePartnerResult: function (caller, partner, token) { return resultDelivery.wake(caller, partner, token); }, getResultOutboxStatus: function () { return ctx.resultOutbox ? ctx.resultOutbox.status() : { records: [], persistenceError: null }; } }; + closeTransaction = attachPairCloseTransaction({ store: store, multiWorkerFeature: ctx.multiWorkerFeature, + resolvePair: function (caller, args) { return groupAndPartner(caller, null, args, { mutation: true, structural: true }); }, + taskControl: taskControl, resultCapture: resultCapture, workerPermission: workerPermission, + responseText: responseText, errorSince: errorSince }); + return { beginHumanTurn: beginHumanTurn, getToolDefs: getToolDefs, handleHumanStop: handleHumanStop, stopOwnedPair: stopOwnedPair, stopAutonomousWork: stopAutonomousWork, stopBarrier: stopBarrier, handleMessage: handleMessage, handleTurnDone: handleTurnDone, getSystemPrompt: pairPrompt.getSystemPrompt, + respondToWorkerProposal: workerProposal.respondToProposal, workerPermission: workerPermission, addWorkerForDriver: addWorkerForDriver, createOwnedPair: createOwnedPair, removeOwnedPair: removeOwnedPair, preflightRuntime: preflightRuntime, + sendToPartner: sendToPartner, closeWorker: closeWorker, rolesFor: rolesFor, resultCapture: resultCapture, resultOutbox: ctx.resultOutbox, wakePartnerResult: function (caller, partner, token) { return resultDelivery.wake(caller, partner, token); }, getResultOutboxStatus: function () { return ctx.resultOutbox ? ctx.resultOutbox.status() : { records: [], persistenceError: null }; } }; } module.exports = { attachSessionPair: attachSessionPair, responseText: responseText, recentTurns: recentTurns }; diff --git a/lib/project-worker-creation-proposal.js b/lib/project-worker-creation-proposal.js new file mode 100644 index 00000000..2e642aea --- /dev/null +++ b/lib/project-worker-creation-proposal.js @@ -0,0 +1,176 @@ +var crypto = require("crypto"); +var driverEligibility = require("./session-driver-eligibility"); +var multiWorkerFeature = require("./multi-worker-feature"); +var proposalControl = require("./worker-proposal-control"); +var splitRoles = require("./session-split-group-roles"); + +function generationOf(session) { + return session && (session._pairGeneration || + session.sessionProvenance && session.sessionProvenance.generation) || null; +} + +function captureMembership(sm, driver, group) { + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles || roles.driver !== driver || roles.role.kind === "adhoc" || !driver.sessionOriginId) return null; + var workers = []; + for (var i = 0; i < roles.workers.length; i++) { + if (!roles.workers[i].sessionOriginId || !Number.isInteger(generationOf(roles.workers[i]))) return null; + workers.push({ id: roles.workers[i].localId, originId: roles.workers[i].sessionOriginId, + generation: generationOf(roles.workers[i]) }); + } + return { groupId: group.id, ownerId: driver.ownerId || null, driverId: driver.localId, + driverOriginId: driver.sessionOriginId, workerIds: roles.role.workerIds.slice(), workers: workers }; +} + +function matchesMembership(sm, driver, group, snapshot) { + if (!snapshot || !driver || !group || group.id !== snapshot.groupId || driver.localId !== snapshot.driverId || + (driver.ownerId || null) !== snapshot.ownerId || driver.sessionOriginId !== snapshot.driverOriginId) return false; + var current = captureMembership(sm, driver, group); + if (!current || current.workerIds.join(",") !== snapshot.workerIds.join(",")) return false; + for (var i = 0; i < current.workers.length; i++) { + var expected = snapshot.workers[i]; + if (!expected || current.workers[i].id !== expected.id || current.workers[i].originId !== expected.originId || + current.workers[i].generation !== expected.generation) return false; + } + return true; +} + +function attachWorkerCreationProposal(ctx) { + var sm = ctx.sm; + var store = ctx.splitStore; + + function initialSnapshot(session) { + if (!ctx.canOffer(session)) return { error: "Split Worker proposals require an eligible Driver session." }; + var group = store.groupForMember(session.localId); + if (!group) return { kind: "initial", driver: session }; + if (!multiWorkerFeature.isEnabled(ctx.multiWorkerFeature)) { + return { error: "This Driver already has a Split Worker." }; + } + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles || roles.driver !== session || roles.role.kind === "adhoc") { + return { error: "The exact Driver/Split Worker membership is not eligible for another Worker." }; + } + if (roles.workers.length >= 2) return { error: "A Driver can have at most two Split Workers." }; + if (roles.workers.length !== 1) return { error: "The existing Split Worker membership is invalid." }; + if (!session.sessionOriginId || !roles.workers[0].sessionOriginId || !Number.isInteger(generationOf(roles.workers[0]))) { + return { error: "The exact Driver and Split Worker identities are not durable enough to add another Worker." }; + } + return { + kind: "add", + group: group, + groupId: group.id, + ownerId: session.ownerId || null, + driverId: session.localId, + driverOriginId: session.sessionOriginId || null, + workerIds: roles.role.workerIds.slice(), + workers: roles.workers.map(function (worker) { + return { id: worker.localId, originId: worker.sessionOriginId || null, generation: generationOf(worker) }; + }), + }; + } + + function validateSnapshot(session, snapshot, requireSameObject) { + if (!snapshot || snapshot.kind === "initial") { + return ctx.isEligible(session) ? { ok: true, group: null } : { ok: false }; + } + if (!multiWorkerFeature.isEnabled(ctx.multiWorkerFeature) || !ctx.canOffer(session) || + (session.ownerId || null) !== snapshot.ownerId || session.localId !== snapshot.driverId || + (session.sessionOriginId || null) !== snapshot.driverOriginId) return { ok: false }; + var group = store.groupForMember(session.localId); + if (!group || group.id !== snapshot.groupId || requireSameObject && group !== snapshot.group) return { ok: false }; + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles || roles.role.kind === "adhoc" || roles.driver !== session || roles.role.workerIds.length !== 1 || + roles.role.workerIds.join(",") !== snapshot.workerIds.join(",")) return { ok: false }; + for (var i = 0; i < snapshot.workers.length; i++) { + var expected = snapshot.workers[i]; + var worker = sm.sessions.get(expected.id); + if (!worker || roles.workers.indexOf(worker) === -1 || + (worker.ownerId || null) !== snapshot.ownerId || + (worker.sessionOriginId || null) !== expected.originId || generationOf(worker) !== expected.generation) { + return { ok: false }; + } + } + return { ok: true, group: group, roles: roles }; + } + + function snapshotFromProposal(proposal) { + if (!proposal || proposal.action !== "add") return { kind: "initial" }; + return { + kind: "add", + groupId: proposal.sourceGroupId, + ownerId: proposal.sourceOwnerId || null, + driverId: proposal.sourceDriverId, + driverOriginId: proposal.sourceDriverOriginId || null, + workerIds: Array.isArray(proposal.sourceWorkerIds) ? proposal.sourceWorkerIds.slice() : [], + workers: Array.isArray(proposal.sourceWorkers) ? proposal.sourceWorkers.slice() : [], + }; + } + + async function propose(args, session) { + var snapshot = initialSnapshot(session); + if (snapshot.error) return ctx.toolResult({ error: snapshot.error }); + var superseded = proposalControl.pendingProposal(session); + var supersedeId = typeof args.supersedeProposalId === "string" ? args.supersedeProposalId.trim() : ""; + if (superseded && superseded.proposalId !== supersedeId) return ctx.toolResult({ error: "A Split Worker suggestion is already awaiting a decision. Inspect it or supply its exact id to supersede it." }); + if (!superseded && supersedeId) return ctx.toolResult({ error: "The Split Worker suggestion to supersede is no longer pending." }); + if (superseded && superseded.status !== "pending") return ctx.toolResult({ error: "The Split Worker suggestion is already starting and cannot be superseded." }); + var summary = typeof args.summary === "string" ? args.summary.trim() : ""; + var plan = typeof args.plan === "string" ? args.plan.trim() : ""; + var task = typeof args.message === "string" ? args.message.trim() : ""; + var rationale = typeof args.recommendationRationale === "string" ? args.recommendationRationale.trim() : ""; + if (!summary || !plan || !task || !rationale) return ctx.toolResult({ error: "summary, plan, message, and recommendationRationale are required." }); + if (task.length > ctx.maxTaskChars) return ctx.toolResult({ error: "The Split Worker task is too long." }); + await ctx.ensureModelCatalogs(); + if (!validateSnapshot(session, snapshot, true).ok) { + return ctx.toolResult({ error: "The Driver/Split Worker membership changed while runtimes were loading." }); + } + if (proposalControl.pendingProposal(session) !== superseded) return ctx.toolResult({ error: "The pending Split Worker suggestion changed while runtimes were loading." }); + var options = ctx.proposalOptions(); + if (options.installedVendors.length === 0) return ctx.toolResult({ error: "No coding agent is installed for a Split Worker session." }); + var recommendation = ctx.chooseRecommendation(args, session, options); + var proposal = { + type: "worker_proposal", proposalId: "worker_" + crypto.randomUUID(), + summary: summary.slice(0, ctx.maxSummaryChars), plan: plan.slice(0, ctx.maxPlanChars), message: task, + status: "pending", createdAt: Date.now(), recommendedVendor: recommendation.vendor, + recommendedModel: recommendation.model, recommendedEffort: recommendation.effort, + recommendationRationale: rationale.slice(0, ctx.maxRationaleChars), options: options, + }; + if (snapshot.kind === "add") Object.assign(proposal, { + action: "add", sourceGroupId: snapshot.groupId, sourceOwnerId: snapshot.ownerId, + sourceDriverId: snapshot.driverId, sourceDriverOriginId: snapshot.driverOriginId, + sourceWorkerIds: snapshot.workerIds.slice(), sourceWorkers: snapshot.workers.slice(), + }); + if (superseded) ctx.updateProposal(session, superseded, { status: "superseded", supersededBy: proposal.proposalId }); + sm.sendAndRecord(session, proposal); + if (ctx.skipPermissionsEnabled(session) && ctx.recommendationCanAutoAccept(args, recommendation, options)) { + var accepted = await ctx.acceptProposal(session, proposal, { + vendor: recommendation.vendor, model: recommendation.model, effort: recommendation.effort, + }, ctx.autoAcceptanceWs(session), true); + return ctx.toolResult({ status: accepted.ok ? "auto_accepted" : "posted", proposalId: proposal.proposalId, + instruction: accepted.ok ? "The recorded Split Worker configuration was auto-accepted. End this turn while the exact Worker runs." : + "The automatic decision failed closed. The configuration card remains pending for the user." }); + } + return ctx.toolResult({ status: "posted", proposalId: proposal.proposalId, + instruction: "The Split Worker suggestion is visible in the chat. End this turn now and wait for the user's decision." }); + } + + function canMountProposal(session) { + if (!ctx.canOffer(session)) return false; + var group = store.groupForMember(session.localId); + if (!group) return true; + if (!multiWorkerFeature.isEnabled(ctx.multiWorkerFeature)) return false; + var roles = splitRoles.resolveLiveRoles(sm, group); + return !!roles && roles.role.kind !== "adhoc" && roles.driver === session; + } + + return { + propose: propose, + canMountProposal: canMountProposal, + validateAcceptance: function (session, proposal) { + return validateSnapshot(session, snapshotFromProposal(proposal), false); + }, + }; +} + +module.exports = { attachWorkerCreationProposal: attachWorkerCreationProposal, + captureMembership: captureMembership, matchesMembership: matchesMembership }; diff --git a/lib/project-worker-permission.js b/lib/project-worker-permission.js index a6df4632..cd964827 100644 --- a/lib/project-worker-permission.js +++ b/lib/project-worker-permission.js @@ -43,6 +43,7 @@ // same user. Anything else is refused and the request stays pending. var crypto = require("crypto"); +var splitRoles = require("./session-split-group-roles"); // Tools that collect input from the person rather than authorize an action. // These are not permission approvals, so they are never routed to a Driver; @@ -104,20 +105,18 @@ function attachWorkerPermission(ctx) { if (sm.sessions.get(session.localId) !== session) return null; var group = store.groupForMember(session.localId); if (!group || !group.pair) return null; - if (group.pair.workerId !== session.localId) return null; - var driver = sm.sessions.get(group.pair.driverId); - if (!driver) return null; - if ((driver.ownerId || null) !== (session.ownerId || null)) return null; - if (driver.destroying) return null; - return { group: group, driver: driver }; + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles || roles.workers.indexOf(session) === -1) return null; + return { group: group, driver: roles.driver }; } // Is the record's pair still exactly the pair it was created for? function pairStillExact(record) { var group = store ? store.groupForMember(record.workerId) : null; if (!group || group.id !== record.groupId || !group.pair) return false; - if (group.pair.driverId !== record.driverId) return false; - if (group.pair.workerId !== record.workerId) return false; + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles || roles.driver.localId !== record.driverId) return false; + if (roles.workers.indexOf(record.workerRef) === -1 || record.workerRef.localId !== record.workerId) return false; var driver = sm.sessions.get(record.driverId); var worker = sm.sessions.get(record.workerId); if (!driver || !worker || driver.destroying || worker.destroying) return false; diff --git a/lib/project-worker-proposal-runner.js b/lib/project-worker-proposal-runner.js new file mode 100644 index 00000000..ee321db7 --- /dev/null +++ b/lib/project-worker-proposal-runner.js @@ -0,0 +1,45 @@ +function attachProposalRunner(ctx) { + function resumeDriver(session, text) { + var sdk = ctx.getSdk(); + if (!sdk) return Promise.reject(new Error("SDK bridge is not ready")); + ctx.sm.sendAndRecord(session, { type: "user_message", text: text, _internal: true }); + session.sentToolResults = {}; + if (!session.isProcessing) { + session.isProcessing = true; + ctx.onProcessingChanged(); + ctx.sm.sendToSession(session, { type: "status", status: "processing" }); + } + if (sdk.pushMessage(session, text)) return Promise.resolve(); + return Promise.resolve(sdk.startQuery(session, text, undefined, ctx.getLinuxUserForSession(session))); + } + + function parseResult(result) { + if (!result || result.isError || !result.content || !result.content[0]) { + return { status: "error", error: "Split Worker execution failed." }; + } + try { return JSON.parse(result.content[0].text); } + catch (error) { return { status: "error", error: result.content[0].text || "Split Worker execution failed." }; } + } + + async function runWorker(session, proposal) { + var result = parseResult(await ctx.sendToPartner({ message: proposal.message, wait: true, + timeoutSeconds: 900, workerId: proposal.workerId }, session)); + ctx.updateProposal(session, proposal, { status: result.status === "complete" ? "completed" : result.status || "error", + resultPreview: result.response ? result.response.slice(0, 1200) : "", error: result.error || null }); + var followup; + if (result.status === "complete") { + followup = "[Split Worker execution completed]\nReview and verify the Split Worker's result. The Split Worker session remains available: if the implementation needs corrections or additional edits, send a follow-up with send_to_partner instead of taking over the Split Worker-owned files yourself. If that Split Worker is no longer available, keep the work in the visible Split Worker flow and use send_to_partner again after the stale pair is removed; never substitute a background Sub-agent.\n\n" + (result.response || "The Split Worker completed without a text summary."); + } else if (result.status === "interrupted") { + followup = "[Split Worker execution interrupted]\nThe Split Worker stopped mid-turn. Its work is PARTIAL and unverified — do not treat it as finished. Check partner_status before deciding next steps; this result alone does not identify who interrupted it. If the human stopped it, do not retry until a new human Driver message. Otherwise continue within the user's authorized task."; + } else if (result.status === "running") { + followup = "[Split Worker execution is still running]\nUse read_partner to inspect progress before completing the task."; + } else { + followup = "[Split Worker execution failed]\nThe existing Split Worker may still be reusable. Inspect partner_status and send a narrower follow-up with send_to_partner when the Worker and pair still exist; do not replace it merely because its task or transport failed. Retry replacement only when the replacement transaction reports that no new Worker was created and the failure is explicitly safe to retry.\n\n" + (result.error || result.response || "Unknown Split Worker error."); + } + await resumeDriver(session, followup); + } + + return { resumeDriver: resumeDriver, runWorker: runWorker }; +} + +module.exports = { attachProposalRunner: attachProposalRunner }; diff --git a/lib/project-worker-proposal.js b/lib/project-worker-proposal.js index fc0d92a3..a8e44351 100644 --- a/lib/project-worker-proposal.js +++ b/lib/project-worker-proposal.js @@ -3,8 +3,12 @@ var yoke = require("./yoke"); var buildShape = require("./session-spawn-mcp-server").buildShape; var driverEligibility = require("./session-driver-eligibility"); var driverOrchestration = require("./session-driver-orchestration"); +var pairTarget = require("./session-pair-target"); +var splitRoles = require("./session-split-group-roles"); var proposalControl = require("./worker-proposal-control"); var runtimeCatalog = require("./worker-runtime-catalog"); +var proposalRunner = require("./project-worker-proposal-runner"); +var creationProposalModule = require("./project-worker-creation-proposal"); var MAX_SUMMARY_CHARS = 600, MAX_PLAN_CHARS = 6000, MAX_TASK_CHARS = 30000, MAX_RATIONALE_CHARS = 1000; var EVALUATION_OUTCOMES = ["succeeded", "partial", "failed", "abandoned"]; function normalizeEvaluation(raw) { @@ -161,66 +165,18 @@ function attachWorkerProposal(ctx) { proposalId: proposal.proposalId, }, patch)); } - async function propose(args, session) { - if (!isEligible(session)) return toolResult({ error: "Split Worker proposals are only available in an eligible unpaired Driver session." }); - var superseded = proposalControl.pendingProposal(session); - var supersedeId = typeof args.supersedeProposalId === "string" ? args.supersedeProposalId.trim() : ""; - if (superseded && superseded.proposalId !== supersedeId) return toolResult({ error: "A Split Worker suggestion is already awaiting a decision. Inspect it or supply its exact id to supersede it." }); - if (!superseded && supersedeId) return toolResult({ error: "The Split Worker suggestion to supersede is no longer pending." }); - if (superseded && superseded.status !== "pending") return toolResult({ error: "The Split Worker suggestion is already starting and cannot be superseded." }); - var summary = typeof args.summary === "string" ? args.summary.trim() : ""; - var plan = typeof args.plan === "string" ? args.plan.trim() : ""; - var task = typeof args.message === "string" ? args.message.trim() : ""; - var rationale = typeof args.recommendationRationale === "string" ? args.recommendationRationale.trim() : ""; - if (!summary || !plan || !task || !rationale) { - return toolResult({ error: "summary, plan, message, and recommendationRationale are required." }); - } - if (task.length > MAX_TASK_CHARS) return toolResult({ error: "The Split Worker task is too long." }); - await ensureModelCatalogs(); - if (!isEligible(session)) { - return toolResult({ error: "The Driver session changed while Split Worker runtimes were loading." }); - } - var currentPending = proposalControl.pendingProposal(session); - if (currentPending !== superseded) return toolResult({ error: "The pending Split Worker suggestion changed while runtimes were loading." }); - var options = proposalOptions(); - if (options.installedVendors.length === 0) return toolResult({ error: "No coding agent is installed for a Split Worker session." }); - var recommendation = chooseRecommendation(args, session, options); - var proposal = { - type: "worker_proposal", - proposalId: "worker_" + crypto.randomUUID(), - summary: summary.slice(0, MAX_SUMMARY_CHARS), - plan: plan.slice(0, MAX_PLAN_CHARS), - message: task, - status: "pending", - createdAt: Date.now(), - recommendedVendor: recommendation.vendor, - recommendedModel: recommendation.model, - recommendedEffort: recommendation.effort, - recommendationRationale: rationale.slice(0, MAX_RATIONALE_CHARS), - options: options, - }; - if (superseded) updateProposal(session, superseded, { status: "superseded", supersededBy: proposal.proposalId }); - sm.sendAndRecord(session, proposal); - if (skipPermissionsEnabled(session) && recommendationCanAutoAccept(args, recommendation, options)) { - var accepted = await acceptProposal(session, proposal, { - vendor: recommendation.vendor, - model: recommendation.model, - effort: recommendation.effort, - }, autoAcceptanceWs(session), true); - return toolResult({ - status: accepted.ok ? "auto_accepted" : "posted", - proposalId: proposal.proposalId, - instruction: accepted.ok - ? "The recorded Split Worker configuration was auto-accepted under the Driver session's full-access mode. End this turn now while the exact paired Worker runs." - : "The automatic decision failed closed. The configuration card remains pending for the user; end this turn and wait for their decision.", - }); - } - return toolResult({ - status: "posted", - proposalId: proposal.proposalId, - instruction: "The Split Worker suggestion is visible in the chat. End this turn now and wait for the user's decision.", - }); - } + var runner = proposalRunner.attachProposalRunner(Object.assign({}, ctx, { updateProposal: updateProposal })); + var resumeDriver = runner.resumeDriver; + var runWorker = runner.runWorker; + var creationProposal = creationProposalModule.attachWorkerCreationProposal({ + sm: sm, splitStore: store, multiWorkerFeature: ctx.multiWorkerFeature, + canOffer: canOffer, isEligible: isEligible, ensureModelCatalogs: ensureModelCatalogs, + proposalOptions: proposalOptions, chooseRecommendation: chooseRecommendation, + skipPermissionsEnabled: skipPermissionsEnabled, recommendationCanAutoAccept: recommendationCanAutoAccept, + autoAcceptanceWs: autoAcceptanceWs, acceptProposal: acceptProposal, updateProposal: updateProposal, + toolResult: toolResult, maxSummaryChars: MAX_SUMMARY_CHARS, maxPlanChars: MAX_PLAN_CHARS, + maxTaskChars: MAX_TASK_CHARS, maxRationaleChars: MAX_RATIONALE_CHARS, + }); async function proposeReplacement(args, session) { var group = session && store.groupForMember(session.localId); if (ctx.isMate || !isLiveSession(session) || !driverEligibility.isEligibleDriverSession(session, sm) || !group || @@ -228,7 +184,17 @@ function attachWorkerProposal(ctx) { return toolResult({ error: "Split Worker replacement proposals require the exact paired Driver session." }); } var sourceGroupId = group.id; - var sourceWorkerId = group.pair.workerId; + var addressed; + try { addressed = pairTarget.resolveGroupTarget(sm, group, session, args, { + mutation: true, multiWorkerFeature: ctx.multiWorkerFeature, + }); } + catch (e) { return toolResult({ error: e.message || String(e) }); } + var sourceWorkerId = addressed.workerId; + var sourceWorkerOriginId = addressed.partner.sessionOriginId || null; + var sourceGeneration = addressed.partner._pairGeneration || + addressed.partner.sessionProvenance && addressed.partner.sessionProvenance.generation || null; + var sourceMembership = creationProposalModule.captureMembership(sm, session, group); + if (!sourceMembership) return toolResult({ error: "The exact Driver/Split Worker identities are not durable enough to replace a Worker." }); var superseded = proposalControl.pendingProposal(session); var supersedeId = typeof args.supersedeProposalId === "string" ? args.supersedeProposalId.trim() : ""; if (superseded && superseded.proposalId !== supersedeId) return toolResult({ error: "A Split Worker proposal is already awaiting a decision. Inspect it or supply its exact id to supersede it." }); @@ -245,9 +211,17 @@ function attachWorkerProposal(ctx) { var liveGroup = store.groupForMember(session.localId); if (!isLiveSession(session) || !driverEligibility.isEligibleDriverSession(session, sm) || !liveGroup || liveGroup.id !== sourceGroupId || !liveGroup.pair || - liveGroup.pair.driverId !== session.localId || liveGroup.pair.workerId !== sourceWorkerId) { + liveGroup.pair.driverId !== session.localId || liveGroup.members.indexOf(sourceWorkerId) === -1 || + sm.sessions.get(sourceWorkerId) !== addressed.partner || + (addressed.partner.sessionOriginId || null) !== sourceWorkerOriginId || + (addressed.partner._pairGeneration || addressed.partner.sessionProvenance && addressed.partner.sessionProvenance.generation || null) !== sourceGeneration || + !creationProposalModule.matchesMembership(sm, session, liveGroup, sourceMembership)) { return toolResult({ error: "The Driver/Split Worker pair changed while replacement runtimes were loading." }); } + try { pairTarget.resolveGroupTarget(sm, liveGroup, session, args, { + mutation: true, multiWorkerFeature: ctx.multiWorkerFeature, + }); } + catch (e) { return toolResult({ error: e.message || String(e) }); } var currentPending = proposalControl.pendingProposal(session); if (currentPending !== superseded) return toolResult({ error: "The pending Split Worker proposal changed while runtimes were loading." }); var options = proposalOptions(); @@ -274,6 +248,10 @@ function attachWorkerProposal(ctx) { options: options, sourceGroupId: sourceGroupId, sourceWorkerId: sourceWorkerId, + sourceWorkerOriginId: sourceWorkerOriginId, + sourceGeneration: sourceGeneration, + sourceMembership: sourceMembership, + workerId: sourceWorkerId, interrupt: args.interrupt === true, evaluation: normalizedEvaluation ? normalizedEvaluation.value : null, transactionId: "replace_" + crypto.randomUUID(), @@ -300,50 +278,6 @@ function attachWorkerProposal(ctx) { instruction: "The replacement configuration is visible in the chat. End this turn now and wait for the user's decision.", }); } - function resumeDriver(session, text) { - var sdk = ctx.getSdk(); - if (!sdk) return Promise.reject(new Error("SDK bridge is not ready")); - var message = { type: "user_message", text: text, _internal: true }; - sm.sendAndRecord(session, message); - session.sentToolResults = {}; - if (!session.isProcessing) { - session.isProcessing = true; - ctx.onProcessingChanged(); - sm.sendToSession(session, { type: "status", status: "processing" }); - } - if (sdk.pushMessage(session, text)) return Promise.resolve(); - return Promise.resolve(sdk.startQuery(session, text, undefined, ctx.getLinuxUserForSession(session))); - } - - function parsePartnerResult(result) { - if (!result || result.isError || !result.content || !result.content[0]) { - return { status: "error", error: "Split Worker execution failed." }; - } - try { return JSON.parse(result.content[0].text); } - catch (e) { return { status: "error", error: result.content[0].text || "Split Worker execution failed." }; } - } - - async function runWorker(session, proposal) { - var result = parsePartnerResult(await ctx.sendToPartner({ message: proposal.message, wait: true, timeoutSeconds: 900 }, session)); - var status = result.status === "complete" ? "completed" : result.status; - updateProposal(session, proposal, { - status: status || "error", - resultPreview: result.response ? result.response.slice(0, 1200) : "", - error: result.error || null, - }); - var followup; - if (result.status === "complete") { - followup = "[Split Worker execution completed]\nReview and verify the Split Worker's result. The Split Worker session remains available: if the implementation needs corrections or additional edits, send a follow-up with send_to_partner instead of taking over the Split Worker-owned files yourself. If that Split Worker is no longer available, keep the work in the visible Split Worker flow and use send_to_partner again after the stale pair is removed; never substitute a background Sub-agent.\n\n" + (result.response || "The Split Worker completed without a text summary."); - } else if (result.status === "interrupted") { - followup = "[Split Worker execution interrupted]\nThe Split Worker stopped mid-turn. Its work is PARTIAL and unverified — do not treat it as finished. Check partner_status before deciding next steps; this result alone does not identify who interrupted it. If the human stopped it, do not retry until a new human Driver message. Otherwise continue within the user's authorized task."; - } else if (result.status === "running") { - followup = "[Split Worker execution is still running]\nUse read_partner to inspect progress before completing the task."; - } else { - followup = "[Split Worker execution failed]\nThe existing Split Worker may still be reusable. Inspect partner_status and send a narrower follow-up with send_to_partner when the Worker and pair still exist; do not replace it merely because its task or transport failed. Retry replacement only when the replacement transaction reports that no new Worker was created and the failure is explicitly safe to retry.\n\n" + (result.error || result.response || "Unknown Split Worker error."); - } - await resumeDriver(session, followup); - } - function sessionForResponse(ws) { var session = sm.sessions.get(ws._clayActiveSession); if (!session) throw new Error("active Driver session not found"); @@ -356,19 +290,28 @@ function attachWorkerProposal(ctx) { async function acceptProposal(session, proposal, msg, ws, trustedAutoAcceptance) { if (!isLiveSession(session)) throw new Error("The Driver session is no longer live"); - if (proposal.action !== "replace" && !isEligible(session)) { + if (!proposal.action && !isEligible(session)) { throw new Error("The Driver is no longer eligible to create this Split Worker"); } var approvedGroup = null; - if (proposal.action === "replace") { + if (proposal.action === "add") { + var addition = creationProposal.validateAcceptance(session, proposal); + if (!addition.ok) throw new Error("The Driver/Split Worker membership changed before the addition was approved"); + approvedGroup = addition.group; + } else if (proposal.action === "replace") { approvedGroup = store.groupForMember(session.localId); + var approvedRoles = splitRoles.normalizePair(approvedGroup && approvedGroup.pair, approvedGroup && approvedGroup.members); + var approvedWorker = sm.sessions.get(proposal.sourceWorkerId); if (!driverEligibility.isEligibleDriverSession(session, sm) || !approvedGroup || - approvedGroup.id !== proposal.sourceGroupId || !approvedGroup.pair || - approvedGroup.pair.driverId !== session.localId || approvedGroup.pair.workerId !== proposal.sourceWorkerId) { + approvedGroup.id !== proposal.sourceGroupId || !approvedRoles.ok || approvedRoles.kind === "adhoc" || + approvedRoles.driverId !== session.localId || approvedRoles.workerIds.indexOf(proposal.sourceWorkerId) === -1 || + !approvedWorker || (approvedWorker.sessionOriginId || null) !== (proposal.sourceWorkerOriginId || null) || + (approvedWorker._pairGeneration || approvedWorker.sessionProvenance && approvedWorker.sessionProvenance.generation || null) !== proposal.sourceGeneration || + !creationProposalModule.matchesMembership(sm, session, approvedGroup, proposal.sourceMembership)) { throw new Error("The Driver/Split Worker pair changed before the replacement was approved"); } } - var options = proposalOptions(); + var options = proposal.options || proposalOptions(); var vendor = msg.vendor || proposal.recommendedVendor; var model = msg.model || ""; if (options.installedVendors.indexOf(vendor) === -1) throw new Error("Selected Split Worker vendor is not installed"); @@ -390,6 +333,19 @@ function attachWorkerProposal(ctx) { }; updateProposal(session, proposal, startingPatch); try { + if (proposal.action === "add") { + var added = ctx.addWorkerForDriver(session, { + workerVendor: vendor, workerModel: model, workerEffort: effort, + expectedGroup: approvedGroup, expectedWorkerIds: proposal.sourceWorkerIds, + }); + ctx.recordGenerationStart(session, added.worker); + updateProposal(session, proposal, { status: "running", groupId: added.group.id, workerId: added.worker.localId }); + runWorker(session, proposal).catch(function (err) { + updateProposal(session, proposal, { status: "error", error: err.message || String(err) }); + resumeDriver(session, "[Split Worker execution failed]\n" + (err.message || String(err))).catch(function () {}); + }); + return { ok: true, status: "running", group: added.group, addition: added }; + } if (proposal.action === "replace") { var liveGroup = approvedGroup; var replaced = await ctx.replacePartner({ @@ -399,6 +355,7 @@ function attachWorkerProposal(ctx) { workerEffort: effort, evaluation: proposal.evaluation || undefined, transactionId: proposal.transactionId, + workerId: proposal.workerId, }, session); liveGroup = store.groupForMember(session.localId); updateProposal(session, proposal, { status: "running", groupId: liveGroup.id, workerId: replaced.workerSessionId }); @@ -423,9 +380,11 @@ function attachWorkerProposal(ctx) { } catch (err) { if (proposal.action === "replace") { var restoredGroup = store.groupForMember(session.localId); - if (restoredGroup && restoredGroup.pair && restoredGroup.pair.driverId === session.localId && - restoredGroup.pair.workerId === proposal.sourceWorkerId) { + var restoredRoles = splitRoles.normalizePair(restoredGroup && restoredGroup.pair, restoredGroup && restoredGroup.members); + if (restoredGroup && restoredRoles.ok && restoredRoles.kind !== "adhoc" && + restoredRoles.driverId === session.localId && restoredRoles.workerIds.indexOf(proposal.sourceWorkerId) !== -1) { proposal.sourceGroupId = restoredGroup.id; + proposal.sourceMembership = creationProposalModule.captureMembership(sm, session, restoredGroup); } proposal.transactionId = "replace_" + crypto.randomUUID(); } @@ -449,6 +408,8 @@ function attachWorkerProposal(ctx) { updateProposal(session, proposal, { status: "declined" }); var declinedText = proposal.action === "replace" ? "[Split Worker replacement declined]\nContinue with the existing Split Worker and the current task." + : proposal.action === "add" + ? "[Additional Split Worker declined]\nContinue with the existing Split Worker and current plan." : "[Split Worker proposal declined]\nContinue this task in the current Driver session using the plan you already prepared."; await resumeDriver(session, declinedText); return { ok: true, status: "declined" }; @@ -468,7 +429,7 @@ function attachWorkerProposal(ctx) { var persistent = !!(options && options.persistent); var controlsOnly = !!(options && options.controlsOnly); if (!isEligible(session) && !(persistent && canOffer(session))) return []; - var proposalTools = controlsOnly ? [] : [{ + var proposalTools = controlsOnly && !creationProposal.canMountProposal(session) ? [] : [{ name: "propose_worker", description: "Propose a visible Split Worker for implementation-heavy execution. This non-mutating tool only shows the user a card where they choose vendor, model, and effort. It never creates a session or delegates work until the user accepts.", inputSchema: buildShape({ @@ -481,7 +442,7 @@ function attachWorkerProposal(ctx) { recommendationRationale: { type: "string", description: "Concise Driver-authored explanation of why the recommended vendor, model, and effort fit this exact task." }, supersedeProposalId: { type: "string", description: "Exact pending proposal id to resolve as superseded when posting this replacement suggestion." }, }, ["summary", "plan", "message", "recommendationRationale"]), - handler: function (args) { return propose(args || {}, session); }, + handler: function (args) { return creationProposal.propose(args || {}, session); }, }]; return proposalTools.concat(proposalControl.getToolDefs({ inspect: function () { return toolResult(isLiveSession(session) ? proposalControl.projection(proposalControl.pendingProposal(session)) : { status: "rejected", reason: "The Driver session is no longer live." }); }, @@ -501,6 +462,9 @@ function attachWorkerProposal(ctx) { } function getSystemPrompt(session) { + if (creationProposal.canMountProposal(session) && !isEligible(session)) { + return "This Driver may configure one additional Split Worker through propose_worker. With two Workers, pass workerId to every targeted partner, queue, interrupt, replacement, close, permission, and status operation. Give each Worker independent file ownership and bounded work. Do not assign simultaneous overlapping file edits unless the work is explicitly separated, such as by distinct paths or worktrees."; + } if (!isEligible(session) || !driverOrchestration.isHighTierDriverSession(session, sm)) return ""; return "For implementation-heavy work, use propose_worker before substantial execution. Work is implementation-heavy when it spans multiple modules, crosses client/server/data boundaries, requires a migration, or contains independent investigation and execution. Do not skip delegation merely because you can implement it yourself; preserve the Driver for decomposition, runtime selection, judgment, review, and integration. The user's explicit request to work directly takes precedence, and work that must remain sequential in this session stays here. Recommend an exact available vendor, model, and effort, and give a concise recommendationRationale explaining why all three fit the task. Clay always records and shows the runtime configuration card. In full-access mode Clay may auto-accept that exact validated recommendation; otherwise it waits for the user's explicit choice. After posting, end the turn. If accepted, Clay creates the exact Driver/Split Worker pair and delegates the proposed task once. If declined, continue in this Driver session. Do not call send_to_partner while unpaired."; } diff --git a/lib/project.js b/lib/project.js index 61566cb4..a3a24038 100644 --- a/lib/project.js +++ b/lib/project.js @@ -68,6 +68,7 @@ var { attachProjectLogs } = require("./project-logs"); var { attachProjectLogFeedbackDelivery } = require("./project-log-feedback-delivery"); var { attachProjectMateKnowledge } = require("./project-mate-knowledge"); var { attachSplitGroups } = require("./session-split-groups"); +var multiWorkerFeature = require("./multi-worker-feature"); var toolControlMcp = require("./tool-control-mcp-server"); var capsuleFloor = require("./capsule-display-floor"); var toolLlm = require("./tool-llm"); @@ -207,6 +208,7 @@ function safeAbsPath(requested) { * opts: { cwd, slug, title, pushModule, debug, dangerouslySkipPermissions, currentVersion } */ function createProjectContext(opts) { + var multiWorkerGate = multiWorkerFeature.fromServerConfig(opts); var cwd = opts.cwd; var slug = opts.slug; var project = path.basename(cwd); @@ -661,6 +663,7 @@ function createProjectContext(opts) { clients: clients, sendTo: sendTo, usersModule: usersModule, + multiWorkerFeature: multiWorkerGate, onPairChanged: function (group) { if (!sdk || !group || !Array.isArray(group.members)) return; for (var i = 0; i < group.members.length; i++) { @@ -828,6 +831,7 @@ function createProjectContext(opts) { sm: sm, isMate: isMate, splitStore: _splitGroups.store, + multiWorkerFeature: multiWorkerGate, getSdk: function () { return sdk; }, send: send, sendTo: sendTo, @@ -851,6 +855,16 @@ function createProjectContext(opts) { if (_autonomousRun) _autonomousRun.onPartnerResult(driver, worker, token); }, }); + var _pairWorkerClose = require("./project-pair-worker-close").attachPairWorkerClose({ + sm: sm, + splitStore: _splitGroups.store, + multiWorkerFeature: multiWorkerGate, + isMate: isMate, + projectSlug: slug, + hasSocket: function (ws) { return clients.has(ws); }, + sendTo: sendTo, + closeWorker: _sessionPair.closeWorker, + }); _pairResultRecovery = attachPairResultRecovery({ sm: sm, store: _splitGroups.store, outbox: _pairResultOutbox, resultCapture: _sessionPair.resultCapture, wake: _sessionPair.wakePartnerResult, projectSlug: slug, sendTo: sendTo, @@ -1757,6 +1771,7 @@ function createProjectContext(opts) { // --- Sessions, config, project mgmt (delegated to project-sessions.js) --- if (_pairResultRecovery.handleMessage(ws, msg)) return; + if (_pairWorkerClose.handleMessage(ws, msg)) return; if (_sessionPair.handleMessage(ws, msg)) return; if (_sessionHandoff.handleMessage(ws, msg)) return; if (_driverContinuation.handleMessage(ws, msg)) return; diff --git a/lib/public/css/pane.css b/lib/public/css/pane.css index ed570aa2..84f79035 100644 --- a/lib/public/css/pane.css +++ b/lib/public/css/pane.css @@ -73,6 +73,28 @@ body.pane-mode #input-area { display: flex; } +/* A configured two-Worker group is a deliberate workspace hierarchy: the + Driver gets the broad working canvas, while both Workers remain visible as + equal-height execution surfaces on the right. Legacy/ad-hoc two-pane + splits keep the ordinary equal columns below. */ +#split-host:has(.split-pane-role-worker:nth-of-type(3)) { + display: grid; + grid-template-columns: minmax(0, 3fr) minmax(260px, 2fr); + grid-template-rows: repeat(2, minmax(0, 1fr)); + gap: 1px; + background: var(--border-subtle); +} + +#split-host:has(.split-pane-role-worker:nth-of-type(3)) .split-pane-role-driver { + grid-row: 1 / span 2; + grid-column: 1; +} + +#split-host:has(.split-pane-role-worker:nth-of-type(3)) .split-pane-role-worker { + grid-column: 2; + grid-row: var(--split-worker-row, auto); +} + .split-pane { flex: 1; display: flex; @@ -441,6 +463,11 @@ body.pane-mode #input-area { color: var(--text); } +.split-pane-close:disabled { + opacity: 0.42; + cursor: not-allowed; +} + .split-pane-close svg { width: 12px; height: 12px; @@ -478,6 +505,25 @@ body.pane-mode #input-area { --split-worker-height: calc(30% - 4.8px); } + #split-host:has(.split-pane-role-worker:nth-of-type(3)) { + display: flex; + flex-direction: column; + gap: var(--split-pane-gap); + background: transparent; + } + + #split-host:has(.split-pane-role-worker:nth-of-type(3)) .split-pane-role-driver, + #split-host:has(.split-pane-role-worker:nth-of-type(3)) .split-pane-role-worker { + grid-row: auto; + grid-column: auto; + } + + #split-host:has(.split-pane-role-worker:nth-of-type(3)) .split-pane-role-worker { + flex: 0 1 24%; + min-height: 104px; + order: -1; + } + #split-host.visible { flex-direction: column; gap: var(--split-pane-gap); @@ -495,11 +541,11 @@ body.pane-mode #input-area { } .split-pane-role-worker { - order: -1; + order: var(--split-pane-order, -1); } .split-pane-role-driver { - order: 0; + order: var(--split-pane-order, 0); } /* A configured Worker is an execution summary; the Driver keeps the diff --git a/lib/public/modules/app-connection.js b/lib/public/modules/app-connection.js index 84ff7a63..4c38845a 100644 --- a/lib/public/modules/app-connection.js +++ b/lib/public/modules/app-connection.js @@ -23,6 +23,7 @@ import { clearProjectSplitState } from './split-session-boundary.js'; import { clearPermissionModePending } from './permission-control.js'; import { clearMcpPermissionModePending } from './mcp-ui.js'; import { createWebSocketLifecycle } from './websocket-lifecycle.js'; +import { createWebSocketWatchdog } from './websocket-watchdog.js'; var connectOverlay = null; var hasConnectedOnce = false; @@ -42,16 +43,9 @@ var overlayGraceTimer = null; // server-global-ws.js. Well under the common 60s proxy read timeout, and one // timer per live socket. var HEARTBEAT_MS = 25000; -var heartbeatTimer = null; -var heartbeatDeadlineTimer = null; -var heartbeatAwaitingSocket = null; var disconnectedAt = 0; var attemptStartedAt = 0; var attemptSocket = null; -var probeTimer = null; -var probeSocket = null; -var probeEpoch = 0; -var probeStartedAt = 0; var lifecycle = createWebSocketLifecycle({ handshakeTimeoutMs: 12000, onHandshakeTimeout: function (epoch) { @@ -60,6 +54,21 @@ var lifecycle = createWebSocketLifecycle({ } } }); +var watchdog = createWebSocketWatchdog({ + heartbeatIntervalMs: HEARTBEAT_MS, + heartbeatDeadlineMs: 10000, + probeTimeoutMs: 4000, + now: function () { return Date.now(); }, + isCurrent: function (socket, epoch) { + return getWs() === socket && socket.readyState === 1 && lifecycle.current(epoch); + }, + sendPing: function (socket) { + socket.send(JSON.stringify({ type: "ping" })); + }, + onFailure: function (reason, epoch, latency) { + forceReplaceSocket(reason, 0, epoch, latency); + } +}); function clearOverlayGrace() { if (overlayGraceTimer) { @@ -88,19 +97,10 @@ function recordConnectionDiagnostic(reason, code, latency) { }); } -function clearProbe() { - if (probeTimer) clearTimeout(probeTimer); - probeTimer = null; - probeSocket = null; - probeEpoch = 0; - probeStartedAt = 0; -} - -function forceReplaceSocket(reason, code, epoch) { +function forceReplaceSocket(reason, code, epoch, latency) { if (!lifecycle.current(epoch)) return; var socket = getWs(); - var latency = probeStartedAt ? Date.now() - probeStartedAt : null; - clearProbe(); + watchdog.clearProbe(); stopHeartbeat(); if (socket) { socket.onopen = null; @@ -117,19 +117,11 @@ function forceReplaceSocket(reason, code, epoch) { } export function stopHeartbeat() { - if (heartbeatTimer) { - clearInterval(heartbeatTimer); - heartbeatTimer = null; - } - if (heartbeatDeadlineTimer) { - clearTimeout(heartbeatDeadlineTimer); - heartbeatDeadlineTimer = null; - } - heartbeatAwaitingSocket = null; + watchdog.stopHeartbeat(); } function suspendSocketForOffline() { - clearProbe(); + watchdog.clearProbe(); stopHeartbeat(); var socket = getWs(); attemptSocket = null; @@ -150,31 +142,26 @@ function suspendSocketForOffline() { // timeout from expiring. A missing pong also identifies a half-open socket, // which browsers can otherwise leave OPEN while silently losing sends. export function startHeartbeat(socket, attemptEpoch) { - stopHeartbeat(); - if (!socket) return; - heartbeatTimer = setInterval(function () { - if (!socket || socket.readyState !== 1 || getWs() !== socket || !lifecycle.current(attemptEpoch)) { - stopHeartbeat(); - return; - } - try { socket.send(JSON.stringify({ type: "ping" })); } catch (e) { - forceReplaceSocket("heartbeat_error", 0, attemptEpoch); - return; - } - heartbeatAwaitingSocket = socket; - if (heartbeatDeadlineTimer) clearTimeout(heartbeatDeadlineTimer); - heartbeatDeadlineTimer = setTimeout(function () { - heartbeatDeadlineTimer = null; - if (getWs() === socket && socket.readyState === 1 && lifecycle.current(attemptEpoch)) { - forceReplaceSocket("heartbeat_timeout", 0, attemptEpoch); - } - }, 10000); - }, HEARTBEAT_MS); + watchdog.startHeartbeat(socket, attemptEpoch); +} + +function suspendSocketWatchdogs() { + // A frozen page cannot promise proxy keepalive. Retire its deadlines without + // closing the socket; resume performs a fresh bounded liveness check. + watchdog.suspend(); +} + +function resumeSocketWatchdogs(reason) { + var resumed = watchdog.resume(); + var socket = getWs(); + if (socket && socket.readyState === 1 && resumed) startHeartbeat(socket, lifecycle.getEpoch()); + probeReconnect(reason); } export function initConnection() { connectOverlay = document.getElementById("connect-overlay"); if (typeof navigator !== "undefined" && navigator.onLine === false) lifecycle.setOffline(true); + if (document.hidden) suspendSocketWatchdogs(); window.addEventListener("clay-message-delivery-timeout", function (event) { var socket = event.detail && event.detail.socket; recordConnectionDiagnostic("ack-timeout", 0, null); @@ -186,17 +173,19 @@ export function initConnection() { }); window.addEventListener("online", function () { lifecycle.wake(function () { - var socket = getWs(); - if (socket && socket.readyState === 1) startHeartbeat(socket, lifecycle.getEpoch()); - probeReconnect("online"); + if (document.hidden) return; + resumeSocketWatchdogs("online"); }); }); document.addEventListener("visibilitychange", function () { - if (!document.hidden) { - var socket = getWs(); - if (socket && socket.readyState === 1) startHeartbeat(socket, lifecycle.getEpoch()); - probeReconnect("visible"); - } + if (document.hidden) suspendSocketWatchdogs(); + else resumeSocketWatchdogs("visible"); + }); + document.addEventListener("freeze", function () { + suspendSocketWatchdogs(); + }); + document.addEventListener("resume", function () { + if (!document.hidden) resumeSocketWatchdogs("visible"); }); // --- Reactive UI sync for connected/processing state --- @@ -379,7 +368,7 @@ export function connect() { newWs.onclose = function (e) { if (getWs() !== newWs || !lifecycle.current(attemptEpoch)) return; stopHeartbeat(); - clearProbe(); + watchdog.clearProbe(); disconnectedAt = Date.now(); recordConnectionDiagnostic("close", e && e.code, attemptStartedAt ? Date.now() - attemptStartedAt : null); closeDmUserPicker(); @@ -408,18 +397,13 @@ export function connect() { var msg; try { msg = JSON.parse(event.data); } catch (e) { return; } lifecycle.markLive(attemptEpoch); - if (msg.type === "pong" && heartbeatDeadlineTimer && heartbeatAwaitingSocket === newWs) { - clearTimeout(heartbeatDeadlineTimer); - heartbeatDeadlineTimer = null; - heartbeatAwaitingSocket = null; - } - if (msg.type === "pong" && probeSocket === newWs && lifecycle.current(probeEpoch)) clearProbe(); + if (msg.type === "pong") watchdog.acceptPong(newWs, attemptEpoch); processMessage(msg); }; } export function cancelReconnect() { - clearProbe(); + watchdog.clearProbe(); stopHeartbeat(); lifecycle.cancel(); } @@ -429,18 +413,7 @@ function probeReconnect(reason) { var socket = getWs(); if (socket && socket.readyState === 0) return; if (socket && socket.readyState === 1) { - if (probeTimer) return; - probeSocket = socket; - probeEpoch = lifecycle.getEpoch(); - probeStartedAt = Date.now(); - try { socket.send(JSON.stringify({ type: "ping" })); } catch (e) { - forceReplaceSocket("probe_error", 0, probeEpoch); - return; - } - probeTimer = setTimeout(function () { - probeTimer = null; - if (probeSocket === socket && lifecycle.current(probeEpoch) && getWs() === socket && socket.readyState === 1) forceReplaceSocket("probe_timeout", 0, probeEpoch); - }, 4000); + watchdog.probe(socket, lifecycle.getEpoch()); return; } scheduleReconnect(reason); diff --git a/lib/public/modules/app-messages.js b/lib/public/modules/app-messages.js index c072a100..4845cd53 100644 --- a/lib/public/modules/app-messages.js +++ b/lib/public/modules/app-messages.js @@ -737,6 +737,10 @@ export function processMessage(msg) { if (!msg.ok && msg.error) showToast(msg.error, "error"); break; + case "split_worker_close_result": + if (!msg.ok && msg.error) showToast(msg.error, "error"); + break; + case "pair_session_options": showPairDialog(msg); break; diff --git a/lib/public/modules/project-activation.js b/lib/public/modules/project-activation.js index cc084633..a2d2b60b 100644 --- a/lib/public/modules/project-activation.js +++ b/lib/public/modules/project-activation.js @@ -59,19 +59,22 @@ export function isProjectSessionReady(state, socket) { } export function splitPanesMatchProject(split, slug) { - if (!split || !Array.isArray(split.panes) || split.panes.length !== 2 || !slug) return false; - return split.panes[0].slug === slug && split.panes[1].slug === slug; + if (!split || !Array.isArray(split.panes) || split.panes.length < 2 || split.panes.length > 3 || !slug) return false; + for (var i = 0; i < split.panes.length; i++) if (split.panes[i].slug !== slug) return false; + return true; } export function hasCurrentSplitGroup(split, groups) { if (!split || !split.groupId) return true; var members = split.panes && split.panes.map(function (pane) { return pane.sessionId; }); - if (!members || members.length !== 2) return false; + if (!members || members.length < 2 || members.length > 3) return false; var available = Array.isArray(groups) ? groups : []; for (var i = 0; i < available.length; i++) { var group = available[i]; - if (group.id !== split.groupId || !Array.isArray(group.members) || group.members.length !== 2) continue; - if (group.members.indexOf(members[0]) !== -1 && group.members.indexOf(members[1]) !== -1) return true; + if (group.id !== split.groupId || !Array.isArray(group.members) || group.members.length !== members.length) continue; + var matches = true; + for (var mi = 0; mi < members.length; mi++) if (group.members.indexOf(members[mi]) === -1) matches = false; + if (matches) return true; } return false; } @@ -84,8 +87,8 @@ export function reconcileRestoredSplit(split, groups) { if (group.id !== split.groupId) continue; var oldMembers = split.panes && split.panes.map(function (pane) { return pane.sessionId; }); var members = Array.isArray(group.members) ? group.members : []; - if (oldMembers && oldMembers.length === 2 && members.length === 2 - && members.indexOf(oldMembers[0]) !== -1 && members.indexOf(oldMembers[1]) !== -1) { + if (oldMembers && oldMembers.length === members.length + && oldMembers.every(function (id) { return members.indexOf(id) !== -1; })) { return { action: "keep", group: group }; } return { action: "rebuild", group: group }; diff --git a/lib/public/modules/sidebar-sessions.js b/lib/public/modules/sidebar-sessions.js index f1d7a9fc..ddce0aeb 100644 --- a/lib/public/modules/sidebar-sessions.js +++ b/lib/public/modules/sidebar-sessions.js @@ -1429,14 +1429,20 @@ function renderSplitGroupItem(group, members) { escapeHtml(VENDOR_NAMES[vendor] || vendor) + '">'; } html += ""; - if (members[0].isProcessing || members[1].isProcessing) html += ''; - var splitBackgroundTaskCount = (members[0].backgroundTaskCount || 0) + (members[1].backgroundTaskCount || 0); + var splitProcessing = false; + var splitBackgroundTaskCount = 0; + for (var si = 0; si < members.length; si++) { + if (members[si].isProcessing) splitProcessing = true; + splitBackgroundTaskCount += members[si].backgroundTaskCount || 0; + } + if (splitProcessing) html += ''; if (splitBackgroundTaskCount) html += '' + splitBackgroundTaskCount + ''; html += '' + highlightMatch(group.name || "Split", searchQuery) + ""; text.innerHTML = html; el.appendChild(text); - var unread = (members[0].unread || 0) + (members[1].unread || 0); + var unread = 0; + for (var ui = 0; ui < members.length; ui++) unread += members[ui].unread || 0; var badge = document.createElement("span"); badge.className = "session-unread-badge" + (unread > 0 ? " has-unread" : ""); badge.dataset.splitGroupId = group.id; @@ -1532,15 +1538,24 @@ export function renderSessionList(sessions) { } for (var sg = 0; sg < splitGroups.length; sg++) { var splitGroup = splitGroups[sg]; - var leftMember = sessionsById.get(splitGroup.members[0]); - var rightMember = sessionsById.get(splitGroup.members[1]); - if (!leftMember || !rightMember) continue; - if (hierarchySessionIds.has(leftMember.id) || hierarchySessionIds.has(rightMember.id)) continue; + var splitMembers = []; + for (var smi = 0; smi < splitGroup.members.length; smi++) { + var splitMember = sessionsById.get(splitGroup.members[smi]); + if (splitMember) splitMembers.push(splitMember); + } + if (splitMembers.length !== splitGroup.members.length) continue; + var hasHierarchyMember = false; + for (var hmi = 0; hmi < splitMembers.length; hmi++) { + if (hierarchySessionIds.has(splitMembers[hmi].id)) hasHierarchyMember = true; + } + if (hasHierarchyMember) continue; + var splitLastActivity = 0; + for (var lai = 0; lai < splitMembers.length; lai++) splitLastActivity = Math.max(splitLastActivity, splitMembers[lai].lastActivity || 0); items.push({ type: "split-group", group: splitGroup, - members: [leftMember, rightMember], - lastActivity: Math.max(leftMember.lastActivity || 0, rightMember.lastActivity || 0), + members: splitMembers, + lastActivity: splitLastActivity, }); } var groupKeys = Object.keys(loopGroups); @@ -1567,8 +1582,13 @@ export function renderSessionList(sessions) { continue; } if ((item.type === "driver-hierarchy" || item.type === "orphan-workers") && !hierarchyItemMatches(item, searchMatchIds)) continue; - if (item.type === "split-group" && searchMatchIds !== null && - !searchMatchIds.has(item.members[0].id) && !searchMatchIds.has(item.members[1].id)) continue; + if (item.type === "split-group" && searchMatchIds !== null) { + var splitSearchMatch = false; + for (var splitSearchIndex = 0; splitSearchIndex < item.members.length; splitSearchIndex++) { + if (searchMatchIds.has(item.members[splitSearchIndex].id)) splitSearchMatch = true; + } + if (!splitSearchMatch) continue; + } if ((item.type === "session" || item.type === "driver-hierarchy") && item.data && item.data.bookmarked) { bookmarkedItems.push(item); } else { diff --git a/lib/public/modules/split-group-helpers.js b/lib/public/modules/split-group-helpers.js index 2c0748f2..911dc094 100644 --- a/lib/public/modules/split-group-helpers.js +++ b/lib/public/modules/split-group-helpers.js @@ -12,15 +12,87 @@ export function findSplitGroup(groups, memberIds) { var list = groups || []; for (var i = 0; i < list.length; i++) { var members = list[i].members || []; - if (members.length === 2 && members[0] === memberIds[0] && members[1] === memberIds[1]) return list[i]; + if (members.length !== memberIds.length) continue; + var matches = true; + for (var j = 0; j < memberIds.length; j++) { + if (members.indexOf(memberIds[j]) === -1) { matches = false; break; } + } + if (matches) return list[i]; } return null; } +// Resolve the visible order from the explicit role contract. Member array +// order is persistence detail and must never decide who is the Driver. +export function splitGroupRoles(group) { + if (!group || !Array.isArray(group.members)) return null; + var pair = group.pair; + if (!pair || typeof pair !== "object" || Array.isArray(pair)) return null; + var hasVersion = Object.prototype.hasOwnProperty.call(pair, "version"); + var hasLegacyWorker = Object.prototype.hasOwnProperty.call(pair, "workerId"); + var hasWorkerIds = Object.prototype.hasOwnProperty.call(pair, "workerIds"); + if (hasVersion && pair.version !== 2) return null; + if (hasVersion && hasLegacyWorker) return null; + if (hasVersion && !hasWorkerIds) return null; + if (!hasVersion && hasWorkerIds) return null; + if (!hasVersion && !hasLegacyWorker) return null; + var driverId = pair.driverId; + var workerIds = hasWorkerIds ? (Array.isArray(pair.workerIds) ? pair.workerIds.slice() : []) : + (pair.workerId !== undefined ? [pair.workerId] : []); + if (!Number.isInteger(driverId) || workerIds.length < 1 || workerIds.length > 2) return null; + if (group.members.indexOf(driverId) === -1) return null; + var ordered = [driverId]; + for (var i = 0; i < workerIds.length; i++) { + if (!Number.isInteger(workerIds[i]) || workerIds[i] === driverId || group.members.indexOf(workerIds[i]) === -1 || ordered.indexOf(workerIds[i]) !== -1) return null; + ordered.push(workerIds[i]); + } + if (group.members.length !== ordered.length) return null; + for (var mi = 0; mi < group.members.length; mi++) if (ordered.indexOf(group.members[mi]) === -1) return null; + return { driverId: driverId, workerIds: workerIds, orderedIds: ordered, version: pair.version || null }; +} + +export function splitGroupMemberIds(group) { + var roles = splitGroupRoles(group); + return roles ? roles.orderedIds : (group && Array.isArray(group.members) ? group.members.slice() : []); +} + +export function splitGroupActiveAnchor(group, activeId) { + var roles = splitGroupRoles(group); + if (roles) return roles.driverId; + var members = group && Array.isArray(group.members) ? group.members : []; + return members.indexOf(activeId) !== -1 ? activeId : (members[0] || null); +} + export function isConfiguredWorker(groups, sessionId) { var list = groups || []; for (var i = 0; i < list.length; i++) { - if (list[i].pair && list[i].pair.workerId === sessionId) return true; + var roles = splitGroupRoles(list[i]); + if (roles && roles.workerIds.indexOf(sessionId) !== -1) return true; } return false; } + +export function splitWorkerCloseRequest(group, sessions, projectSlug, workerId, requestId) { + var roles = splitGroupRoles(group); + if (!roles || roles.version !== 2 || roles.workerIds.indexOf(workerId) === -1) return null; + var anchors = group && group.pairAnchors; + if (!anchors || anchors.version !== 2 || !anchors.driver || !Array.isArray(anchors.workers)) return null; + var workerIndex = roles.workerIds.indexOf(workerId); + var workerAnchor = anchors.workers[workerIndex]; + var worker = null; + var list = sessions || []; + for (var i = 0; i < list.length; i++) if (list[i].id === workerId) worker = list[i]; + if (!workerAnchor || !anchors.driver.origin || !workerAnchor.origin || !worker || !Number.isInteger(worker.workerGeneration)) return null; + return { + type: "split_worker_close", + requestId: requestId, + projectSlug: projectSlug, + groupId: group.id, + driverId: roles.driverId, + driverOriginId: anchors.driver.origin, + workerId: workerId, + workerOriginId: workerAnchor.origin, + generation: worker.workerGeneration, + expectedWorkerIds: roles.workerIds.slice(), + }; +} diff --git a/lib/public/modules/split-pair-ui.js b/lib/public/modules/split-pair-ui.js index c1c892a6..789f74cf 100644 --- a/lib/public/modules/split-pair-ui.js +++ b/lib/public/modules/split-pair-ui.js @@ -5,6 +5,7 @@ import { buildAgentVendorSelect, fillAgentModels, buildAgentEffortSelect, fillAg import { showToast } from './utils.js'; import { getCachedSessions } from './sidebar-sessions.js'; import { appendWorkerRuntime } from './split-worker-runtime.js'; +import { splitGroupRoles } from './split-group-helpers.js'; var pairDialog = null; // Set when the dialog was opened via "Add Split Worker" on an existing session: @@ -245,6 +246,8 @@ export function syncPairChrome(host, split) { var paneEls = host.querySelectorAll(".split-pane"); for (var paneIndex = 0; paneIndex < paneEls.length; paneIndex++) { paneEls[paneIndex].classList.remove("split-pane-role-driver", "split-pane-role-worker"); + paneEls[paneIndex].style.removeProperty("--split-pane-order"); + paneEls[paneIndex].style.removeProperty("--split-worker-row"); } var runtimeHeaders = host.querySelectorAll(".has-worker-runtime"); for (var ri = 0; ri < runtimeHeaders.length; ri++) runtimeHeaders[ri].classList.remove("has-worker-runtime"); @@ -268,10 +271,23 @@ export function syncPairChrome(host, split) { } return; } + var roles = splitGroupRoles(group); + if (!roles) return; + var workerIndex = 0; for (var pi = 0; pi < split.panes.length && pi < headers.length; pi++) { - (function (sessionId, header) { - var isDriver = sessionId === group.pair.driverId; - if (paneEls[pi]) paneEls[pi].classList.add(isDriver ? "split-pane-role-driver" : "split-pane-role-worker"); + var desiredPane = split.panes[pi]; + var desiredPaneEl = null; + for (var pei = 0; pei < paneEls.length; pei++) { + if (paneEls[pei].dataset.sessionId === String(desiredPane.sessionId)) desiredPaneEl = paneEls[pei]; + } + if (!desiredPaneEl) continue; + (function (sessionId, header, paneEl) { + var isDriver = sessionId === roles.driverId; + if (paneEl) { + paneEl.classList.add(isDriver ? "split-pane-role-driver" : "split-pane-role-worker"); + paneEl.style.setProperty("--split-pane-order", isDriver ? "2" : String(workerIndex)); + if (!isDriver) paneEl.style.setProperty("--split-worker-row", String(workerIndex + 1)); + } var role = isDriver ? "Driver" : "Split Worker"; var badge = document.createElement("span"); badge.className = "split-pair-role split-pair-role-" + role.toLowerCase().replace(/ /g, "-"); @@ -287,18 +303,24 @@ export function syncPairChrome(host, split) { if (sessions[wi].id === sessionId) { workerSession = sessions[wi]; break; } } appendWorkerRuntime(header, workerSession); + workerIndex++; } - })(split.panes[pi].sessionId, headers[pi]); + })(desiredPane.sessionId, desiredPaneEl.querySelector(".split-pane-header"), desiredPaneEl); } var active = (store.get('splitDelegations') || {})[group.id]; if (!active || !active.active) return; host.classList.add("split-delegating"); var fromIndex = split.panes[0].sessionId === active.from ? 0 : 1; host.classList.add(fromIndex === 0 ? "split-direction-right" : "split-direction-left"); - for (var si = 0; si < split.panes.length && si < headers.length; si++) { + for (var si = 0; si < split.panes.length; si++) { + var livePane = null; + for (var lpi = 0; lpi < paneEls.length; lpi++) { + if (paneEls[lpi].dataset.sessionId === String(split.panes[si].sessionId)) livePane = paneEls[lpi]; + } + if (!livePane) continue; var live = document.createElement("span"); live.className = "split-pair-live"; live.textContent = split.panes[si].sessionId === active.to ? "Following direction" : "Waiting"; - headers[si].appendChild(live); + livePane.querySelector(".split-pane-header").appendChild(live); } } diff --git a/lib/public/modules/split-pane-reconciler.js b/lib/public/modules/split-pane-reconciler.js new file mode 100644 index 00000000..0dd3f2d2 --- /dev/null +++ b/lib/public/modules/split-pane-reconciler.js @@ -0,0 +1,28 @@ +// Keyed DOM reconciliation for the parent split shell. Existing pane nodes +// (and their iframe browsing contexts) are reused by stable project/session +// identity while metadata and visual order are refreshed. + +export function splitPaneIdentity(pane) { + return String(pane && pane.slug || "") + "#" + String(pane && pane.sessionId || ""); +} + +export function reconcileSplitPanes(host, panes, createPane, updatePane) { + var desired = Array.isArray(panes) ? panes : []; + var existing = host.querySelectorAll(".split-pane"); + var byKey = new Map(); + for (var i = 0; i < existing.length; i++) byKey.set(existing[i].dataset.splitPaneKey, existing[i]); + var used = new Set(); + for (var pi = 0; pi < desired.length; pi++) { + var pane = desired[pi]; + var key = splitPaneIdentity(pane); + var paneEl = byKey.get(key); + if (!paneEl) paneEl = createPane(pane, pi); + paneEl.dataset.splitPaneKey = key; + updatePane(paneEl, pane, pi); + used.add(paneEl); + if (!paneEl.parentNode) host.appendChild(paneEl); + } + for (var ei = existing.length - 1; ei >= 0; ei--) { + if (!used.has(existing[ei])) existing[ei].remove(); + } +} diff --git a/lib/public/modules/split-pane-renderer.js b/lib/public/modules/split-pane-renderer.js new file mode 100644 index 00000000..6a9e14e4 --- /dev/null +++ b/lib/public/modules/split-pane-renderer.js @@ -0,0 +1,107 @@ +import { store } from './store.js'; +import { getWs } from './ws-ref.js'; +import { getCachedSessions } from './sidebar-sessions.js'; +import { iconHtml } from './icons.js'; +import { formatTokens } from './app-panels.js'; +import { VENDOR_AVATARS, VENDOR_NAMES } from './app-rendering.js'; +import { createPermissionControl, bindPermissionControl, renderPermissionControl } from './permission-control.js'; +import { splitGroupRoles } from './split-group-helpers.js'; + +function sessionById(sessionId) { + var sessions = getCachedSessions() || []; + for (var i = 0; i < sessions.length; i++) if (sessions[i].id === sessionId) return sessions[i]; + return null; +} + +function paneUrl(pane) { + return "/p/" + encodeURIComponent(pane.slug) + "/?pane=1&session=" + encodeURIComponent(pane.sessionId); +} + +function startPaneRename(header, titleEl, pane) { + if (header.querySelector(".split-pane-rename-input")) return; + var input = document.createElement("input"); + input.type = "text"; input.className = "split-pane-rename-input"; input.value = pane.title; + titleEl.style.display = "none"; header.insertBefore(input, titleEl); input.focus(); input.select(); + var done = false; + function finish(commit) { + if (done) return; + done = true; + var newTitle = input.value.trim(); input.remove(); titleEl.style.display = ""; + if (!commit || !newTitle || newTitle === pane.title) return; + pane.title = newTitle; titleEl.textContent = newTitle; + var ws = getWs(); + if (ws && ws.readyState === 1) ws.send(JSON.stringify({ type: "rename_session", id: pane.sessionId, title: newTitle })); + } + input.addEventListener("keydown", function (event) { + if (event.key === "Enter") { event.preventDefault(); finish(true); } + if (event.key === "Escape") { event.preventDefault(); finish(false); } + }); + input.addEventListener("blur", function () { finish(true); }); + input.addEventListener("click", function (event) { event.stopPropagation(); }); +} + +function permissionState(session) { + var mode = session && (session.runtimeMode || session.mode || "gui"); + var worker = !!session && isConfiguredWorker(store.get('splitGroups'), session.id); + return { + projectSlug: store.get('currentSlug'), sessionId: session && session.id, + vendor: session && session.vendor || store.get('currentVendor') || "claude", + permissionMode: session && session.permissionMode || "default", + effectivePermissionMode: session && session.effectivePermissionMode || null, + permissionCapabilities: session && session.permissionCapabilities || { auto: false, mcpOverride: false }, + connected: store.get('connected'), globalPermissionModeForced: store.get('skipPermsEnabled') === true, + visible: !!session && !worker && mode === "gui", locked: worker, + }; +} + +function isConfiguredWorker(groups, sessionId) { + for (var i = 0; i < (groups || []).length; i++) { + var roles = splitGroupRoles(groups[i]); + if (roles && roles.workerIds.indexOf(sessionId) !== -1) return true; + } + return false; +} + +export function updatePaneFullAccessButton(button, session) { + if (button) renderPermissionControl(button, permissionState(session)); +} + +export function createSplitPane(pane, closePane) { + var paneEl = document.createElement("section"); paneEl.className = "split-pane"; + var header = document.createElement("header"); header.className = "split-pane-header"; header.title = pane.title; + var session = sessionById(pane.sessionId); var vendor = session && session.vendor || "claude"; + var vendorIcon = document.createElement("img"); vendorIcon.className = "split-pane-vendor"; + vendorIcon.src = VENDOR_AVATARS[vendor] || VENDOR_AVATARS.claude; vendorIcon.alt = ""; vendorIcon.title = VENDOR_NAMES[vendor] || vendor; header.appendChild(vendorIcon); + var title = document.createElement("span"); title.className = "split-pane-title"; title.textContent = pane.title; title.title = "Rename session"; + title.addEventListener("click", function () { startPaneRename(header, title, pane); }); header.appendChild(title); + var frame = document.createElement("iframe"); frame.className = "split-pane-frame"; frame.dataset.projectSlug = pane.slug; + frame.dataset.sessionId = String(pane.sessionId); frame.src = paneUrl(pane); frame.title = pane.title; + var ctxChip = document.createElement("button"); ctxChip.type = "button"; ctxChip.className = "split-pane-context"; ctxChip.title = "Context usage"; + ctxChip.innerHTML = ''; + ctxChip.addEventListener("click", function () { if (frame.contentWindow) frame.contentWindow.postMessage({ type: "clay-pane-toggle-context" }, window.location.origin); }); header.appendChild(ctxChip); + var fullAccess = createPermissionControl("split-pane-full-access hidden", "Driver session permission mode"); + updatePaneFullAccessButton(fullAccess, session); bindPermissionControl(fullAccess, function () { return permissionState(sessionById(pane.sessionId)); }); header.appendChild(fullAccess); header.insertBefore(fullAccess, ctxChip); + var close = document.createElement("button"); close.type = "button"; close.className = "split-pane-close"; close.title = "Close pane"; close.setAttribute("aria-label", "Close pane"); close.innerHTML = iconHtml("x"); + close.dataset.sessionId = String(pane.sessionId); close.addEventListener("click", function () { closePane(Number(this.dataset.sessionId)); }); header.appendChild(close); + paneEl.appendChild(header); paneEl.appendChild(frame); return paneEl; +} + +export function updateSplitPaneMetadata(paneEl, pane) { + var session = sessionById(pane.sessionId); var header = paneEl.querySelector(".split-pane-header"); + var title = paneEl.querySelector(".split-pane-title"); var frame = paneEl.querySelector(".split-pane-frame"); var vendor = paneEl.querySelector(".split-pane-vendor"); + paneEl.dataset.sessionId = String(pane.sessionId); paneEl.dataset.projectSlug = pane.slug; + if (header) header.title = pane.title; if (title && title.style.display !== "none") title.textContent = pane.title; if (frame) frame.title = pane.title; + if (vendor) { var vendorName = session && session.vendor || "claude"; vendor.src = VENDOR_AVATARS[vendorName] || VENDOR_AVATARS.claude; vendor.title = VENDOR_NAMES[vendorName] || vendorName; } + var close = paneEl.querySelector(".split-pane-close"); + if (close) { + close.dataset.sessionId = String(pane.sessionId); close.disabled = false; close.title = "Close pane"; close.setAttribute("aria-label", close.title); + } +} + +export function updatePaneContextChip(paneEl, msg) { + if (!paneEl) return; var chip = paneEl.querySelector(".split-pane-context"); if (!chip) return; var pct = msg.pct || 0; if (pct <= 0) return; + chip.classList.add("has-data"); var fill = chip.querySelector(".split-pane-context-fill"); var label = chip.querySelector(".split-pane-context-label"); + fill.style.width = Math.min(100, pct).toFixed(1) + "%"; fill.className = "split-pane-context-fill" + (msg.cls || ""); label.textContent = pct.toFixed(0) + "%"; + var tip = "Context " + pct.toFixed(0) + "% (" + formatTokens(msg.used || 0) + " / " + formatTokens(msg.win || 0) + " tokens)"; + if (msg.cost) tip += " · $" + msg.cost.toFixed(4); if (msg.model && msg.model !== "-") tip += " · " + msg.model; chip.title = tip; +} diff --git a/lib/public/modules/split-view.js b/lib/public/modules/split-view.js index 979e671e..475f6550 100644 --- a/lib/public/modules/split-view.js +++ b/lib/public/modules/split-view.js @@ -3,20 +3,19 @@ import { store } from './store.js'; import { getWs } from './ws-ref.js'; import { getCachedSessions } from './sidebar-sessions.js'; -import { iconHtml, refreshIcons } from './icons.js'; +import { refreshIcons } from './icons.js'; import { detachTuiView } from './session-tui-view.js'; -import { formatTokens } from './app-panels.js'; -import { VENDOR_AVATARS, VENDOR_NAMES } from './app-rendering.js'; -import { groupedSessionIds, findSplitGroup, isConfiguredWorker } from './split-group-helpers.js'; +import { groupedSessionIds, findSplitGroup, isConfiguredWorker, splitGroupRoles, splitGroupMemberIds, splitGroupActiveAnchor, splitWorkerCloseRequest } from './split-group-helpers.js'; import { syncPairChrome } from './split-pair-ui.js'; import { presentMarkdownEdit } from './filebrowser.js'; import { autoStartLoginIfNeeded } from './vendor-login.js'; import { isCurrentProjectSessionReady } from './split-session-boundary.js'; -import { splitPanesMatchProject, hasCurrentSplitGroup, reconcileRestoredSplit } from './project-activation.js'; +import { splitPanesMatchProject, reconcileRestoredSplit } from './project-activation.js'; import { handlePaneIssueMessage } from './pane-issue-bridge.js'; import { handlePaneFileMessage, revealFilesPanel } from './pane-file-bridge.js'; import { openFile } from './filebrowser.js'; -import { bindPermissionControl, createPermissionControl, renderPermissionControl } from './permission-control.js'; +import { reconcileSplitPanes } from './split-pane-reconciler.js'; +import { createSplitPane, updateSplitPaneMetadata, updatePaneContextChip, updatePaneFullAccessButton } from './split-pane-renderer.js'; var host = null; var nativeApp = null; @@ -59,10 +58,6 @@ function paneForSession(sessionId) { }; } -function paneUrl(pane) { - return "/p/" + encodeURIComponent(pane.slug) + "/?pane=1&session=" + encodeURIComponent(pane.sessionId); -} - // Arc-style drop preview: hovering one half folds the live app into the // other half and shows a ghost pane where the dragged session will land. function setPreviewSide(side) { @@ -99,48 +94,28 @@ function switchNativeSession(sessionId, expectedSlug) { } } -function closePane(index) { +function closePane(sessionId) { var split = store.get('splitPanes'); - if (!split || !split.panes || split.panes.length !== 2) return; + if (!split || !split.panes || split.panes.length < 2) return; + var index = -1; + for (var pi = 0; pi < split.panes.length; pi++) if (split.panes[pi].sessionId === sessionId) index = pi; + if (index === -1) return; if (split.groupId && getWs() && getWs().readyState === 1) { - getWs().send(JSON.stringify({ type: "split_group_dissolve", id: split.groupId })); - } - switchNativeSession(split.panes[index === 0 ? 1 : 0].sessionId, split.panes[index === 0 ? 1 : 0].slug); -} - -function startPaneRename(header, titleEl, pane) { - if (header.querySelector(".split-pane-rename-input")) return; - var input = document.createElement("input"); - input.type = "text"; - input.className = "split-pane-rename-input"; - input.value = pane.title; - titleEl.style.display = "none"; - header.insertBefore(input, titleEl); - input.focus(); - input.select(); - - var done = false; - function finish(commit) { - if (done) return; - done = true; - var newTitle = input.value.trim(); - input.remove(); - titleEl.style.display = ""; - if (!commit || !newTitle || newTitle === pane.title) return; - pane.title = newTitle; - titleEl.textContent = newTitle; - var ws = getWs(); - if (ws && ws.readyState === 1) { - ws.send(JSON.stringify({ type: "rename_session", id: pane.sessionId, title: newTitle })); + var group = null; + var groups = store.get('splitGroups') || []; + for (var gi = 0; gi < groups.length; gi++) if (groups[gi].id === split.groupId) group = groups[gi]; + var roles = splitGroupRoles(group); + var closingId = sessionId; + if (roles && roles.version === 2 && roles.workerIds.indexOf(closingId) !== -1) { + var request = splitWorkerCloseRequest(group, getCachedSessions(), store.get('currentSlug'), closingId, + "worker-close-" + Date.now().toString(36) + "-" + Math.random().toString(36).slice(2, 8)); + if (request) getWs().send(JSON.stringify(request)); + return; } + getWs().send(JSON.stringify({ type: "split_group_dissolve", id: split.groupId })); } - - input.addEventListener("keydown", function (event) { - if (event.key === "Enter") { event.preventDefault(); finish(true); } - if (event.key === "Escape") { event.preventDefault(); finish(false); } - }); - input.addEventListener("blur", function () { finish(true); }); - input.addEventListener("click", function (event) { event.stopPropagation(); }); + var next = split.panes[index === 0 ? 1 : 0]; + switchNativeSession(next.sessionId, next.slug); } // Called on session_list so pane headers follow renames made elsewhere @@ -163,117 +138,6 @@ export function syncPaneTitles() { syncPairChrome(host, split); } -function panePermissionControlState(session) { - var mode = session && (session.runtimeMode || session.mode || "gui"); - var worker = !!session && isConfiguredWorker(store.get('splitGroups'), session.id); - return { - projectSlug: store.get('currentSlug'), - sessionId: session && session.id, - vendor: session && session.vendor || store.get('currentVendor') || "claude", - permissionMode: session && session.permissionMode || "default", - effectivePermissionMode: session && session.effectivePermissionMode || null, - permissionCapabilities: session && session.permissionCapabilities || { auto: false, mcpOverride: false }, - connected: store.get('connected'), - globalPermissionModeForced: store.get('skipPermsEnabled') === true, - visible: !!session && !worker && mode === "gui", - locked: worker, - }; -} - -function updatePaneFullAccessButton(button, session) { - if (!button) return; - // Permission policy belongs to the Driver. A configured Worker inherits it - // server-side and must not present a second, misleading control. - renderPermissionControl(button, panePermissionControlState(session)); -} - -function createPane(pane, index) { - var paneEl = document.createElement("section"); - paneEl.className = "split-pane"; - - var header = document.createElement("header"); - header.className = "split-pane-header"; - header.title = pane.title; - - var session = sessionById(pane.sessionId); - var vendor = (session && session.vendor) || "claude"; - var vendorIcon = document.createElement("img"); - vendorIcon.className = "split-pane-vendor"; - vendorIcon.src = VENDOR_AVATARS[vendor] || VENDOR_AVATARS.claude; - vendorIcon.alt = ""; - vendorIcon.title = VENDOR_NAMES[vendor] || vendor; - header.appendChild(vendorIcon); - - var title = document.createElement("span"); - title.className = "split-pane-title"; - title.textContent = pane.title; - title.title = "Rename session"; - title.addEventListener("click", function () { startPaneRename(header, title, pane); }); - header.appendChild(title); - - var frame = document.createElement("iframe"); - frame.className = "split-pane-frame"; - frame.dataset.projectSlug = pane.slug; - frame.dataset.sessionId = String(pane.sessionId); - frame.src = paneUrl(pane); - frame.title = pane.title; - - // Context-usage chip, fed by clay-pane-context messages from the pane - // iframe. Hidden until the pane reports a non-zero context percentage. - var ctxChip = document.createElement("button"); - ctxChip.type = "button"; - ctxChip.className = "split-pane-context"; - ctxChip.title = "Context usage"; - ctxChip.innerHTML = ''; - ctxChip.addEventListener("click", function () { - if (frame.contentWindow) { - frame.contentWindow.postMessage({ type: "clay-pane-toggle-context" }, window.location.origin); - } - }); - header.appendChild(ctxChip); - - var fullAccess = createPermissionControl("split-pane-full-access hidden", "Driver session permission mode"); - updatePaneFullAccessButton(fullAccess, session); - bindPermissionControl(fullAccess, function() { - var current = sessionById(pane.sessionId); - return panePermissionControlState(current); - }); - header.appendChild(fullAccess); - // Permission state belongs to the session identity on the left. Keep it - // immediately after the title instead of grouping it with usage and close. - header.insertBefore(fullAccess, ctxChip); - - var close = document.createElement("button"); - close.type = "button"; - close.className = "split-pane-close"; - close.title = "Close pane"; - close.setAttribute("aria-label", "Close pane"); - close.innerHTML = iconHtml("x"); - close.addEventListener("click", function () { closePane(index); }); - header.appendChild(close); - paneEl.appendChild(header); - paneEl.appendChild(frame); - return paneEl; -} - -function updatePaneContextChip(paneEl, msg) { - if (!paneEl) return; - var chip = paneEl.querySelector(".split-pane-context"); - if (!chip) return; - var pct = msg.pct || 0; - if (pct <= 0) return; - chip.classList.add("has-data"); - var fill = chip.querySelector(".split-pane-context-fill"); - var label = chip.querySelector(".split-pane-context-label"); - fill.style.width = Math.min(100, pct).toFixed(1) + "%"; - fill.className = "split-pane-context-fill" + (msg.cls || ""); - label.textContent = pct.toFixed(0) + "%"; - var tip = "Context " + pct.toFixed(0) + "% (" + formatTokens(msg.used || 0) + " / " + formatTokens(msg.win || 0) + " tokens)"; - if (msg.cost) tip += " · $" + msg.cost.toFixed(4); - if (msg.model && msg.model !== "-") tip += " · " + msg.model; - chip.title = tip; -} - function handlePaneMessage(event) { if (event.origin !== window.location.origin) return; var msg = event.data; @@ -301,34 +165,20 @@ function handlePaneMessage(event) { } } -var renderedPaneKey = null; - -function paneKey(panes) { - return panes.map(function (pane) { return pane.slug + "#" + pane.sessionId; }).join("|"); -} - function renderSplit(split) { if (!host || !nativeApp) return; var panes = split && split.panes; - if (!panes || panes.length !== 2) { + if (!panes || panes.length < 2 || panes.length > 3) { placeStickyNotesOverlay(false); host.innerHTML = ""; - renderedPaneKey = null; host.classList.remove("visible"); nativeApp.classList.remove("split-native-hidden"); return; } - // splitPanes is replaced (same panes, new object) when the server confirms - // the groupId. Rebuilding then would reload both iframes, so skip DOM work - // when the rendered pane set is unchanged. - var key = paneKey(panes); placeStickyNotesOverlay(true); - if (key === renderedPaneKey && host.classList.contains("visible")) return; - host.innerHTML = ""; - renderedPaneKey = key; nativeApp.classList.add("split-native-hidden"); host.classList.add("visible"); - for (var i = 0; i < panes.length; i++) host.appendChild(createPane(panes[i], i)); + reconcileSplitPanes(host, panes, function (pane) { return createSplitPane(pane, closePane); }, updateSplitPaneMetadata); syncPairChrome(host, split); refreshIcons(); } @@ -360,23 +210,26 @@ function openSplit(side, draggedId) { } export function openGroup(group) { - if (!group || !Array.isArray(group.members) || group.members.length !== 2) return false; - if (!sessionById(group.members[0]) || !sessionById(group.members[1])) return false; + if (!group || !Array.isArray(group.members) || group.members.length < 2 || group.members.length > 3) return false; + var memberIds = splitGroupMemberIds(group); + if (memberIds.length !== group.members.length) return false; + for (var mi = 0; mi < memberIds.length; mi++) if (!sessionById(memberIds[mi])) return false; detachTuiView(); store.set({ splitPanes: { groupId: group.id, - panes: [paneForSession(group.members[0]), paneForSession(group.members[1])], + panes: memberIds.map(function (id) { return paneForSession(id); }), }, }); // Anchor the parent's active session (and server-side presence) to a // member while the split is open, so a hard refresh restores into this // group instead of whatever was viewed before it. var activeId = store.get('activeSessionId'); - if (activeId !== group.members[0] && activeId !== group.members[1]) { + var anchorId = splitGroupActiveAnchor(group, activeId); + if (anchorId && anchorId !== activeId) { var ws = getWs(); if (isCurrentProjectSessionReady(store.snap(), ws)) { - ws.send(JSON.stringify({ type: "switch_session", id: group.members[0] })); + ws.send(JSON.stringify({ type: "switch_session", id: anchorId })); } } dismissSplitOverlays(); @@ -403,13 +256,13 @@ export function maybeRestoreSplitGroup() { } if (reconciliation.action === "rebuild") { var rebuilt = reconciliation.group; - if (!sessionById(rebuilt.members[0]) || !sessionById(rebuilt.members[1])) { + if (rebuilt.members.some(function (id) { return !sessionById(id); })) { store.set({ splitPanes: null }); return; } store.set({ splitPanes: { groupId: rebuilt.id, - panes: [paneForSession(rebuilt.members[0]), paneForSession(rebuilt.members[1])], + panes: splitGroupMemberIds(rebuilt).map(function (id) { return paneForSession(id); }), } }); } return; @@ -427,13 +280,13 @@ export function maybeRestoreSplitGroup() { } export function separateGroup(group) { - if (!group || !Array.isArray(group.members) || group.members.length !== 2) return; + if (!group || !Array.isArray(group.members) || group.members.length < 2) return; var ws = getWs(); if (ws && ws.readyState === 1) { ws.send(JSON.stringify({ type: "split_group_dissolve", id: group.id })); } var split = store.get('splitPanes'); - if (split && split.groupId === group.id) switchNativeSession(group.members[0]); + if (split && split.groupId === group.id) switchNativeSession(split.panes[0].sessionId); } function dismissSplitOverlays() { @@ -531,24 +384,41 @@ export function initSplitView() { // notification click) closes the split UI; the group itself persists. if (state.activeSessionId !== prev.activeSessionId && state.splitPanes && state.splitPanes.panes) { var sp = state.splitPanes.panes; - if (state.activeSessionId !== sp[0].sessionId && state.activeSessionId !== sp[1].sessionId) { + var activePane = false; + for (var spi = 0; spi < sp.length; spi++) if (state.activeSessionId === sp[spi].sessionId) activePane = true; + if (!activePane) { store.set({ splitPanes: null }); } } if (state.splitGroups !== prev.splitGroups) { if (!isCurrentProjectSessionReady(state, getWs())) return; var split = state.splitPanes; - if (!split || !split.panes) return; + if (!split || !split.panes || split.panes.length < 2) return; if (!splitPanesMatchProject(split, state.currentSlug)) { store.set({ splitPanes: null }); return; } syncPairChrome(host, split); if (split.groupId) { - if (!hasCurrentSplitGroup(split, state.splitGroups)) switchNativeSession(split.panes[0].sessionId, state.currentSlug); + var currentGroup = null; + for (var cgi = 0; cgi < state.splitGroups.length; cgi++) { + if (state.splitGroups[cgi].id === split.groupId) currentGroup = state.splitGroups[cgi]; + } + if (!currentGroup) { + switchNativeSession(split.panes[0].sessionId, state.currentSlug); + return; + } + var projectedIds = splitGroupMemberIds(currentGroup); + if (projectedIds.length < 2 || projectedIds.length > 3) return; + var projectedPanes = projectedIds.map(function (id) { return paneForSession(id); }); + var projectedKey = projectedPanes.map(function (pane) { return pane.slug + "#" + pane.sessionId; }).join("|"); + var currentKey = split.panes.map(function (pane) { return pane.slug + "#" + pane.sessionId; }).join("|"); + if (projectedKey !== currentKey) { + store.set({ splitPanes: { groupId: split.groupId, panes: projectedPanes } }); + } return; } - var ids = [split.panes[0].sessionId, split.panes[1].sessionId]; + var ids = split.panes.map(function (pane) { return pane.sessionId; }); var confirmed = findSplitGroup(state.splitGroups, ids); if (confirmed) store.set({ splitPanes: { groupId: confirmed.id, panes: split.panes } }); } diff --git a/lib/public/modules/websocket-watchdog.js b/lib/public/modules/websocket-watchdog.js new file mode 100644 index 00000000..ba72a295 --- /dev/null +++ b/lib/public/modules/websocket-watchdog.js @@ -0,0 +1,126 @@ +// websocket-watchdog.js - suspendable liveness timers for the browser socket + +function createWebSocketWatchdog(options) { + var suspended = false; + var heartbeatTimer = null; + var heartbeatDeadlineTimer = null; + var heartbeatSocket = null; + var heartbeatEpoch = 0; + var heartbeatGeneration = 0; + var deadlineGeneration = 0; + var probeTimer = null; + var probeSocket = null; + var probeEpoch = 0; + var probeStartedAt = 0; + var probeGeneration = 0; + + function clearHeartbeatDeadline() { + deadlineGeneration += 1; + if (heartbeatDeadlineTimer) clearTimeout(heartbeatDeadlineTimer); + heartbeatDeadlineTimer = null; + } + + function stopHeartbeat() { + heartbeatGeneration += 1; + clearHeartbeatDeadline(); + if (heartbeatTimer) clearInterval(heartbeatTimer); + heartbeatTimer = null; + heartbeatSocket = null; + heartbeatEpoch = 0; + } + + function clearProbe() { + probeGeneration += 1; + if (probeTimer) clearTimeout(probeTimer); + probeTimer = null; + probeSocket = null; + probeEpoch = 0; + probeStartedAt = 0; + } + + function suspend() { + suspended = true; + stopHeartbeat(); + clearProbe(); + } + + function resume() { + var changed = suspended; + suspended = false; + return changed; + } + + function startHeartbeat(socket, epoch) { + stopHeartbeat(); + if (suspended || !socket) return false; + heartbeatSocket = socket; + heartbeatEpoch = epoch; + var generation = heartbeatGeneration; + heartbeatTimer = setInterval(function () { + if (suspended || generation !== heartbeatGeneration) return; + if (!options.isCurrent(socket, epoch)) { + stopHeartbeat(); + return; + } + clearHeartbeatDeadline(); + var currentDeadlineGeneration = deadlineGeneration; + heartbeatDeadlineTimer = setTimeout(function () { + if (suspended || currentDeadlineGeneration !== deadlineGeneration) return; + heartbeatDeadlineTimer = null; + if (heartbeatSocket === socket && heartbeatEpoch === epoch && options.isCurrent(socket, epoch)) { + options.onFailure("heartbeat_timeout", epoch, null); + } + }, options.heartbeatDeadlineMs); + try { + options.sendPing(socket); + } catch (e) { + options.onFailure("heartbeat_error", epoch, null); + } + }, options.heartbeatIntervalMs); + return true; + } + + function probe(socket, epoch) { + if (suspended) return "suspended"; + if (probeTimer) return "pending"; + probeSocket = socket; + probeEpoch = epoch; + probeStartedAt = options.now(); + probeGeneration += 1; + var generation = probeGeneration; + probeTimer = setTimeout(function () { + if (suspended || generation !== probeGeneration) return; + probeTimer = null; + if (probeSocket === socket && probeEpoch === epoch && options.isCurrent(socket, epoch)) { + options.onFailure("probe_timeout", epoch, options.now() - probeStartedAt); + } + }, options.probeTimeoutMs); + try { + options.sendPing(socket); + } catch (e) { + if (generation !== probeGeneration) return "cleared"; + var latency = options.now() - probeStartedAt; + clearProbe(); + options.onFailure("probe_error", epoch, latency); + return "failed"; + } + return "started"; + } + + function acceptPong(socket, epoch) { + if (heartbeatSocket === socket && heartbeatEpoch === epoch) clearHeartbeatDeadline(); + if (probeSocket === socket && probeEpoch === epoch) clearProbe(); + } + + return { + acceptPong: acceptPong, + clearProbe: clearProbe, + probe: probe, + resume: resume, + startHeartbeat: startHeartbeat, + stopHeartbeat: stopHeartbeat, + suspend: suspend + }; +} + +export { createWebSocketWatchdog }; diff --git a/lib/public/modules/worker-pane-lock.js b/lib/public/modules/worker-pane-lock.js index b4f62461..84e037ae 100644 --- a/lib/public/modules/worker-pane-lock.js +++ b/lib/public/modules/worker-pane-lock.js @@ -27,6 +27,18 @@ import { store } from './store.js'; var STATUS_ID = "worker-pane-status"; var LOCK_CLASS = "worker-pane-locked"; +function configuredWorkerIds(group) { + if (!group || !group.pair || !Array.isArray(group.members)) return []; + var pair = group.pair; + var ids = Array.isArray(pair.workerIds) ? pair.workerIds : + (pair.workerId !== undefined ? [pair.workerId] : []); + if (pair.driverId === undefined || ids.length < 1 || ids.length > 2 || group.members.indexOf(pair.driverId) === -1) return []; + for (var i = 0; i < ids.length; i++) { + if (ids[i] === pair.driverId || group.members.indexOf(ids[i]) === -1) return []; + } + return ids; +} + // Every interactive composer affordance. Queried lazily: a pane iframe and the // main window share this markup, and some ids are mobile-only. var LOCKED_CONTROL_IDS = [ @@ -64,9 +76,7 @@ export function isDriverOperatedView(state) { if (!sessionId) return false; var groups = snapshot.splitGroups || []; for (var i = 0; i < groups.length; i++) { - var pair = groups[i] && groups[i].pair; - if (!pair) continue; - if (pair.workerId === sessionId) return true; + if (configuredWorkerIds(groups[i]).indexOf(sessionId) !== -1) return true; } return false; } diff --git a/lib/server.js b/lib/server.js index ae00af66..c144de91 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1336,6 +1336,7 @@ function createServer(opts) { worktreeMeta: worktreeMeta || null, mcpBridgeToken: extra.mcpBridgeToken || null, isMate: extra.isMate || false, + multiWorkerRuntimeEnabled: !extra.isMate, mateId: extra.mateId || (extra.isMate ? path.basename(cwd) : null), mateDisplayName: extra.mateDisplayName || "", isHostAgent: !!extra.isHostAgent, diff --git a/lib/session-pair-factory.js b/lib/session-pair-factory.js index 7822f3d3..6da78ead 100644 --- a/lib/session-pair-factory.js +++ b/lib/session-pair-factory.js @@ -34,6 +34,7 @@ var driverEligibility = require("./session-driver-eligibility"); var models = require("./project-models"); var yoke = require("./yoke"); var sessionProvenance = require("./session-provenance"); +var splitRoles = require("./session-split-group-roles"); function attachPairFactory(ctx) { var sm = ctx.sm; @@ -230,6 +231,69 @@ function attachPairFactory(ctx) { return created; } + function createUnpairedWorkerForDriver(driver, args) { + var runtime = preflightWorkerForDriver(driver, args); + var worker = sm.createSessionRaw({ + ownerId: driver.ownerId || null, + vendor: runtime.vendor, + model: runtime.model, + effort: runtime.effort, + }); + try { + worker.title = "Split Worker · " + ((yoke.getVendorInfo(runtime.vendor) || {}).displayName || runtime.vendor); + if (runtime.explicitEffort) worker.loopSettings = { effort: runtime.effort }; + sessionProvenance.markWorker(driver, worker, sm.sessions); + if (typeof sm.saveSessionFile === "function") { + if (sm.saveSessionFile(driver) === false || sm.saveSessionFile(worker) === false) { + throw new Error("Split Worker session persistence returned false"); + } + } + return worker; + } catch (error) { + removeCreated(worker); + throw error; + } + } + + function exactMutation(driver, worker, args, method) { + var group = store.groupForMember(driver.localId); + if (!group) throw new Error("split group not found"); + var role = splitRoles.normalizePair(group.pair, group.members); + if (!role.ok || role.kind === "adhoc") throw new Error("split group roles changed"); + var created = createUnpairedWorkerForDriver(driver, args); + var request = { + groupId: group.id, + driverId: driver.localId, + driverOriginId: driver.sessionOriginId || null, + expectedGroup: args.expectedGroup || group, + expectedWorkerIds: Array.isArray(args.expectedWorkerIds) ? args.expectedWorkerIds.slice() : role.workerIds.slice(), + newWorkerId: created.localId, + newWorkerOriginId: created.sessionOriginId || null, + }; + if (worker) { + request.workerId = worker.localId; + request.workerOriginId = worker.sessionOriginId || null; + request.generation = worker._pairGeneration || + worker.sessionProvenance && worker.sessionProvenance.generation || null; + } + var result = store[method](driver.ownerId || null, request, ctx.multiWorkerFeature); + if (!result.ok) { + removeCreated(created); + throw new Error(result.error); + } + sm.broadcastSessionList(); + sm.sendToSession(driver, { type: "pair_session_created", ok: true, group: result.group }); + return { driver: driver, worker: created, group: result.group }; + } + + function addWorkerForDriver(driver, args) { + return exactMutation(driver, null, args, "addWorker"); + } + + function replaceWorkerForDriver(driver, worker, args) { + return exactMutation(driver, worker, args, "replaceWorker"); + } + function createOwnedPair(ownerId, args) { if (ctx.isMate) throw new Error("Pair sessions are only available in projects"); var request = args || {}; @@ -276,6 +340,8 @@ function attachPairFactory(ctx) { createPair: createPair, createPairRecord: createPairRecord, createWorkerForDriver: createWorkerForDriver, + addWorkerForDriver: addWorkerForDriver, + replaceWorkerForDriver: replaceWorkerForDriver, createOwnedPair: createOwnedPair, removeOwnedPair: removeOwnedPair, preflightRuntime: preflightRuntime, diff --git a/lib/session-pair-mcp-server.js b/lib/session-pair-mcp-server.js index a6c848e6..671e21d6 100644 --- a/lib/session-pair-mcp-server.js +++ b/lib/session-pair-mcp-server.js @@ -23,6 +23,7 @@ function getToolDefs(handlers, options) { timeoutSeconds: { type: "number", description: "Maximum wait in seconds, from 1 to 900. Defaults to 300." }, operationId: { type: "string", description: "Optional stable id for this delegation. Reusing it within the same human turn returns the original operation instead of sending twice." }, taskId: { type: "string", description: "Optional stable task id for generation and completion correlation." }, + workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }, }, ["message"]), handler: function (args) { return handlers.send(args || {}); }, }, @@ -31,19 +32,20 @@ function getToolDefs(handlers, options) { description: "Read the Split Worker's current status and recent conversation turns, plus the same bounded capacity report partner_status returns. Use only for a requested status, concrete failure or stall diagnosis, or capacity/queue decision; do not poll routinely. Completed results are pushed automatically.", inputSchema: buildShape({ lastTurns: { type: "number", description: "Number of recent user-message-delimited turns, from 0 to 5. Use 0 for status only. Defaults to 1." }, + workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }, }), handler: function (args) { return handlers.read(args || {}); }, }, { name: "interrupt_partner", description: "Interrupt the Split Worker's current task. Use this when the task is going in the wrong direction, needs to be reprioritized, or must stop before the next instruction. The Split Worker returns any partial result to you for review.", - inputSchema: buildShape({ reason: { type: "string", description: "Why the Driver is interrupting this task." } }), + inputSchema: buildShape({ reason: { type: "string", description: "Why the Driver is interrupting this task." }, workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." } }), handler: function (args) { return handlers.interrupt(args || {}); }, }, { name: "close_partner", description: "Close the visible Split Worker pane and dissolve the Driver/Split Worker pair while preserving both session histories. If the Split Worker is still running, Clay interrupts it before closing the pair.", - inputSchema: buildShape({}), + inputSchema: buildShape({ workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." } }), handler: function (args) { return handlers.close(args || {}); }, }, ]; @@ -55,13 +57,14 @@ function getToolDefs(handlers, options) { inputSchema: buildShape({ message: { type: "string", description: "Bounded correction or context for the current active Worker turn." }, requestId: { type: "string", description: "Optional stable request id. Reuse it to deduplicate a retried delivery within this human turn." }, + workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }, }, ["message"]), handler: function (args) { return handlers.message(args || {}); }, }, { name: "partner_status", description: "Bounded capacity and continuity report for your Split Worker, for deciding reuse against replacement: context tokens used and the ratio of its window, current task and activity, vendor/model/effort, history size and idle time, whether it is safe to replace right now, and your own recorded results for earlier Worker generations. Returns no transcript. Prefer this over reading turns when you only need to decide.", - inputSchema: buildShape({}), + inputSchema: buildShape({ workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." } }), handler: function (args) { return handlers.status(args || {}); }, }, { @@ -80,6 +83,7 @@ function getToolDefs(handlers, options) { }, operationId: { type: "string", description: "Optional stable id for this replacement. Reusing it within the same human turn returns the original operation instead of replacing twice." }, supersedeProposalId: { type: "string", description: "Exact pending replacement proposal id to supersede with this new configuration." }, + workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }, }, ["message", "recommendationRationale"]), handler: function (args) { return handlers.replace(args || {}); }, }, @@ -90,6 +94,7 @@ function getToolDefs(handlers, options) { outcome: { enum: EVALUATION_OUTCOMES, description: 'One of "succeeded", "partial", "failed", "abandoned".' }, note: { type: "string", description: "Optional short reason, up to 400 characters." }, generation: { type: "number", description: "Optional generation number. Defaults to the current Worker." }, + workerId: { type: "number", description: "Optional exact Worker session id; required when multiple Workers are configured." }, }, ["outcome"]), handler: function (args) { return handlers.evaluate(args || {}); }, }, diff --git a/lib/session-pair-prompts.js b/lib/session-pair-prompts.js index d872d34f..e59be429 100644 --- a/lib/session-pair-prompts.js +++ b/lib/session-pair-prompts.js @@ -4,32 +4,42 @@ // the size limit and so the guidance can be reviewed as prose. var WORKER_RUNTIME_SELECTION = [ - "Use a high-low model mix by default: keep decomposition, judgment, review, and independent verification", - "with the Driver. Give the Split Worker concrete, bounded execution tasks with clear acceptance criteria.", - "For routine bounded implementation, start with a lightweight Worker at low effort. When the installed", - "catalog offers it, codex / gpt-5.6-luna / low is the starting recommendation; otherwise choose a comparable", - "lightweight option from the offered catalog. Do not routinely recommend gpt-5.6-sol at high effort.", - "Do not mirror the Driver's model or default to the", - "strongest available model merely because it is available. Explicitly fill the vendor, model, and thinking", - "effort recommendation fields for both creation and replacement; never leave model selection to card defaults.", - "Select model capability and thinking effort separately. Start with the lowest effort adequate for the task,", - "not high effort by habit. Choose from the installed, offered catalog; do not invent model availability or prices.", - "Explain why the lighter choice can handle the task. Escalate only for a concrete requirement beyond its", - "capabilities or observed failures it cannot adequately resolve, and state that reason. Small corrections", - "found during Driver review are not by themselves a reason to escalate. Send those corrections back to the", - "Worker and verify the final result. Optimize for time to a completed, verified outcome, not first-draft speed.", - "Use recorded Worker outcomes when available,", - "and reconsider a lighter model for the next bounded task after an escalation. Honor the user's explicit", - "runtime choice; this is a recommendation policy, not a restriction on which model the user may select.", + "Use a role-based mix by default: keep decomposition, judgment, review, and independent verification with", + "the Driver, and give the Split Worker concrete, bounded execution tasks with clear acceptance criteria.", + "Recommend the lowest-capability model and thinking effort that are adequate for the specific task, based on", + "the requirements, the server-provided capability evidence, the offered runtime catalog, and recorded local", + "Worker outcomes. Choose vendor, model, and effort separately. Do not mirror the Driver, prefer a vendor by", + "Explicitly fill the vendor, model, and thinking effort recommendation fields for both creation and replacement;", + "never rely on card defaults or leave model selection implicit.", + "brand, or default to the strongest available option. Do not invent availability, prices, rankings, or missing", + "metadata. If the catalog or capability evidence is incomplete, say what is unknown and choose only from what", + "is actually offered. Honor an explicit user runtime choice; this is guidance, not a restriction.", + "Escalate only for a concrete task requirement or an observed failure the current runtime cannot resolve, and", + "state the evidence. A small review correction is not by itself an escalation reason; send one consolidated", + "bounded correction to the Worker and verify the result. Repeated same-class failures require reassessing the", + "task contract, model, and effort with evidence, not endless cheap-model loops or escalation of every correction.", + "Aim for a role-based workload direction near 30 percent Driver and 70 percent Worker effort when useful, but", + "never promise that ratio or treat it as a quota, token, or cost guarantee, and never pad work to reach it.", ].join(" ").replace(/ {2,}/g, " "); var DRIVER_DELEGATION = [ - "Your management objective: protect your own context, keep the Split Worker compact, and put execution where", - "it runs best. Delegate implementation-heavy work rather than doing it here. Treat work spanning multiple modules,", - "crossing client/server/data boundaries, requiring a migration, or containing independent investigation and", - "execution as implementation-heavy. Your own capability is not a reason to retain that execution: preserve", - "the Driver for decomposition, runtime selection, judgment, review, and integration unless the user explicitly", - "asks you to work directly or the work must remain sequential in this session.", + "Your management objective: protect your own context, keep the Split Worker compact, and put substantive execution where it runs best.", + "Delegate implementation-heavy work rather than doing it here. Treat work spanning multiple modules, crossing", + "client/server/data boundaries, requiring independent investigation, testing, or self-correction as implementation-heavy.", + "Your own capability is not a reason to retain that execution: preserve the Driver for decomposition,", + "runtime selection, judgment, review, and integration unless the user explicitly asks you to work directly or the", + "work must remain sequential in this session.", + "Give each Worker one bounded contract: goal and scope, constraints and non-goals, acceptance commands and the", + "expected behaviors, and the required handoff. Delegate substantive investigation, implementation, testing, and", + "self-correction. The Worker must finish its allowed scope, inspect its diff against the criteria, run meaningful", + "production-path tests, and correct failures before reporting. It must not hand back an unchecked draft. Require", + "a bounded self-correction attempt; when progress is not possible, require a precise blocker and evidence.", + "Review once per coherent completed milestone with risk-focused source review and independent verification. Do not", + "rubber-stamp or sacrifice quality for an efficiency direction. Consolidate review feedback into one correction", + "task rather than sending overlapping live and queued work.", + "When discussing measured efficiency, distinguish requests, input tokens, output tokens, cached tokens, and cost.", + "Unknown usage is not zero; the 30/70 direction is role-based and aspirational, not a guaranteed token or cost", + "ratio or quota.", ].join(" ").replace(/ {2,}/g, " "); var DRIVER_CORE = [ @@ -91,7 +101,7 @@ var DRIVER_CORE = [ var DRIVER = DRIVER_CORE + " " + DRIVER_DELEGATION; function worker(taskId) { - return "You are the configured Split Worker. For task " + (taskId || "unknown") + ", call report_partner_outcome before finishing when the tool is available. Report only what you observed: changed files, verification commands/results, unverified work, running processes, and next action. Never claim Driver verification."; + return "You are the configured Split Worker. For task " + (taskId || "unknown") + ", finish only the allowed bounded scope. Inspect your diff against the stated acceptance criteria, run meaningful production-path tests, and correct your own failures within a bounded self-correction attempt before reporting. Do not hand back an unchecked draft or hide a failure; if progress is blocked, report the precise blocker and evidence. Call report_partner_outcome before finishing when the tool is available. Report only what you observed: changed files and behavior, commands with actual exit results, known limits or unverified work, running processes, and next action. Never claim Driver verification."; } var UNPAIRED = [ diff --git a/lib/session-pair-target.js b/lib/session-pair-target.js new file mode 100644 index 00000000..734307f0 --- /dev/null +++ b/lib/session-pair-target.js @@ -0,0 +1,42 @@ +var splitRoles = require("./session-split-group-roles"); +var multiWorkerFeature = require("./multi-worker-feature"); + +function resolveGroupTarget(sm, group, caller, args, options) { + if (!caller || sm.sessions.get(caller.localId) !== caller) throw new Error("this session is no longer live; the partner tools are bound to an exact session"); + var requestedWorkerId = args && args.workerId; + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok) throw new Error(normalized.error); + var workerId; + if (normalized.kind === "adhoc") { + workerId = group.members[0] === caller.localId ? group.members[1] : group.members[0]; + if (requestedWorkerId != null && requestedWorkerId !== workerId) throw new Error("workerId must identify the exact split partner"); + } else { + if (normalized.driverId !== caller.localId) throw new Error("only the configured Driver can direct this pair"); + var target = splitRoles.resolveWorkerTarget(normalized, requestedWorkerId); + if (!target.ok) throw new Error(target.error); + workerId = target.workerId; + if (normalized.kind === "versioned" && (!options || options.readOnly !== true) && + (!options || !multiWorkerFeature.isEnabled(options.multiWorkerFeature))) { + throw new Error("multi-Worker mutations remain gated until lifecycle, result, and Stop routing are migrated"); + } + } + var worker = sm.sessions.get(workerId); + if (!worker || sm.sessions.get(worker.localId) !== worker) throw new Error("split partner session was not found"); + if ((caller.ownerId || null) !== (worker.ownerId || null)) throw new Error("split partner access denied"); + if (group.members.indexOf(caller.localId) === -1 || group.members.indexOf(worker.localId) === -1) { + throw new Error("the exact caller and Worker must both belong to this split group"); + } + return { group: group, partner: worker, role: normalized, workerId: workerId }; +} + +function rolesFor(sm, store, session) { + if (!session || sm.sessions.get(session.localId) !== session) return null; + var group = store.groupForMember(session.localId); + var roles = group && group.pair ? splitRoles.resolveLiveRoles(sm, group) : null; + if (!roles) return null; + roles.worker = roles.workers.length === 1 ? roles.workers[0] : + roles.workers.find(function (worker) { return worker === session; }) || null; + return roles; +} + +module.exports = { resolveGroupTarget: resolveGroupTarget, rolesFor: rolesFor }; diff --git a/lib/session-pair-turn-control.js b/lib/session-pair-turn-control.js index 8a7d7ea3..866dd08d 100644 --- a/lib/session-pair-turn-control.js +++ b/lib/session-pair-turn-control.js @@ -10,6 +10,7 @@ var MAX_CREATIONS_PER_TURN = 2; var MAX_REPLACEMENTS_PER_TURN = 1; var MAX_FAILED_REPLACEMENT_ATTEMPTS_PER_TURN = 2; var MAX_OPERATION_ID_CHARS = 120; +var splitRoles = require("./session-split-group-roles"); function attachPairTurnControl(ctx) { var sm = ctx.sm; @@ -22,6 +23,7 @@ function attachPairTurnControl(ctx) { humanStopped: false, stoppedAt: null, stoppedWorkerId: null, + stoppedWorkerIds: [], creations: 0, replacements: 0, failedReplacementAttempts: 0, @@ -42,11 +44,11 @@ function attachPairTurnControl(ctx) { if (!liveSession(session)) return null; var group = store.groupForMember(session.localId); if (!group || !group.pair) return null; - var driver = sm.sessions.get(group.pair.driverId); - var worker = sm.sessions.get(group.pair.workerId); - if (!liveSession(driver) || !liveSession(worker)) return null; - if ((driver.ownerId || null) !== (worker.ownerId || null)) return null; - return { group: group, driver: driver, worker: worker }; + var roles = splitRoles.resolveLiveRoles(sm, group); + if (!roles) return null; + roles.worker = roles.workers.length === 1 ? roles.workers[0] : + roles.workers.find(function (worker) { return worker === session; }) || null; + return roles; } function beginHumanTurn(session) { @@ -59,6 +61,7 @@ function attachPairTurnControl(ctx) { state.humanStopped = false; state.stoppedAt = null; state.stoppedWorkerId = null; + state.stoppedWorkerIds = []; state.creations = 0; state.replacements = 0; state.failedReplacementAttempts = 0; @@ -72,7 +75,8 @@ function attachPairTurnControl(ctx) { var state = stateFor(roles.driver); state.humanStopped = true; state.stoppedAt = Date.now(); - state.stoppedWorkerId = roles.worker.localId; + state.stoppedWorkerId = roles.worker ? roles.worker.localId : null; + state.stoppedWorkerIds = roles.workers.map(function (worker) { return worker.localId; }); return roles; } @@ -147,6 +151,7 @@ function attachPairTurnControl(ctx) { humanStopped: state.humanStopped, stoppedAt: state.stoppedAt, stoppedWorkerId: state.stoppedWorkerId, + stoppedWorkerIds: state.stoppedWorkerIds.slice(), turnSerial: state.serial, creationsThisTurn: state.creations, replacementsThisTurn: state.replacements, diff --git a/lib/session-split-group-anchors.js b/lib/session-split-group-anchors.js index 3e800e82..2fdaa8c9 100644 --- a/lib/session-split-group-anchors.js +++ b/lib/session-split-group-anchors.js @@ -2,6 +2,8 @@ function value(input) { return typeof input === "string" && input ? input : null; } +var splitRoles = require("./session-split-group-roles"); + function pairFor(session, previousCli, previousOrigin) { return { cli: value(session && session.cliSessionId) || value(previousCli), @@ -9,16 +11,87 @@ function pairFor(session, previousCli, previousOrigin) { }; } +function versionedAnchor(session) { + var cli = value(session && session.cliSessionId); + var origin = value(session && session.sessionOriginId); + if (!cli && !origin) return null; + return { cli: cli, origin: origin }; +} + +function versionedPairAnchors(group, sessions) { + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok || normalized.kind !== "versioned") return null; + var driver = versionedAnchor(sessions.get(group.pair.driverId)); + if (!driver) return null; + var workers = []; + for (var i = 0; i < normalized.workerIds.length; i++) { + var worker = versionedAnchor(sessions.get(normalized.workerIds[i])); + if (!worker) return null; + workers.push(worker); + } + return { version: 2, driver: driver, workers: workers }; +} + +function versionedMemberAnchors(group, sessions) { + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok || normalized.kind !== "versioned") return null; + var members = []; + for (var i = 0; i < group.members.length; i++) { + var anchor = versionedAnchor(sessions.get(group.members[i])); + if (!anchor) return null; + members.push(anchor); + } + return { version: 2, members: members }; +} + +function resolveVersionedPair(sessions, anchors) { + if (!anchors || anchors.version !== 2 || !anchors.driver || !Array.isArray(anchors.workers) || + (anchors.workers.length !== 1 && anchors.workers.length !== 2)) return null; + var driver = findSession(sessions, anchors.driver.cli, anchors.driver.origin); + if (!driver) return null; + var workers = []; + for (var i = 0; i < anchors.workers.length; i++) { + var workerAnchor = anchors.workers[i]; + if (!workerAnchor || (!value(workerAnchor.cli) && !value(workerAnchor.origin))) return null; + var worker = workerAnchor && findSession(sessions, workerAnchor.cli, workerAnchor.origin); + if (!worker || workers.indexOf(worker) !== -1 || worker === driver) return null; + workers.push(worker); + } + return { driver: driver, workers: workers }; +} + +function resolveVersionedMembers(sessions, anchors) { + if (!anchors || anchors.version !== 2 || !Array.isArray(anchors.members) || + (anchors.members.length !== 2 && anchors.members.length !== 3)) return null; + var resolved = []; + for (var i = 0; i < anchors.members.length; i++) { + var anchor = anchors.members[i]; + if (!anchor || (!value(anchor.cli) && !value(anchor.origin))) return null; + var session = anchor && findSession(sessions, anchor.cli, anchor.origin); + if (!session || resolved.indexOf(session) !== -1) return null; + resolved.push(session); + } + return resolved; +} + function memberAnchors(group, sessions) { - var previousCli = Array.isArray(group.memberCliIds) ? group.memberCliIds : [null, null]; - var previousOrigin = Array.isArray(group.memberOriginIds) ? group.memberOriginIds : [null, null]; - var left = pairFor(sessions.get(group.members[0]), previousCli[0], previousOrigin[0]); - var right = pairFor(sessions.get(group.members[1]), previousCli[1], previousOrigin[1]); - return { cli: [left.cli, right.cli], origin: [left.origin, right.origin] }; + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok) return null; + if (normalized.kind === "versioned") return versionedMemberAnchors(group, sessions); + var previousCli = Array.isArray(group.memberCliIds) ? group.memberCliIds : []; + var previousOrigin = Array.isArray(group.memberOriginIds) ? group.memberOriginIds : []; + var cli = [], origin = []; + for (var i = 0; i < group.members.length; i++) { + var anchor = pairFor(sessions.get(group.members[i]), previousCli[i], previousOrigin[i]); + cli.push(anchor.cli); origin.push(anchor.origin); + } + return { cli: cli, origin: origin }; } function pairAnchors(group, sessions) { - if (!group.pair) return null; + var normalized = splitRoles.normalizePair(group && group.pair, group && group.members); + if (!normalized.ok || normalized.kind === "adhoc") return null; + if (normalized.kind === "versioned") return versionedPairAnchors(group, sessions); var previousCli = Array.isArray(group.pairCliIds) ? group.pairCliIds : [null, null]; var previousOrigin = Array.isArray(group.pairOriginIds) ? group.pairOriginIds : [null, null]; var driver = pairFor(sessions.get(group.pair.driverId), previousCli[0], previousOrigin[0]); @@ -41,4 +114,13 @@ function findSession(sessions, cliId, originId) { return ambiguous ? null : found; } -module.exports = { findSession: findSession, memberAnchors: memberAnchors, pairAnchors: pairAnchors }; +module.exports = { + findSession: findSession, + memberAnchors: memberAnchors, + pairAnchors: pairAnchors, + resolveVersionedMembers: resolveVersionedMembers, + resolveVersionedPair: resolveVersionedPair, + versionedAnchor: versionedAnchor, + versionedMemberAnchors: versionedMemberAnchors, + versionedPairAnchors: versionedPairAnchors, +}; diff --git a/lib/session-split-group-persistence.js b/lib/session-split-group-persistence.js new file mode 100644 index 00000000..8cd8a1d5 --- /dev/null +++ b/lib/session-split-group-persistence.js @@ -0,0 +1,136 @@ +var fs = require("fs"); +var path = require("path"); +var anchors = require("./session-split-group-anchors"); +var roles = require("./session-split-group-roles"); + +var ANCHOR_KEYS = ["memberCliIds", "memberOriginIds", "pairCliIds", "pairOriginIds", "memberAnchors", "pairAnchors"]; + +function attachPersistence(opts) { + var sessions = opts.sessions; + var groupsFile = path.join(opts.sessionsDir, "split-groups.json"); + var preserved = []; + + function anchorPair(group) { + if (!group.pair) return; + var pair = anchors.pairAnchors(group, sessions); + if (!pair) return; + if (pair.version === 2) { + group.pairAnchors = pair; + delete group.pairCliIds; delete group.pairOriginIds; + } else { + group.pairCliIds = pair.cli; group.pairOriginIds = pair.origin; + } + } + + function anchoredClone(group) { + var clone = Object.assign({}, group, { members: group.members.slice(), + pair: group.pair ? Object.assign({}, group.pair) : undefined }); + if (clone.pair && Array.isArray(clone.pair.workerIds)) clone.pair.workerIds = clone.pair.workerIds.slice(); + var member = anchors.memberAnchors(clone, sessions); + if (!member) throw new Error("Split group does not have durable session anchors"); + if (member.version === 2) { + clone.memberAnchors = member; + delete clone.memberCliIds; delete clone.memberOriginIds; + } else { + clone.memberCliIds = member.cli; clone.memberOriginIds = member.origin; + } + anchorPair(clone); + return clone; + } + + function save(groups) { + var saved = groups.map(anchoredClone); + var serialized = JSON.stringify(saved.concat(preserved), null, 2) + "\n"; + if (typeof opts.persistGroups === "function") { + if (opts.persistGroups(groupsFile, serialized) === false) throw new Error("Split group persistence returned false"); + } else { + var tmp = groupsFile + ".tmp." + process.pid; + fs.mkdirSync(opts.sessionsDir, { recursive: true }); + fs.writeFileSync(tmp, serialized); + if (process.platform !== "win32") try { fs.chmodSync(tmp, 0o600); } catch (e) {} + fs.renameSync(tmp, groupsFile); + } + for (var i = 0; i < groups.length; i++) { + (function (live, stored) { + ANCHOR_KEYS.forEach(function (key) { + if (Object.prototype.hasOwnProperty.call(stored, key)) live[key] = stored[key]; + else delete live[key]; + }); + })(groups[i], saved[i]); + } + } + + function resolveVersioned(group, normalized) { + if (typeof group.id !== "string" || !group.id) return false; + var members = anchors.resolveVersionedMembers(sessions, group.memberAnchors); + var pair = anchors.resolveVersionedPair(sessions, group.pairAnchors); + if (!members || !pair) return false; + var memberIds = members.map(function (session) { return session.localId; }); + var workerIds = pair.workers.map(function (session) { return session.localId; }); + if (memberIds.length !== group.members.length || workerIds.length !== normalized.workerIds.length) return false; + var nextPair = { version: 2, driverId: pair.driver.localId, workerIds: workerIds }; + var nextRoles = roles.normalizePair(nextPair, memberIds); + if (!nextRoles.ok || nextRoles.kind !== "versioned") return false; + var changed = group.members.join(",") !== memberIds.join(",") || + group.pair.driverId !== pair.driver.localId || group.pair.workerIds.join(",") !== workerIds.join(","); + group.members = memberIds; + group.pair = nextPair; + return { changed: changed }; + } + + function resolveLegacy(group) { + var changed = false; + if (Array.isArray(group.memberCliIds) || Array.isArray(group.memberOriginIds)) { + var cli = Array.isArray(group.memberCliIds) ? group.memberCliIds : [null, null]; + var origin = Array.isArray(group.memberOriginIds) ? group.memberOriginIds : [null, null]; + var left = anchors.findSession(sessions, cli[0], origin[0]); + var right = anchors.findSession(sessions, cli[1], origin[1]); + if (!left || !right || left === right) return false; + if (group.members[0] !== left.localId || group.members[1] !== right.localId) { + group.members = [left.localId, right.localId]; changed = true; + } + } + if (group.pair && (Array.isArray(group.pairCliIds) || Array.isArray(group.pairOriginIds))) { + var pairCli = Array.isArray(group.pairCliIds) ? group.pairCliIds : [null, null]; + var pairOrigin = Array.isArray(group.pairOriginIds) ? group.pairOriginIds : [null, null]; + var driver = anchors.findSession(sessions, pairCli[0], pairOrigin[0]); + var worker = anchors.findSession(sessions, pairCli[1], pairOrigin[1]); + if (driver && worker && (group.pair.driverId !== driver.localId || group.pair.workerId !== worker.localId)) { + group.pair.driverId = driver.localId; group.pair.workerId = worker.localId; changed = true; + } + } + return roles.validateStoredGroup(group, sessions) ? { changed: changed } : false; + } + + function load() { + var loaded = [], rewrite = false, claimed = {}; + preserved = []; + try { loaded = JSON.parse(fs.readFileSync(groupsFile, "utf8")); if (!Array.isArray(loaded)) loaded = []; } + catch (error) { if (error.code !== "ENOENT") rewrite = true; } + var groups = loaded.filter(function (group) { + var normalized = group && roles.normalizePair(group.pair, group.members); + if (!normalized || !normalized.ok) { + if (group && roles.isUnsupportedPair(group.pair)) preserved.push(group); + return false; + } + var versioned = !!(normalized && normalized.ok && normalized.kind === "versioned"); + if (versioned && !opts.versionedEnabled) { preserved.push(group); return false; } + var resolved = versioned ? resolveVersioned(group, normalized) : resolveLegacy(group); + if (!resolved) { + if (versioned || group && roles.isUnsupportedPair(group.pair)) preserved.push(group); + return false; + } + if (resolved.changed) rewrite = true; + for (var i = 0; i < group.members.length; i++) if (claimed[group.members[i]]) return false; + for (var j = 0; j < group.members.length; j++) claimed[group.members[j]] = true; + return true; + }); + if (groups.length + preserved.length !== loaded.length) rewrite = true; + if (rewrite) save(groups); + return groups; + } + + return { load: load, save: save }; +} + +module.exports = { attachPersistence: attachPersistence }; diff --git a/lib/session-split-group-roles.js b/lib/session-split-group-roles.js new file mode 100644 index 00000000..9d78c7e1 --- /dev/null +++ b/lib/session-split-group-roles.js @@ -0,0 +1,164 @@ +// Shared role and target identity rules for Split Groups. +// +// Version 1 is the persisted two-session pair shape. Version 2 describes one +// Driver and up to two Workers, but callers must explicitly opt into it until +// all production consumers can address more than one Worker. +// Remaining direct pair.workerId reads are deliberate compatibility branches: +// session-split-groups owns legacy-only create and Driver-transfer transactions, +// while session-split-group-anchors and persistence resolve legacy records. +// Production activation must still migrate the client hierarchy, pane lock, and +// split-group helper projections before the server gate is enabled. + +var MAX_WORKERS = 2; + +function isInteger(value) { + return Number.isInteger(value); +} + +function containsOnce(values, value) { + var count = 0; + for (var i = 0; i < values.length; i++) if (values[i] === value) count++; + return count === 1; +} + +function validateMembers(members) { + if (!Array.isArray(members) || (members.length !== 2 && members.length !== 3)) { + return { ok: false, error: "A split group requires exactly two sessions until multi-Worker consumers are enabled" }; + } + for (var i = 0; i < members.length; i++) { + if (!isInteger(members[i])) return { ok: false, error: "Split group members must be distinct integer session ids" }; + if (!containsOnce(members, members[i])) return { ok: false, error: "Split group members must be distinct integer session ids" }; + } + return { ok: true }; +} + +function validateRoleIds(driverId, workerIds, members) { + if (!isInteger(driverId)) return { ok: false, error: "The Driver role must reference an integer session id" }; + if (!Array.isArray(workerIds) || workerIds.length < 1 || workerIds.length > MAX_WORKERS) { + return { ok: false, error: "A configured role requires one to two Workers" }; + } + if (!containsOnce(members, driverId)) return { ok: false, error: "The Driver role must reference exactly one split group member" }; + var seen = [driverId]; + for (var i = 0; i < workerIds.length; i++) { + if (!isInteger(workerIds[i]) || seen.indexOf(workerIds[i]) !== -1 || !containsOnce(members, workerIds[i])) { + return { ok: false, error: "Worker roles must reference distinct split group members" }; + } + seen.push(workerIds[i]); + } + if (seen.length !== members.length) return { ok: false, error: "Every split group member must have exactly one role" }; + return { ok: true }; +} + +function normalizePair(pair, members) { + var memberCheck = validateMembers(members); + if (!memberCheck.ok) return memberCheck; + if (pair == null) { + if (members.length !== 2) return { ok: false, error: "An ad-hoc split group requires exactly two sessions" }; + return { ok: true, kind: "adhoc", version: null, driverId: null, workerIds: [] }; + } + if (!pair || typeof pair !== "object" || Array.isArray(pair)) return { ok: false, error: "Split group roles must be an object" }; + + var hasVersion = Object.prototype.hasOwnProperty.call(pair, "version"); + var hasLegacyWorker = Object.prototype.hasOwnProperty.call(pair, "workerId"); + var hasWorkerIds = Object.prototype.hasOwnProperty.call(pair, "workerIds"); + if (hasVersion && pair.version !== 2) return { ok: false, error: "Unknown split group role version" }; + if (hasVersion && hasLegacyWorker) return { ok: false, error: "Versioned roles cannot also define legacy workerId" }; + if (hasVersion && !hasWorkerIds) return { ok: false, error: "Version 2 roles require workerIds" }; + if (!hasVersion && hasWorkerIds) return { ok: false, error: "workerIds requires version 2 roles" }; + if (!hasVersion && !hasLegacyWorker) return { ok: false, error: "Legacy roles require workerId" }; + + var workerIds = hasVersion ? pair.workerIds : [pair.workerId]; + var checked = validateRoleIds(pair.driverId, workerIds, members); + if (!checked.ok) return checked; + return { + ok: true, + kind: hasVersion ? "versioned" : "legacy", + version: hasVersion ? 2 : null, + driverId: pair.driverId, + workerIds: workerIds.slice(), + }; +} + +function validateStorePair(pair, members) { + var normalized = normalizePair(pair, members); + if (!normalized.ok) return normalized; + if (normalized.kind === "versioned") { + return { ok: false, error: "Version 2 split group roles are reserved until multi-Worker consumers are migrated" }; + } + if (normalized.kind === "adhoc" && members.length !== 2) { + return { ok: false, error: "An ad-hoc split group requires exactly two sessions" }; + } + return normalized; +} + +function isUnsupportedPair(pair) { + return !!(pair && typeof pair === "object" && Object.prototype.hasOwnProperty.call(pair, "version")); +} + +function validateStoredGroup(group, sessions) { + if (!group || typeof group.id !== "string" || !Array.isArray(group.members) || group.members.length !== 2) return false; + if (!sessions.has(group.members[0]) || !sessions.has(group.members[1])) return false; + return validateStorePair(group.pair, group.members).ok; +} + +function resolveWorkerTarget(role, requestedWorkerId) { + if (!role || !role.ok || role.kind === "adhoc") return { ok: false, error: "The split group has no configured Workers" }; + if (requestedWorkerId != null) { + if (!isInteger(requestedWorkerId) || role.workerIds.indexOf(requestedWorkerId) === -1) { + return { ok: false, error: "workerId must identify an exact configured Worker" }; + } + return { ok: true, workerId: requestedWorkerId }; + } + if (role.workerIds.length !== 1) return { ok: false, error: "workerId is required when more than one Worker is configured" }; + return { ok: true, workerId: role.workerIds[0] }; +} + +function resolveLiveRoles(sm, group) { + var normalized = normalizePair(group && group.pair, group && group.members); + if (!normalized.ok || normalized.kind === "adhoc") return null; + var driver = sm.sessions.get(normalized.driverId); + if (!driver || sm.sessions.get(driver.localId) !== driver || driver.destroying) return null; + var workers = []; + for (var i = 0; i < normalized.workerIds.length; i++) { + var worker = sm.sessions.get(normalized.workerIds[i]); + if (!worker || sm.sessions.get(worker.localId) !== worker || worker.destroying) return null; + if ((worker.ownerId || null) !== (driver.ownerId || null)) return null; + workers.push(worker); + } + return { group: group, role: normalized, driver: driver, workers: workers }; +} + +function matchesLiveTarget(sm, group, caller, worker) { + if (!group || !caller || !worker) return false; + if (sm.sessions.get(caller.localId) !== caller || sm.sessions.get(worker.localId) !== worker) return false; + if ((caller.ownerId || null) !== (worker.ownerId || null)) return false; + var normalized = normalizePair(group.pair, group.members); + if (!normalized.ok) return false; + if (normalized.kind === "adhoc") { + return group.members.indexOf(caller.localId) !== -1 && group.members.indexOf(worker.localId) !== -1 && caller !== worker; + } + var roles = resolveLiveRoles(sm, group); + return !!(roles && roles.driver === caller && roles.workers.indexOf(worker) !== -1); +} + +function taskGeneration(worker) { + return worker && (worker._pairGeneration || worker.sessionProvenance && worker.sessionProvenance.generation) || null; +} + +function tokenMatchesWorker(worker, token) { + return !!(worker && token && token.workerSessionId === worker.localId && token.generation === taskGeneration(worker)); +} + +module.exports = { + MAX_WORKERS: MAX_WORKERS, + normalizePair: normalizePair, + matchesLiveTarget: matchesLiveTarget, + resolveLiveRoles: resolveLiveRoles, + resolveWorkerTarget: resolveWorkerTarget, + tokenMatchesWorker: tokenMatchesWorker, + validateMembers: validateMembers, + validateRoleIds: validateRoleIds, + validateStoredGroup: validateStoredGroup, + validateStorePair: validateStorePair, + isUnsupportedPair: isUnsupportedPair, +}; diff --git a/lib/session-split-group-v2-store.js b/lib/session-split-group-v2-store.js new file mode 100644 index 00000000..1f71203c --- /dev/null +++ b/lib/session-split-group-v2-store.js @@ -0,0 +1,146 @@ +var multiWorkerFeature = require("./multi-worker-feature"); +var splitRoles = require("./session-split-group-roles"); + +function sameOwner(session, ownerId) { + return !!session && (session.ownerId || null) === (ownerId || null); +} + +function generationOf(session) { + return session && (session._pairGeneration || + session.sessionProvenance && session.sessionProvenance.generation) || null; +} + +function clone(value) { return JSON.parse(JSON.stringify(value)); } + +function replace(target, source) { + var keys = Object.keys(target); + for (var i = 0; i < keys.length; i++) delete target[keys[i]]; + Object.assign(target, clone(source)); +} + +function attachVersionedMutations(ctx) { + function authorized(feature) { + return ctx.enabled && multiWorkerFeature.isEnabled(feature); + } + + function rejectCapability(capability) { + if (authorized(capability)) return null; + return { ok: false, error: "Version 2 split groups require the server multi-Worker feature" }; + } + + function exactGroup(ownerId, msg, targetRequired) { + var group = ctx.groups().find(function (item) { return item.id === (msg && msg.groupId); }); + if (!group) return { error: "Split group not found" }; + if (msg.expectedGroup && msg.expectedGroup !== group) return { error: "The exact split group changed" }; + if ((group.ownerId || null) !== (ownerId || null)) return { error: "Split group owner changed" }; + var normalized = splitRoles.normalizePair(group.pair, group.members); + if (!normalized.ok || normalized.kind === "adhoc") return { error: "Split group roles changed" }; + if (normalized.driverId !== msg.driverId) return { error: "The configured Driver changed" }; + if (Array.isArray(msg.expectedWorkerIds) && + normalized.workerIds.join(",") !== msg.expectedWorkerIds.join(",")) { + return { error: "The configured Worker membership changed" }; + } + var driver = ctx.sessions.get(msg.driverId); + if (!sameOwner(driver, ownerId) || ctx.sessions.get(driver.localId) !== driver || + (msg.driverOriginId || null) !== (driver.sessionOriginId || null)) { + return { error: "The exact Driver identity changed" }; + } + var target = null; + if (targetRequired) { + if (normalized.workerIds.indexOf(msg.workerId) === -1) return { error: "The selected Worker changed" }; + target = ctx.sessions.get(msg.workerId); + if (!sameOwner(target, ownerId) || ctx.sessions.get(target.localId) !== target || + (msg.workerOriginId || null) !== (target.sessionOriginId || null) || + msg.generation !== generationOf(target)) { + return { error: "The exact Worker identity or generation changed" }; + } + } + return { group: group, normalized: normalized, driver: driver, target: target }; + } + + function newWorker(ownerId, msg) { + var worker = ctx.sessions.get(msg.newWorkerId); + if (!sameOwner(worker, ownerId) || ctx.sessions.get(worker.localId) !== worker || + (msg.newWorkerOriginId || null) !== (worker.sessionOriginId || null)) { + return { error: "The exact replacement Worker identity changed" }; + } + if (ctx.membershipGroup(worker.localId)) return { error: "The Worker already belongs to a split group" }; + return { worker: worker }; + } + + function commit(group, nextMembers, nextPair, hooks) { + var before = clone(group); + group.members = nextMembers; + group.pair = nextPair; + try { ctx.save(); } + catch (error) { + replace(group, before); + return { ok: false, error: error.message || String(error) }; + } + var committed = clone(group); + if (hooks && typeof hooks.afterPersist === "function") { + replace(group, before); + var accepted; + try { accepted = hooks.afterPersist(); } + catch (error) { accepted = { ok: false, error: error.message || String(error) }; } + if (!accepted || accepted.ok === false) { + try { ctx.save(); } + catch (rollbackError) { + return { ok: false, error: (accepted && accepted.error || "close transaction rejected") + + "; persistence rollback failed: " + (rollbackError.message || String(rollbackError)) }; + } + return { ok: false, error: accepted && accepted.error || "close transaction rejected" }; + } + replace(group, committed); + } + ctx.changed(group); + return { ok: true, group: group }; + } + + function addWorker(ownerId, msg, capability) { + var denied = rejectCapability(capability); + if (denied) return denied; + var exact = exactGroup(ownerId, msg || {}, false); + if (exact.error) return { ok: false, error: exact.error }; + if (exact.normalized.workerIds.length >= splitRoles.MAX_WORKERS) { + return { ok: false, error: "A Driver can have at most two Workers" }; + } + var added = newWorker(ownerId, msg); + if (added.error) return { ok: false, error: added.error }; + var workerIds = exact.normalized.workerIds.concat([added.worker.localId]); + return commit(exact.group, [exact.driver.localId].concat(workerIds), { + version: 2, driverId: exact.driver.localId, workerIds: workerIds, + }); + } + + function replaceWorker(ownerId, msg, capability) { + var denied = rejectCapability(capability); + if (denied) return denied; + var exact = exactGroup(ownerId, msg || {}, true); + if (exact.error) return { ok: false, error: exact.error }; + var added = newWorker(ownerId, msg); + if (added.error) return { ok: false, error: added.error }; + var workerIds = exact.normalized.workerIds.slice(); + workerIds[workerIds.indexOf(exact.target.localId)] = added.worker.localId; + var pair = exact.normalized.kind === "legacy" + ? { driverId: exact.driver.localId, workerId: added.worker.localId } + : { version: 2, driverId: exact.driver.localId, workerIds: workerIds }; + return commit(exact.group, [exact.driver.localId].concat(workerIds), pair); + } + + function removeWorker(ownerId, msg, capability, hooks) { + var denied = rejectCapability(capability); + if (denied) return denied; + var exact = exactGroup(ownerId, msg || {}, true); + if (exact.error) return { ok: false, error: exact.error }; + var workerIds = exact.normalized.workerIds.filter(function (id) { return id !== exact.target.localId; }); + if (workerIds.length === 0) return ctx.remove(exact.group, hooks); + return commit(exact.group, [exact.driver.localId].concat(workerIds), { + version: 2, driverId: exact.driver.localId, workerIds: workerIds, + }, hooks); + } + + return { addWorker: addWorker, replaceWorker: replaceWorker, removeWorker: removeWorker }; +} + +module.exports = { attachVersionedMutations: attachVersionedMutations, generationOf: generationOf }; diff --git a/lib/session-split-groups.js b/lib/session-split-groups.js index 8993f3a7..1f0567b6 100644 --- a/lib/session-split-groups.js +++ b/lib/session-split-groups.js @@ -1,7 +1,9 @@ var crypto = require("crypto"); -var fs = require("fs"); -var path = require("path"); var durableAnchors = require("./session-split-group-anchors"); +var splitRoles = require("./session-split-group-roles"); +var versionedStore = require("./session-split-group-v2-store"); +var multiWorkerFeature = require("./multi-worker-feature"); +var groupPersistence = require("./session-split-group-persistence"); function truncateTitle(title) { var value = String(title || "New Session"); return value.length > 20 ? value.slice(0, 19) + "…" : value; @@ -11,92 +13,14 @@ function autoGroupName(leftTitle, rightTitle) { } function createSplitGroupStore(opts) { var sessions = opts.sessions; - var groupsFile = path.join(opts.sessionsDir, "split-groups.json"); var usersModule = opts.usersModule; var broadcast = opts.broadcast || function () {}; var onPairChanged = opts.onPairChanged || function () {}; var groups = []; var stagedTransfers = []; - function refreshPairAnchors(group) { - if (!group.pair) return; - var anchors = durableAnchors.pairAnchors(group, sessions); - group.pairCliIds = anchors.cli; - group.pairOriginIds = anchors.origin; - } - function save() { - for (var gi = 0; gi < groups.length; gi++) { - var anchors = durableAnchors.memberAnchors(groups[gi], sessions); - groups[gi].memberCliIds = anchors.cli; - groups[gi].memberOriginIds = anchors.origin; - refreshPairAnchors(groups[gi]); - } - var serialized = JSON.stringify(groups, null, 2) + "\n"; - if (typeof opts.persistGroups === "function") { - if (opts.persistGroups(groupsFile, serialized) === false) throw new Error("Split group persistence returned false"); - return; - } - var tmp = groupsFile + ".tmp." + process.pid; - fs.mkdirSync(opts.sessionsDir, { recursive: true }); - fs.writeFileSync(tmp, serialized); - if (process.platform !== "win32") { - try { fs.chmodSync(tmp, 0o600); } catch (e) {} - } - fs.renameSync(tmp, groupsFile); - } - function validRecord(group) { - return group && typeof group.id === "string" && Array.isArray(group.members) && - group.members.length === 2 && group.members[0] !== group.members[1] && - Number.isInteger(group.members[0]) && Number.isInteger(group.members[1]) && - sessions.has(group.members[0]) && sessions.has(group.members[1]); - } - function load() { - var loaded = []; - var shouldRewrite = false; - try { - loaded = JSON.parse(fs.readFileSync(groupsFile, "utf8")); - if (!Array.isArray(loaded)) loaded = []; - } catch (e) { - if (e.code !== "ENOENT") shouldRewrite = true; - } - var claimed = {}; - groups = loaded.filter(function (group) { - // Remap members from the durable cliSessionIds; the stored localIds - // are only trusted for legacy records that predate memberCliIds. - if (group && (Array.isArray(group.memberCliIds) || Array.isArray(group.memberOriginIds))) { - var memberCliIds = Array.isArray(group.memberCliIds) ? group.memberCliIds : [null, null]; - var memberOriginIds = Array.isArray(group.memberOriginIds) ? group.memberOriginIds : [null, null]; - var left = durableAnchors.findSession(sessions, memberCliIds[0], memberOriginIds[0]); - var right = durableAnchors.findSession(sessions, memberCliIds[1], memberOriginIds[1]); - if (!left || !right || left === right) return false; - if (group.members[0] !== left.localId || group.members[1] !== right.localId) { - group.members = [left.localId, right.localId]; - shouldRewrite = true; - } - } - if (group && group.pair && (Array.isArray(group.pairCliIds) || Array.isArray(group.pairOriginIds))) { - var pairCliIds = Array.isArray(group.pairCliIds) ? group.pairCliIds : [null, null]; - var pairOriginIds = Array.isArray(group.pairOriginIds) ? group.pairOriginIds : [null, null]; - var driver = durableAnchors.findSession(sessions, pairCliIds[0], pairOriginIds[0]); - var worker = durableAnchors.findSession(sessions, pairCliIds[1], pairOriginIds[1]); - if (driver && worker) { - group.pair.driverId = driver.localId; - group.pair.workerId = worker.localId; - } - } - if (group && group.pair) { - var pairValid = Array.isArray(group.members) && group.pair.driverId !== group.pair.workerId && - group.members.indexOf(group.pair.driverId) !== -1 && - group.members.indexOf(group.pair.workerId) !== -1; - if (!pairValid) return false; - } - if (!validRecord(group) || claimed[group.members[0]] || claimed[group.members[1]]) return false; - claimed[group.members[0]] = true; - claimed[group.members[1]] = true; - return true; - }); - if (groups.length !== loaded.length) shouldRewrite = true; - if (shouldRewrite) save(); - } + var versionedEnabled = multiWorkerFeature.isEnabled(opts.multiWorkerFeature); + var persistence = groupPersistence.attachPersistence(Object.assign({}, opts, { versionedEnabled: versionedEnabled })); + function save() { return persistence.save(groups); } function isMultiUser() { return !!(usersModule && usersModule.isMultiUser && usersModule.isMultiUser()); } @@ -153,6 +77,8 @@ function createSplitGroupStore(opts) { if (!left || !right) return { ok: false, error: "Session not found" }; if (!canAccess(ws, left) || !canAccess(ws, right)) return { ok: false, error: "Session access denied" }; if (membershipGroup(members[0]) || membershipGroup(members[1])) return { ok: false, error: "A session can belong to only one split group" }; + var roleCheck = splitRoles.validateStorePair(msg.pair, members); + if (!roleCheck.ok) return { ok: false, error: roleCheck.error }; var requestedName = typeof msg.name === "string" ? msg.name.trim().slice(0, 80) : ""; var group = { id: "sg_" + Date.now().toString(36) + "_" + crypto.randomBytes(3).toString("hex"), @@ -183,7 +109,8 @@ function createSplitGroupStore(opts) { if ((left.ownerId || null) !== (ownerId || null) || (right.ownerId || null) !== (ownerId || null)) return { ok: false, error: "Session owner does not match the scheduled task owner" }; if (membershipGroup(left.localId) || membershipGroup(right.localId)) return { ok: false, error: "A session can belong to only one split group" }; var pair = msg && msg.pair; - if (!pair || pair.driverId !== left.localId || pair.workerId !== right.localId) return { ok: false, error: "Pair roles must reference both split group members" }; + var roleCheck = splitRoles.validateStorePair(pair, members); + if (!roleCheck.ok || !pair || pair.driverId !== left.localId || pair.workerId !== right.localId) return { ok: false, error: roleCheck.error || "Pair roles must reference both split group members" }; var requestedName = typeof msg.name === "string" ? msg.name.trim().slice(0, 80) : ""; var group = { id: "sg_" + Date.now().toString(36) + "_" + crypto.randomBytes(3).toString("hex"), @@ -206,12 +133,13 @@ function createSplitGroupStore(opts) { onPairChanged(removed); return true; } - // Runtime pair lookup sees this isolated projection while ordinary list, - // save, anchor refresh, broadcast, and unrelated mutations keep using the - // durable source-owned record until commit. function beginOwnedDriverTransfer(ownerId, msg) { var group = groups.find(function (item) { return item.id === (msg && msg.id); }); if (!group || (group.ownerId || null) !== (ownerId || null)) return { ok: false, error: "Split group not found" }; + var transferRoles = splitRoles.normalizePair(group.pair, group.members); + if (transferRoles.ok && transferRoles.kind === "versioned") { + return { ok: false, error: "Version 2 Driver transfer is not enabled" }; + } if (!group.pair || group.pair.driverId !== msg.sourceDriverId || group.pair.workerId !== msg.workerId) { return { ok: false, error: "The split pair changed before Driver transfer" }; } @@ -391,6 +319,8 @@ function createSplitGroupStore(opts) { return { ok: false, error: "Driver must be a member of the split group" }; } var workerId = group.members[0] === msg.driverId ? group.members[1] : group.members[0]; + var roleCheck = splitRoles.validateStorePair({ driverId: msg.driverId, workerId: workerId }, group.members); + if (!roleCheck.ok) return { ok: false, error: roleCheck.error }; group.pair = { driverId: msg.driverId, workerId: workerId }; save(); broadcast(); @@ -398,22 +328,57 @@ function createSplitGroupStore(opts) { return { ok: true, group: group }; } - function dissolve(ws, msg) { + function dissolve(ws, msg, hooks) { var index = groups.findIndex(function (item) { return item.id === (msg && msg.id); }); if (index === -1) return { ok: false, error: "Split group not found" }; if (!canOwn(ws, groups[index])) return { ok: false, error: "Only the group owner can separate it" }; var removed = groups.splice(index, 1)[0]; - save(); + try { save(); } + catch (error) { groups.splice(index, 0, removed); return { ok: false, error: error.message || String(error) }; } + if (hooks && typeof hooks.afterPersist === "function") { + groups.splice(index, 0, removed); + var accepted; + try { accepted = hooks.afterPersist(); } + catch (error) { accepted = { ok: false, error: error.message || String(error) }; } + if (!accepted || accepted.ok === false) { + try { save(); } + catch (rollbackError) { + return { ok: false, error: (accepted && accepted.error || "close transaction rejected") + + "; persistence rollback failed: " + (rollbackError.message || String(rollbackError)) }; + } + return { ok: false, error: accepted && accepted.error || "close transaction rejected" }; + } + groups.splice(index, 1); + } broadcast(); return { ok: true, group: removed }; } function dissolveBySession(localId) { - var before = groups.length; - groups = groups.filter(function (group) { return group.members.indexOf(localId) === -1; }); - if (groups.length === before) return false; - save(); + var index = groups.findIndex(function (group) { return group.members.indexOf(localId) !== -1; }); + if (index === -1) return false; + var group = groups[index]; + var normalized = splitRoles.normalizePair(group.pair, group.members); + var beforeMembers = group.members.slice(); + var beforePair = group.pair && Object.assign({}, group.pair); + if (beforePair && Array.isArray(beforePair.workerIds)) beforePair.workerIds = beforePair.workerIds.slice(); + var removed = null; + if (normalized.ok && normalized.kind === "versioned" && normalized.driverId !== localId && + normalized.workerIds.length > 1) { + var remaining = normalized.workerIds.filter(function (id) { return id !== localId; }); + group.members = [normalized.driverId].concat(remaining); + group.pair = { version: 2, driverId: normalized.driverId, workerIds: remaining }; + } else { + removed = groups.splice(index, 1)[0]; + } + try { save(); } + catch (error) { + if (removed) groups.splice(index, 0, removed); + else { group.members = beforeMembers; group.pair = beforePair; } + return false; + } broadcast(); + onPairChanged(group); return true; } @@ -421,6 +386,11 @@ function createSplitGroupStore(opts) { var group = groupForMember(localId); if (!group) return false; var fresh = durableAnchors.memberAnchors(group, sessions); + if (fresh && fresh.version === 2) { + if (JSON.stringify(fresh) === JSON.stringify(group.memberAnchors || null)) return false; + save(); + return true; + } var prevCli = Array.isArray(group.memberCliIds) ? group.memberCliIds : [null, null]; var prevOrigin = Array.isArray(group.memberOriginIds) ? group.memberOriginIds : [null, null]; if (fresh.cli[0] === prevCli[0] && fresh.cli[1] === prevCli[1] && @@ -445,11 +415,54 @@ function createSplitGroupStore(opts) { return true; } - load(); + function removeVersionedGroup(group, hooks) { + var index = groups.indexOf(group); + if (index === -1) return { ok: false, error: "Split group not found" }; + groups.splice(index, 1); + try { save(); } + catch (error) { + groups.splice(index, 0, group); + return { ok: false, error: error.message || String(error) }; + } + if (hooks && typeof hooks.afterPersist === "function") { + groups.splice(index, 0, group); + var accepted; + try { accepted = hooks.afterPersist(); } + catch (error) { accepted = { ok: false, error: error.message || String(error) }; } + if (!accepted || accepted.ok === false) { + try { save(); } + catch (rollbackError) { + return { ok: false, error: (accepted && accepted.error || "close transaction rejected") + + "; persistence rollback failed: " + (rollbackError.message || String(rollbackError)) }; + } + return { ok: false, error: accepted && accepted.error || "close transaction rejected" }; + } + groups.splice(index, 1); + } + broadcast(); + onPairChanged(group); + return { ok: true, group: group }; + } + function versionedChanged(group) { + broadcast(); + onPairChanged(group); + } + var versionedMutations = versionedStore.attachVersionedMutations({ + sessions: sessions, + enabled: versionedEnabled, + groups: function () { return groups; }, + membershipGroup: membershipGroup, + save: save, + remove: removeVersionedGroup, + changed: versionedChanged, + }); + groups = persistence.load(); return { create: create, createOwned: createOwned, dissolveOwned: dissolveOwned, beginOwnedDriverTransfer: beginOwnedDriverTransfer, commitOwnedDriverTransfer: commitOwnedDriverTransfer, rollbackOwnedDriverTransfer: rollbackOwnedDriverTransfer, rename: rename, setPair: setPair, dissolve: dissolve, dissolveBySession: dissolveBySession, + addWorker: versionedMutations.addWorker, replaceWorker: versionedMutations.replaceWorker, + removeWorker: versionedMutations.removeWorker, refreshAutoName: refreshAutoName, refreshAnchors: refreshAnchors, listFor: listFor, groupForMember: groupForMember, get groups() { return groups; } }; } @@ -461,13 +474,11 @@ function attachSplitGroups(ctx) { usersModule: ctx.usersModule, broadcast: broadcast, onPairChanged: ctx.onPairChanged, + multiWorkerFeature: ctx.multiWorkerFeature, }); function sendState(ws) { if (!ws) return; - // Pane clients need the same owner-scoped pair roles as the parent shell. - // The Worker composer lock resolves its role from this projection; hiding - // the projection from panes leaves a configured Worker looking editable. ctx.sendTo(ws, { type: "split_groups", groups: store.listFor(ws) }); } @@ -495,5 +506,4 @@ function attachSplitGroups(ctx) { return { handleMessage: handleMessage, sendConnectionState: sendState, store: store }; } -module.exports = { attachSplitGroups: attachSplitGroups, createSplitGroupStore: createSplitGroupStore, - autoGroupName: autoGroupName }; +module.exports = { attachSplitGroups: attachSplitGroups, createSplitGroupStore: createSplitGroupStore, autoGroupName: autoGroupName }; diff --git a/lib/ws-schema.js b/lib/ws-schema.js index ca7fe350..bce52afa 100644 --- a/lib/ws-schema.js +++ b/lib/ws-schema.js @@ -41,6 +41,8 @@ var schema = { "split_group_rename": { direction: "c2s", handler: "lib/session-split-groups.js", description: "Rename a split group" }, "split_group_dissolve": { direction: "c2s", handler: "lib/session-split-groups.js", description: "Separate a split group without deleting its sessions" }, "split_group_set_pair": { direction: "c2s", handler: "lib/session-split-groups.js", description: "Assign Driver/Worker roles on a split group (driverId null clears them)" }, + "split_worker_close": { direction: "c2s", handler: "lib/project-pair-worker-close.js", description: "Close one exact authenticated Worker generation while preserving its peer and session history" }, + "split_worker_close_result": { direction: "s2c", handler: "lib/public/modules/app-messages.js", description: "Correlated result of an exact targeted Worker close" }, "pair_session_options": { direction: "c2s", handler: "lib/project-session-pair.js", description: "Request vendors/models for the pair-session dialog (response reuses the same type)" }, "pair_session_create": { direction: "c2s", handler: "lib/project-session-pair.js", description: "Create a Driver/Split Worker session pair as a split group" }, "worker_proposal_response": { direction: "c2s", handler: "lib/project-worker-proposal.js", description: "Accept or decline an exact Driver Split Worker configuration with the selected runtime" }, diff --git a/test/app-connection-production.test.js b/test/app-connection-production.test.js index f300eba2..0da70cad 100644 --- a/test/app-connection-production.test.js +++ b/test/app-connection-production.test.js @@ -86,7 +86,9 @@ function productionHarness() { }; vm.createContext(context); var lifecycle = fs.readFileSync(path.join(root, "lib/public/modules/websocket-lifecycle.js"), "utf8"); + var watchdog = fs.readFileSync(path.join(root, "lib/public/modules/websocket-watchdog.js"), "utf8"); vm.runInContext(lifecycle.replace(/export \{[^}]+\};?/, ""), context); + vm.runInContext(watchdog.replace(/export \{[^}]+\};?/, ""), context); vm.runInContext(imported.replace(/^import .*;\n/gm, "").replace(/export /g, ""), context); function tick(duration) { var end = now + duration; @@ -103,7 +105,13 @@ function productionHarness() { } now = end; } - return { context: context, listeners: listeners, sockets: sockets, fetches: fetches, timers: timers, tick: tick, getSocket: function () { return socket; } }; + function jump(duration) { + now += duration; + } + function timerCallbacks() { + return Array.from(timers.values()).map(function (entry) { return entry.fn; }); + } + return { context: context, listeners: listeners, sockets: sockets, fetches: fetches, timers: timers, tick: tick, jump: jump, timerCallbacks: timerCallbacks, getSocket: function () { return socket; } }; } test("production connection cancels close stability, stale messages, probes, and resume deadlines", function () { @@ -157,11 +165,129 @@ test("production connection cancels close stability, stale messages, probes, and }; resumedSocket.onopen(); resumed.tick(25000); + resumed.context.document.hidden = true; + resumed.listeners.visibilitychange(); + resumed.context.document.hidden = false; resumed.listeners.visibilitychange(); resumed.tick(10000); assert.equal(resumed.getSocket(), resumedSocket, "visible resume replaces the old heartbeat deadline"); }); +test("hidden suspension invalidates queued watchdog callbacks before a fresh visible probe", function () { + var f = productionHarness(); + f.context.initConnection(); + f.context.connect(); + var socket = f.sockets[0]; + var pings = 0; + socket.readyState = 1; + socket.send = function (data) { if (JSON.parse(data).type === "ping") pings += 1; }; + socket.onopen(); + f.tick(25000); + f.listeners["clay-message-delivery-timeout"]({ detail: { socket: socket } }); + var queuedCallbacks = f.timerCallbacks(); + + f.context.document.hidden = true; + f.listeners.visibilitychange(); + f.jump(60000); + queuedCallbacks.forEach(function (callback) { callback(); }); + assert.equal(f.getSocket(), socket, "callbacks queued before hidden cannot replace the retained socket"); + + socket.send = function (data) { + if (JSON.parse(data).type !== "ping") return; + pings += 1; + socket.onmessage({ data: JSON.stringify({ type: "pong" }) }); + }; + f.context.document.hidden = false; + f.listeners.visibilitychange(); + f.tick(10000); + assert.equal(f.getSocket(), socket, "the fresh visible probe proves the retained socket is healthy"); + assert.equal(pings, 3, "one heartbeat, one old probe, and one fresh visible probe are sent"); +}); + +test("freeze resume coalesces repeated events and accepts a queued pong", function () { + var f = productionHarness(); + f.context.initConnection(); + f.context.connect(); + var socket = f.sockets[0]; + var pings = 0; + socket.readyState = 1; + socket.send = function (data) { if (JSON.parse(data).type === "ping") pings += 1; }; + socket.onopen(); + f.listeners.freeze(); + f.jump(60000); + f.listeners.resume(); + f.listeners.resume(); + assert.equal(pings, 1, "repeated resume events share one in-flight liveness probe"); + socket.onmessage({ data: JSON.stringify({ type: "pong" }) }); + f.tick(4000); + assert.equal(f.getSocket(), socket, "a queued pong after resume clears the fresh probe"); +}); + +test("online while hidden waits for visibility and hidden open starts no watchdog", function () { + var hiddenOpen = productionHarness(); + hiddenOpen.context.document.hidden = true; + hiddenOpen.context.initConnection(); + hiddenOpen.context.connect(); + var hiddenSocket = hiddenOpen.sockets[0]; + var hiddenPings = 0; + hiddenSocket.readyState = 1; + hiddenSocket.send = function (data) { if (JSON.parse(data).type === "ping") hiddenPings += 1; }; + hiddenSocket.onopen(); + hiddenOpen.tick(25000); + assert.equal(hiddenPings, 0, "a socket opened while hidden has no background watchdog"); + hiddenOpen.context.document.hidden = false; + hiddenOpen.listeners.visibilitychange(); + assert.equal(hiddenPings, 1, "visibility starts one fresh probe for a hidden-opened socket"); + + var offline = productionHarness(); + offline.context.initConnection(); + offline.context.connect(); + var offlineSocket = offline.sockets[0]; + offlineSocket.readyState = 1; + offlineSocket.onopen(); + offline.context.document.hidden = true; + offline.listeners.visibilitychange(); + offline.listeners.offline(); + offline.listeners.online(); + offline.tick(20000); + assert.equal(offline.fetches.length, 0, "online while hidden does not start reconnect authentication"); + assert.equal(offline.sockets.length, 1, "online while hidden does not create a socket"); + offline.context.document.hidden = false; + offline.listeners.visibilitychange(); + offline.tick(1000); + assert.equal(offline.fetches.length, 1, "visible recovery resumes the existing reconnect policy"); +}); + +test("a dead resumed socket is replaced in four seconds and an old socket pong is ignored", function () { + var f = productionHarness(); + f.context.initConnection(); + f.context.connect(); + var first = f.sockets[0]; + first.readyState = 1; + first.send = function () {}; + first.onopen(); + var oldMessage = first.onmessage; + f.context.document.hidden = true; + f.listeners.visibilitychange(); + f.context.document.hidden = false; + f.listeners.visibilitychange(); + f.tick(3999); + assert.equal(f.getSocket(), first, "the resumed probe retains its full four-second budget"); + f.tick(1); + assert.equal(f.getSocket(), null, "an actually dead resumed socket is replaced at four seconds"); + + f.tick(1000); + f.fetches[0].resolve({ status: 200 }); + var second = f.sockets[1]; + second.readyState = 1; + second.send = function () {}; + second.onopen(); + f.listeners.visibilitychange(); + oldMessage({ data: JSON.stringify({ type: "pong" }) }); + f.tick(4000); + assert.equal(f.getSocket(), null, "a late pong from the retired socket cannot clear the current probe"); +}); + test("production handshake timeout replaces a stalled socket and server health resets stability", function () { var f = productionHarness(); f.context.connect(); diff --git a/test/client-connection-blink.test.js b/test/client-connection-blink.test.js index 29f274fb..5262f4e7 100644 --- a/test/client-connection-blink.test.js +++ b/test/client-connection-blink.test.js @@ -5,6 +5,7 @@ var path = require("node:path"); var root = path.join(__dirname, ".."); var connection = fs.readFileSync(path.join(root, "lib/public/modules/app-connection.js"), "utf8"); +var watchdog = fs.readFileSync(path.join(root, "lib/public/modules/websocket-watchdog.js"), "utf8"); var notes = fs.readFileSync(path.join(root, "lib/public/modules/sticky-notes.js"), "utf8"); var notesCss = fs.readFileSync(path.join(root, "lib/public/css/sticky-notes.css"), "utf8"); var sessions = fs.readFileSync(path.join(root, "lib/public/modules/sidebar-sessions.js"), "utf8"); @@ -74,21 +75,21 @@ test("exactly one heartbeat timer exists per live socket", function () { assert.ok(beat > 0 && beat < 60000, "heartbeat is shorter than a typical 60s proxy idle timeout, got " + beat); // Starting always clears first, so a restart cannot leave two timers. - var start = slice(connection, "export function startHeartbeat(socket, attemptEpoch)", "export function initConnection"); + var start = slice(watchdog, "function startHeartbeat(socket, epoch)", "function probe(socket, epoch)"); assert.ok(start.indexOf("stopHeartbeat();") < start.indexOf("heartbeatTimer = setInterval("), "startHeartbeat clears any existing timer before creating one"); - assert.match(start, /if \(!socket\) return;/); + assert.match(start, /if \(suspended \|\| !socket\) return false;/); // The timer is bound to the socket it was started for and stops itself if // that socket is no longer the live one. - assert.match(start, /if \(!socket \|\| socket\.readyState !== 1 \|\| getWs\(\) !== socket \|\| !lifecycle\.current\(attemptEpoch\)\) \{\s*\n\s*stopHeartbeat\(\);\s*\n\s*return;/); - assert.match(start, /socket\.send\(JSON\.stringify\(\{ type: "ping" \}\)\)/); + assert.match(start, /if \(!options\.isCurrent\(socket, epoch\)\) \{\s*\n\s*stopHeartbeat\(\);\s*\n\s*return;/); + assert.match(start, /options\.sendPing\(socket\);/); // A half-open socket must be replaced when the server does not acknowledge // the heartbeat. Browser OPEN state alone does not prove delivery. - assert.match(start, /heartbeatDeadlineTimer = setTimeout\([\s\S]*getWs\(\) === socket[\s\S]*forceReplaceSocket\("heartbeat_timeout"/, + assert.match(start, /heartbeatDeadlineTimer = setTimeout\([\s\S]*options\.isCurrent\(socket, epoch\)[\s\S]*options\.onFailure\("heartbeat_timeout"/, "a missed pong directly replaces the stale socket through the bounded retry path"); - assert.match(connection, /if \(msg\.type === "pong" && heartbeatDeadlineTimer && heartbeatAwaitingSocket === newWs\) \{[\s\S]*clearTimeout\(heartbeatDeadlineTimer\)/, + assert.match(watchdog, /if \(heartbeatSocket === socket && heartbeatEpoch === epoch\) clearHeartbeatDeadline\(\);/, "a pong cancels the reconnect deadline"); }); @@ -104,7 +105,8 @@ test("the heartbeat is cleared on close and before a new socket is created", fun var onopen = slice(connection, "newWs.onopen = function ()", "newWs.onclose"); assert.match(onopen, /startHeartbeat\(newWs, attemptEpoch\);/); - assert.match(connection, /function stopHeartbeat\(\) \{[\s\S]*clearInterval\(heartbeatTimer\);[\s\S]*heartbeatTimer = null;/); + assert.match(connection, /function stopHeartbeat\(\) \{\s*\n\s*watchdog\.stopHeartbeat\(\);/); + assert.match(watchdog, /function stopHeartbeat\(\) \{[\s\S]*clearInterval\(heartbeatTimer\);[\s\S]*heartbeatTimer = null;/); }); test("connection callbacks are epoch-bound and resume replaces stale health deadlines", function () { @@ -112,8 +114,10 @@ test("connection callbacks are epoch-bound and resume replaces stale health dead assert.match(onmessage, /if \(getWs\(\) !== newWs \|\| !lifecycle\.current\(attemptEpoch\)\) return;/); assert.match(connection, /lifecycle\.setOffline\(true\);\s*\n\s*suspendSocketForOffline\(\);/); assert.match(connection, /function suspendSocketForOffline\(\) \{[\s\S]*socket\.onmessage = null;[\s\S]*setWs\(null\);/); - assert.match(connection, /if \(socket && socket\.readyState === 1\) startHeartbeat\(socket, lifecycle\.getEpoch\(\)\);\s*\n\s*probeReconnect\("online"\)/); - assert.match(connection, /if \(socket && socket\.readyState === 1\) startHeartbeat\(socket, lifecycle\.getEpoch\(\)\);\s*\n\s*probeReconnect\("visible"\)/); + assert.match(connection, /if \(document\.hidden\) return;\s*\n\s*resumeSocketWatchdogs\("online"\)/); + assert.match(connection, /if \(document\.hidden\) suspendSocketWatchdogs\(\);\s*\n\s*else resumeSocketWatchdogs\("visible"\)/); + assert.match(connection, /document\.addEventListener\("freeze"[\s\S]*suspendSocketWatchdogs\(\);/); + assert.match(watchdog, /function suspend\(\) \{[\s\S]*stopHeartbeat\(\);[\s\S]*clearProbe\(\);/); assert.match(connection, /if \(lifecycle\.isOffline\(\) \|\| !lifecycle\.current\(lifecycle\.getEpoch\(\)\)\) return;/); assert.match(connection, /var socket = getWs\(\);\s*\n\s*if \(socket && socket\.readyState === 0\) return;/); assert.match(connection, /recordConnectionDiagnostic\("error", 0, null\);\s*\n\s*forceReplaceSocket\("error", 0, attemptEpoch\);/); @@ -239,7 +243,7 @@ test("an identical session_list frame does not tear down the sidebar", function // --- Project rules -------------------------------------------------------- test("the edited client modules follow the project style rules", function () { - [["app-connection.js", connection], ["sticky-notes.js", notes], ["sidebar-sessions.js", sessions]].forEach(function (pair) { + [["app-connection.js", connection], ["websocket-watchdog.js", watchdog], ["sticky-notes.js", notes], ["sidebar-sessions.js", sessions]].forEach(function (pair) { assert.equal(/=>/.test(pair[1]), false, "no arrow functions in " + pair[0]); assert.equal(/^\s*(const|let)\s/m.test(pair[1]), false, "var only in " + pair[0]); }); diff --git a/test/filebrowser-tabs.test.js b/test/filebrowser-tabs.test.js index 7daa49e0..2703db14 100644 --- a/test/filebrowser-tabs.test.js +++ b/test/filebrowser-tabs.test.js @@ -196,11 +196,12 @@ test("split and pane markdown presents use the parent-owned viewer path", functi test("split panes attach live project and session identity for file-chip routing", function() { var split = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); + var renderer = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-pane-renderer.js"), "utf8"); var bridge = fs.readFileSync(path.join(__dirname, "../lib/public/modules/pane-file-bridge.js"), "utf8"); var links = fs.readFileSync(path.join(__dirname, "../lib/public/modules/clay-file-links.js"), "utf8"); - assert.match(split, /frame\.dataset\.projectSlug = pane\.slug/); - assert.match(split, /frame\.dataset\.sessionId = String\(pane\.sessionId\)/); + assert.match(renderer, /frame\.dataset\.projectSlug = pane\.slug/); + assert.match(renderer, /frame\.dataset\.sessionId = String\(pane\.sessionId\)/); assert.match(split, /handlePaneFileMessage\(event, host, openFile/); assert.match(bridge, /event\.origin !== window\.location\.origin/); assert.match(bridge, /paneFrame\(host, event\.source\)/); diff --git a/test/fixtures/split-multi-worker.html b/test/fixtures/split-multi-worker.html new file mode 100644 index 00000000..503fd485 --- /dev/null +++ b/test/fixtures/split-multi-worker.html @@ -0,0 +1,158 @@ + + + + + + Clay split multi-Worker fixture + + + + + +
+
+
+
+ + + + + diff --git a/test/multi-worker-feature.test.js b/test/multi-worker-feature.test.js new file mode 100644 index 00000000..57d74b75 --- /dev/null +++ b/test/multi-worker-feature.test.js @@ -0,0 +1,19 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var feature = require("../lib/multi-worker-feature"); +var fs = require("fs"); +var path = require("path"); + +test("the server-owned multi-Worker feature is default-off and exact", function () { + assert.equal(feature.fromServerConfig({}), null); + assert.equal(feature.fromServerConfig({ multiWorkerRuntimeEnabled: false }), null); + assert.equal(feature.fromServerConfig({ multiWorkerRuntimeEnabled: "true" }), null); + assert.equal(feature.isEnabled({ feature: "multi-worker-runtime" }), false); + var enabled = feature.fromServerConfig({ multiWorkerRuntimeEnabled: true }); + assert.equal(feature.isEnabled(enabled), true); +}); + +test("production project creation enables multi-Worker runtime only for non-Mate contexts", function () { + var source = fs.readFileSync(path.join(__dirname, "../lib/server.js"), "utf8"); + assert.match(source, /multiWorkerRuntimeEnabled:\s*!extra\.isMate/); +}); diff --git a/test/pair-lifecycle.test.js b/test/pair-lifecycle.test.js index 4e916393..80112594 100644 --- a/test/pair-lifecycle.test.js +++ b/test/pair-lifecycle.test.js @@ -203,6 +203,16 @@ function makeLifecycleExceptionWorld(options) { return { driver: driver, worker: worker, store: store, turnControl: turnControl, lifecycle: lifecycle }; } +test("one Driver generation ledger assigns distinct identities to concurrent Workers", function () { + var world = makeLifecycleExceptionWorld(); + var second = { localId: 3, ownerId: "owner", history: [], sessionOriginId: "worker-origin-b", + sessionProvenance: { kind: "worker" } }; + world.lifecycle.recordGenerationStart(world.driver, world.worker); + var generation = world.lifecycle.recordGenerationStart(world.driver, second); + assert.equal(generation, 2); + assert.deepEqual(world.driver._workerGenerations.map(function (record) { return record.generation; }), [1, 2]); +}); + // Delegate, then let the Worker's turn complete the way the real bridge does. // Without the turn-done hook the delegation token stays open, and a Worker // holding an open delegated task is legitimately not safe to replace. @@ -1214,12 +1224,33 @@ test("the Driver prompt explains pending and audited full-access runtime decisio assert.match(prompts.DRIVER, /spanning multiple modules/); assert.match(prompts.DRIVER, /Your own capability is not a reason to retain that execution/); assert.match(prompts.DRIVER, /unless the user explicitly asks you to work directly/); + assert.match(prompts.DRIVER, /lowest-capability model and thinking effort/); + assert.match(prompts.DRIVER, /server-provided capability evidence/); + assert.match(prompts.DRIVER, /Explicitly fill the vendor, model, and thinking effort recommendation fields for both creation and replacement/); + assert.match(prompts.DRIVER, /never rely on card defaults/); + assert.match(prompts.DRIVER, /30 percent Driver and 70 percent Worker/); + assert.match(prompts.DRIVER, /not a guaranteed token or cost ratio/); + assert.match(prompts.DRIVER, /one bounded contract/); + assert.match(prompts.DRIVER, /production-path tests/); + assert.match(prompts.DRIVER, /one correction task/); + assert.doesNotMatch(prompts.DRIVER, /gpt-5\.6-luna|gpt-5\.6-sol|codex/); assert.equal(/[^\x00-\x7F]/.test(prompts.DRIVER), false, "English ASCII only"); assert.equal(/[^\x00-\x7F]/.test(prompts.UNPAIRED), false); assert.match(prompts.UNPAIRED, /call propose_worker/); assert.match(prompts.UNPAIRED, /crosses client\/server\/data boundaries/); assert.match(prompts.UNPAIRED, /Your own capability is not a reason to skip delegation/); assert.match(prompts.UNPAIRED, /unless the user explicitly asks you to work directly/); + assert.match(prompts.UNPAIRED, /lowest-capability model and thinking effort/); + assert.match(prompts.UNPAIRED, /Explicitly fill the vendor, model, and thinking effort recommendation fields for both creation and replacement/); + assert.match(prompts.UNPAIRED, /never rely on card defaults/); + assert.doesNotMatch(prompts.UNPAIRED, /gpt-5\.6-luna|gpt-5\.6-sol|codex/); + var workerPrompt = prompts.worker("task-123"); + assert.match(workerPrompt, /task-123/); + assert.match(workerPrompt, /production-path tests/); + assert.match(workerPrompt, /bounded self-correction/); + assert.match(workerPrompt, /report_partner_outcome/); + assert.match(workerPrompt, /actual exit results/); + assert.match(workerPrompt, /Never claim Driver verification/); var proposalSource = fs.readFileSync(path.join(root, "lib/project-worker-proposal.js"), "utf8"); assert.match(proposalSource, /Do not skip delegation merely because you can implement it yourself/); }); diff --git a/test/permission-mode-ui.test.js b/test/permission-mode-ui.test.js index 88ccd6f7..4440d8a9 100644 --- a/test/permission-mode-ui.test.js +++ b/test/permission-mode-ui.test.js @@ -133,19 +133,20 @@ test("standalone header exposes Ask Auto Skip permissions and runtime status", f test("standalone and split Driver headers use the shared permission control", function () { var header = read("lib/public/modules/app-header.js"); var split = read("lib/public/modules/split-view.js"); + var renderer = read("lib/public/modules/split-pane-renderer.js"); var shared = read("lib/public/modules/permission-control.js"); assert.match(header, /from '.\/permission-control\.js'/); assert.match(header, /bindPermissionControl\(/); assert.match(header, /renderPermissionControl\(/); - assert.match(split, /from '.\/permission-control\.js'/); - assert.match(split, /createPermissionControl\("split-pane-full-access hidden"/); - assert.match(split, /bindPermissionControl\(/); - assert.match(split, /renderPermissionControl\(/); + assert.match(renderer, /from '.\/permission-control\.js'/); + assert.match(renderer, /createPermissionControl\("split-pane-full-access hidden"/); + assert.match(renderer, /bindPermissionControl\(/); + assert.match(renderer, /renderPermissionControl\(/); assert.match(shared, /projectSlug: projectSlug, sessionId: sessionId, mode: mode, requestId: requestId/); assert.match(shared, /Auto unavailable at runtime/); assert.match(shared, /Auto unavailable for this Claude session/); assert.doesNotMatch(shared, /dataset\.projectSlug|dataset\.sessionId/); - assert.doesNotMatch(split, /setPaneFullAccess|Skip permission prompts for this session/); + assert.doesNotMatch(renderer, /setPaneFullAccess|Skip permission prompts for this session/); }); test("permission requests are pending per project and session and include correlation scope", function () { diff --git a/test/project-pair-result-outbox.test.js b/test/project-pair-result-outbox.test.js index cb5cb58a..9b30953f 100644 --- a/test/project-pair-result-outbox.test.js +++ b/test/project-pair-result-outbox.test.js @@ -17,6 +17,35 @@ function atomicPersist(filePath, serialized) { return true; } +test("prepared close survives reload as non-deliverable intent and terminal evidence can supersede it", function () { + var filePath = tempPath(); + var outbox = attach({ filePath: filePath }); + var started = outbox.begin({ ownerId: "owner", projectSlug: "project", groupId: "group", + driverOriginId: "driver-origin", workerOriginId: "worker-origin", + taskId: "task-prepared", generation: 2, deliveryRoute: "callback" }); + var proposed = { taskId: "task-prepared", generation: 2, status: "interrupted", response: "partial" }; + assert.equal(outbox.prepareCapture(started.key, proposed).ok, true); + var reloaded = attach({ filePath: filePath }); + var prepared = reloaded.get(started.key); + assert.equal(prepared.state, "close_prepared"); + assert.equal(prepared.outcome, null); + assert.deepEqual(prepared.preparedCloseOutcome, proposed); + assert.equal(reloaded.beginDelivery(started.key).ok, false); + assert.equal(reloaded.status().records[0].recoveryState, "uncertain"); + var actual = { taskId: "task-prepared", generation: 2, status: "completed", response: "actual result" }; + var restoredWorker = { ownerId: "owner", sessionOriginId: "worker-origin", history: [{ + type: "pair_task_completed", ownerId: "owner", driverOriginId: "driver-origin", + workerOriginId: "worker-origin", taskId: "task-prepared", generation: 2, outcome: actual, + }] }; + var recovered = reloaded.recoverPending(new Map([[1, restoredWorker]])); + assert.equal(recovered.length, 1); + assert.equal(recovered[0].result.ok, true); + assert.deepEqual(reloaded.get(started.key).outcome, actual); + assert.equal(reloaded.get(started.key).preparedCloseOutcome, null); + assert.equal(reloaded.get(started.key).closePreparationState, "superseded_by_terminal_evidence"); + fs.rmSync(path.dirname(filePath), { recursive: true, force: true }); +}); + test("captures one immutable result after false and throw persistence failures", async function () { var filePath = tempPath(); var calls = 0; diff --git a/test/project-pair-result-recovery.test.js b/test/project-pair-result-recovery.test.js index 634bccf2..33d230b8 100644 --- a/test/project-pair-result-recovery.test.js +++ b/test/project-pair-result-recovery.test.js @@ -65,6 +65,40 @@ test("accepted recovery finalizes once without replaying transport", function () fs.rmSync(f.dir, { recursive: true, force: true }); }); +test("unproven prepared close never becomes an outcome from recovery topology", function () { + var mutations = [ + function (f) { f.group.members = []; f.group.pair = null; }, + function (f) { f.group.members = [f.driver.localId]; f.group.pair = { driverId: f.driver.localId, workerId: 999 }; }, + function (f) { + var replacement = { localId: 88, ownerId: "owner", sessionOriginId: "replacement-origin", history: [] }; + f.sessions.set(replacement.localId, replacement); + f.group.members = [f.driver.localId, replacement.localId]; + f.group.pair = { driverId: f.driver.localId, workerId: replacement.localId }; + }, + function (f) { f.worker.ownerId = "changed-owner"; }, + function (f) { f.worker.sessionProvenance.generation = 3; }, + ]; + for (var i = 0; i < mutations.length; i++) { + var f = fixture("owner"); + var begun = f.outbox.begin({ ownerId: "owner", projectSlug: "project", groupId: f.group.id, + driverOriginId: "driver-origin", workerOriginId: "worker-origin", taskId: "close-task-" + i, + generation: 2, historyStartIndex: 0, message: "close task", deliveryRoute: "callback" }); + var proposed = { taskId: "close-task-" + i, generation: 2, status: "interrupted", response: "partial" }; + assert.equal(f.outbox.prepareCapture(begun.key, proposed).ok, true); + mutations[i](f); + f.recovery.reconcile(); + var retained = f.outbox.get(begun.key); + assert.equal(retained.state, "close_prepared"); + assert.equal(retained.outcome, null); + assert.deepEqual(retained.preparedCloseOutcome, proposed); + assert.equal(f.wakeCount(), 0); + var ws = { _clayUser: { id: "owner" }, _clayActiveSession: f.driver.localId, messages: [] }; + f.recovery.sendState(f.driver, ws); + assert.deepEqual(ws.messages[0].items, []); + fs.rmSync(f.dir, { recursive: true, force: true }); + } +}); + test("restart converts attempting to visible uncertain and requires explicit duplicate confirmation", async function () { var f = fixture("owner"), key = f.captured(), attempt = f.outbox.beginDelivery(key); assert.equal(attempt.ok, true); @@ -218,6 +252,67 @@ test("real manager reload and session activation drive the real sender without W fs.rmSync(root, { recursive: true, force: true }); }); +test("real manager reload routes two renumbered Worker results by stable origins without redispatch", async function () { + var root = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-result-reload-")); + var sessionsBase = path.join(root, "sessions"), projectPath = path.join(root, "project"); + function manager() { return createSessionManager({ cwd: projectPath, sessionsBase: sessionsBase, + cliSessionsDir: path.join(root, "cli"), send: function () {}, sendTo: function () {} }); } + var first = manager(); + var padding = first.createSessionRaw({ cliSessionId: "padding", ownerId: "owner" }); + var driver = first.createSessionRaw({ cliSessionId: "driver-v2", ownerId: "owner" }); + var workerA = first.createSessionRaw({ cliSessionId: "worker-a-v2", ownerId: "owner" }); + var workerB = first.createSessionRaw({ cliSessionId: "worker-b-v2", ownerId: "owner" }); + workerA.sessionProvenance = { kind: "worker", parentSessionOriginId: driver.sessionOriginId, generation: 4, createdVia: "split-worker" }; + workerB.sessionProvenance = { kind: "worker", parentSessionOriginId: driver.sessionOriginId, generation: 5, createdVia: "split-worker" }; + first.saveSessionFile(driver); first.saveSessionFile(workerA); first.saveSessionFile(workerB); + first.deleteSession(padding.localId); + var firstOutbox = attachPairResultOutbox({ storageDir: first.sessionsDir, projectSlug: "project" }); + var startedA = firstOutbox.begin({ ownerId: "owner", projectSlug: "project", driverOriginId: driver.sessionOriginId, + workerOriginId: workerA.sessionOriginId, taskId: "reload-a", generation: 4, message: "A", deliveryRoute: "callback" }); + var startedB = firstOutbox.begin({ ownerId: "owner", projectSlug: "project", driverOriginId: driver.sessionOriginId, + workerOriginId: workerB.sessionOriginId, taskId: "reload-b", generation: 5, message: "B", deliveryRoute: "callback" }); + firstOutbox.capture(startedA.key, { taskId: "reload-a", generation: 4, status: "completed", response: "A done" }); + firstOutbox.capture(startedB.key, { taskId: "reload-b", generation: 5, status: "completed", response: "B done" }); + + var reloaded = manager(), restoredDriver, restoredA, restoredB; + reloaded.sessions.forEach(function (session) { + if (session.sessionOriginId === driver.sessionOriginId) restoredDriver = session; + if (session.sessionOriginId === workerA.sessionOriginId) restoredA = session; + if (session.sessionOriginId === workerB.sessionOriginId) restoredB = session; + }); + assert.notEqual(restoredDriver.localId, driver.localId, "removing the padding session renumbers restored sessions"); + var group = { id: "injected-v2", members: [restoredDriver.localId, restoredA.localId, restoredB.localId], + pair: { version: 2, driverId: restoredDriver.localId, workerIds: [restoredA.localId, restoredB.localId] } }; + var store = { groupForMember: function (id) { return group.members.indexOf(id) === -1 ? null : group; } }; + var outbox = attachPairResultOutbox({ storageDir: reloaded.sessionsDir, projectSlug: "project" }); + var driverStarts = 0, workerStarts = 0, recovery; + var sdk = { pushMessage: function () { return false; }, startQuery: function (session, text, images, linuxUser, beforePush, onAccepted) { + if (session === restoredA || session === restoredB) workerStarts++; + else driverStarts++; + if (beforePush() === true) onAccepted(); + return Promise.resolve(true); + } }; + function finish(caller, partner, token) { + var saved = outbox.markFinalized(token.outboxKey); + if (saved.ok) { delete partner._pairDelegation; delete partner._delegatedBy; } + return saved.ok; + } + var delivery = attachPairResultDelivery({ sm: reloaded, store: store, outbox: outbox, getSdk: function () { return sdk; }, + getLinuxUserForSession: function () { return null; }, blockedReason: function () { return null; }, finish: finish, + onStateChange: function (caller, record) { if (recovery) recovery.stateChanged(caller, record); } }); + recovery = attachPairResultRecovery({ sm: reloaded, store: store, outbox: outbox, resultCapture: { finish: finish }, + wake: delivery.wake, projectSlug: "project", blockedReason: function () { return null; }, sendTo: function () {}, + getClients: function () { return []; }, isMultiUser: function () { return true; } }); + recovery.available(restoredDriver); + await new Promise(function (resolve) { setImmediate(resolve); }); + assert.equal(driverStarts, 2); + assert.equal(workerStarts, 0); + assert.equal(outbox.list().filter(function (record) { return record.deliveryState === "accepted" && record.finalized; }).length, 2); + assert.equal(restoredA.history.filter(function (entry) { return entry && entry.type === "user_message"; }).length, 0); + assert.equal(restoredB.history.filter(function (entry) { return entry && entry.type === "user_message"; }).length, 0); + fs.rmSync(root, { recursive: true, force: true }); +}); + test("explicit retry resets live token flags while stale sender acceptance stays isolated", async function () { var f = fixture("owner"), key = f.captured(), callbacks = [], recovery; var sm = { sessions: f.sessions, addOnSessionViewed: function () {}, diff --git a/test/project-pair-worker-close.test.js b/test/project-pair-worker-close.test.js new file mode 100644 index 00000000..b2b85b72 --- /dev/null +++ b/test/project-pair-worker-close.test.js @@ -0,0 +1,373 @@ +var test = require("node:test"); +var assert = require("node:assert"); +var fs = require("node:fs"); +var os = require("node:os"); +var path = require("node:path"); +var attachPairWorkerClose = require("../lib/project-pair-worker-close").attachPairWorkerClose; +var multiWorkerFeature = require("../lib/multi-worker-feature"); +var createSessionManager = require("../lib/sessions").createSessionManager; +var createSplitGroupStore = require("../lib/session-split-groups").createSplitGroupStore; +var attachSessionPair = require("../lib/project-session-pair").attachSessionPair; +var attachPairResultOutbox = require("../lib/project-pair-result-outbox").attachPairResultOutbox; +var provenance = require("../lib/session-provenance"); + +var helperPromise = null; +function loadClientHelpers() { + if (!helperPromise) { + var source = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-group-helpers.js"), "utf8"); + helperPromise = import("data:text/javascript;base64," + Buffer.from(source).toString("base64")); + } + return helperPromise; +} + +function atomicWrite(filePath, serialized) { + var tempPath = filePath + ".test"; + fs.writeFileSync(tempPath, serialized, "utf8"); + fs.renameSync(tempPath, filePath); + return true; +} + +async function realFixture(t) { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-real-worker-close-")); + var mode = "ok"; + var outboxMode = "ok"; + var outboxPersistCalls = 0; + var outboxFailAt = null; + var outboxFailKind = null; + var groupPersistCalls = 0; + var sent = []; + var workerStarts = 0; + var aborts = 0; + var capability = multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }); + var sm = createSessionManager({ cwd: dir, sessionsBase: dir, cliSessionsDir: path.join(dir, "cli"), + send: function () {}, sendTo: function () {} }); + var driver = sm.createSessionRaw({ cliSessionId: "cli-driver", vendor: "codex" }); + var workerA = sm.createSessionRaw({ cliSessionId: "cli-worker-a", vendor: "codex" }); + var workerB = sm.createSessionRaw({ cliSessionId: "cli-worker-b", vendor: "codex" }); + driver.title = "Driver"; workerA.title = "Worker A"; workerB.title = "Worker B"; + driver.queryInstance = {}; + provenance.markWorker(driver, workerA, sm.sessions); + provenance.markWorker(driver, workerB, sm.sessions); + workerA._pairGeneration = workerA.sessionProvenance.generation; + workerB._pairGeneration = workerB.sessionProvenance.generation; + var store = createSplitGroupStore({ sessions: sm.sessions, sessionsDir: sm.sessionsDir, + usersModule: null, multiWorkerFeature: capability, + persistGroups: function (filePath, serialized) { + groupPersistCalls++; + if (mode === "false") return false; + if (mode === "throw") throw new Error("group persistence threw"); + return atomicWrite(filePath, serialized); + } }); + var created = store.create(null, { members: [driver.localId, workerA.localId], + pair: { driverId: driver.localId, workerId: workerA.localId } }); + assert.equal(created.ok, true); + assert.equal(store.addWorker(null, { + groupId: created.group.id, driverId: driver.localId, driverOriginId: driver.sessionOriginId, + newWorkerId: workerB.localId, newWorkerOriginId: workerB.sessionOriginId, + }, capability).ok, true); + var outbox = attachPairResultOutbox({ storageDir: sm.sessionsDir, projectSlug: "project-a", + persist: function (filePath, serialized) { + outboxPersistCalls++; + if (outboxFailAt === outboxPersistCalls) { + if (outboxFailKind === "false") return false; + throw new Error("outbox persistence threw"); + } + if (outboxMode === "false") return false; + if (outboxMode === "throw") throw new Error("outbox persistence threw"); + return atomicWrite(filePath, serialized); + } }); + var sdk = { + pushMessage: function (target) { return target === driver; }, + startQuery: function (target) { if (target === workerA) workerStarts++; return Promise.resolve(true); }, + }; + var pair = attachSessionPair({ sm: sm, splitStore: store, resultOutbox: outbox, + multiWorkerFeature: capability, projectSlug: "project-a", getSdk: function () { return sdk; }, + getLinuxUserForSession: function () { return null; }, send: function () {}, sendTo: function () {}, + usersModule: { isMultiUser: function () { return false; } }, onProcessingChanged: function () {} }); + workerA.abortController = { abort: function () { aborts++; } }; + var activeResult = await pair.sendToPartner({ workerId: workerA.localId, taskId: "task-close-a", + message: "Keep this task live until targeted close.", wait: false }, driver); + assert.equal(JSON.parse(activeResult.content[0].text).status, "running"); + var queueTool = pair.getToolDefs(driver).find(function (tool) { return tool.name === "queue_partner_followup"; }); + var queued = JSON.parse((await queueTool.handler({ workerId: workerA.localId, taskId: "task-queued-a", + message: "Queued work must survive a failed close." })).content[0].text); + assert.equal(queued.status, "queued"); + var permissionPromise = pair.workerPermission.routeIfWorker(workerA, + { toolName: "Write", input: { file_path: "/tmp/close-test" }, toolUseId: "close-permission" }); + assert.equal(pair.workerPermission.pendingCountFor(workerA), 1); + var ws = { readyState: 1, _clayActiveSession: driver.localId }; + var sockets = new Set([ws]); + var handler = attachPairWorkerClose({ sm: sm, splitStore: store, multiWorkerFeature: capability, + isMate: false, projectSlug: "project-a", hasSocket: function (socket) { return sockets.has(socket); }, + sendTo: function (socket, message) { sent.push(message); }, closeWorker: pair.closeWorker }); + var helpers = await loadClientHelpers(); + var groupProjection = store.listFor(ws)[0]; + var sessionProjection = sm.mapSessionsForClient(Array.from(sm.sessions.values()), driver.localId, {}); + var request = helpers.splitWorkerCloseRequest(groupProjection, sessionProjection, "project-a", + workerA.localId, "real-close-request"); + assert.ok(request); + t.after(async function () { + pair.workerPermission.cancelForSession(workerA, "test cleanup"); + workerA.isProcessing = false; + await permissionPromise; + await new Promise(function (resolve) { setTimeout(resolve, 550); }); + fs.rmSync(dir, { recursive: true, force: true }); + }); + return { dir: dir, setMode: function (value) { mode = value; }, + setOutboxMode: function (value) { outboxMode = value; }, sm: sm, store: store, + failOutboxAfter: function (offset, kind) { outboxFailAt = outboxPersistCalls + offset; outboxFailKind = kind; }, + driver: driver, workerA: workerA, workerB: workerB, pair: pair, outbox: outbox, + handler: handler, ws: ws, sent: sent, request: request, permissionPromise: permissionPromise, + workerStarts: function () { return workerStarts; }, aborts: function () { return aborts; }, + groupPersistCalls: function () { return groupPersistCalls; } }; +} + +function session(id, origin, generation) { + return { + localId: id, + ownerId: "owner-a", + sessionOriginId: origin, + _pairGeneration: generation || null, + }; +} + +function fixture() { + var driver = session(1, "origin-driver"); + var workerA = session(2, "origin-worker-a", 4); + var workerB = session(3, "origin-worker-b", 5); + var sessions = new Map([[1, driver], [2, workerA], [3, workerB]]); + var group = { + id: "group-a", + ownerId: "owner-a", + members: [1, 2, 3], + pair: { version: 2, driverId: 1, workerIds: [2, 3] }, + }; + var ws = { readyState: 1, _clayUser: { id: "owner-a" }, _clayActiveSession: 1 }; + var sockets = new Set([ws]); + var sent = []; + var closes = []; + var handler = attachPairWorkerClose({ + sm: { sessions: sessions }, + splitStore: { groupForMember: function (id) { return group.members.indexOf(id) === -1 ? null : group; } }, + multiWorkerFeature: multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }), + isMate: false, + projectSlug: "project-a", + hasSocket: function (socket) { return sockets.has(socket); }, + sendTo: function (socket, message) { sent.push({ socket: socket, message: message }); }, + closeWorker: function (args, caller) { closes.push({ args: args, caller: caller }); return { status: "closed" }; }, + }); + function message() { + return { + type: "split_worker_close", + requestId: "request-a", + projectSlug: "project-a", + groupId: "group-a", + driverId: 1, + driverOriginId: "origin-driver", + workerId: 2, + workerOriginId: "origin-worker-a", + generation: 4, + expectedWorkerIds: [2, 3], + }; + } + return { handler: handler, ws: ws, sockets: sockets, sent: sent, closes: closes, group: group, message: message }; +} + +test("authenticated exact Driver socket closes only the selected Worker generation", function () { + var f = fixture(); + assert.equal(f.handler.handleMessage(f.ws, f.message()), true); + assert.equal(f.closes.length, 1); + assert.deepEqual(f.closes[0].args, { workerId: 2 }); + assert.equal(f.closes[0].caller.localId, 1); + assert.equal(f.sent[0].message.ok, true); + assert.equal(f.sent[0].message.workerId, 2); +}); + +test("real capture failure retains the exact live task and never attempts structural removal", async function (t) { + var f = await realFixture(t); + var groupBefore = JSON.parse(JSON.stringify(f.store.groups[0])); + var token = f.workerA._pairDelegation; + var tokenBefore = JSON.parse(JSON.stringify(token)); + var persistCalls = f.groupPersistCalls(); + f.setOutboxMode("false"); + assert.equal(f.handler.handleMessage(f.ws, f.request), true); + assert.equal(f.sent[f.sent.length - 1].ok, false); + assert.match(f.sent[f.sent.length - 1].error, /persistence returned false|durably capture/); + assert.deepEqual(f.store.groups[0], groupBefore); + assert.equal(f.workerA._pairDelegation, token); + assert.deepEqual(JSON.parse(JSON.stringify(token)), tokenBefore); + assert.equal(f.pair.workerPermission.pendingCountFor(f.workerA), 1); + assert.equal(f.aborts(), 0); + assert.equal(f.outbox.get(token.outboxKey).outcome, null); + assert.equal(f.groupPersistCalls(), persistCalls); +}); + +["false", "throw"].forEach(function (rollbackMode) { + test("failed structural close with rollback " + rollbackMode + " reloads as non-deliverable prepared intent", async function (t) { + var f = await realFixture(t); + var token = f.workerA._pairDelegation; + var tokenBefore = JSON.parse(JSON.stringify(token)); + f.setMode("false"); + f.failOutboxAfter(2, rollbackMode); + assert.equal(f.handler.handleMessage(f.ws, f.request), true); + assert.equal(f.sent[f.sent.length - 1].ok, false); + assert.match(f.sent[f.sent.length - 1].error, /rollback failed/); + assert.equal(f.workerA._pairDelegation, token); + assert.deepEqual(JSON.parse(JSON.stringify(token)), tokenBefore); + assert.equal(f.pair.workerPermission.pendingCountFor(f.workerA), 1); + assert.equal(f.aborts(), 0); + var reloaded = attachPairResultOutbox({ storageDir: f.sm.sessionsDir, projectSlug: "project-a" }); + var record = reloaded.get(token.outboxKey); + assert.equal(record.state, "close_prepared"); + assert.equal(record.outcome, null); + assert.equal(record.preparedCloseOutcome.status, "interrupted"); + assert.equal(reloaded.beginDelivery(token.outboxKey).ok, false); + }); +}); + +test("stale identity, membership, project, socket, and generation never reach structural close", function () { + var mutations = [ + function (f, msg) { msg.projectSlug = "project-b"; }, + function (f, msg) { msg.driverOriginId = "stale-driver"; }, + function (f, msg) { msg.expectedWorkerIds = [3, 2]; }, + function (f, msg) { msg.workerOriginId = "stale-worker"; }, + function (f, msg) { msg.generation = 3; }, + function (f) { f.sockets.clear(); }, + ]; + for (var i = 0; i < mutations.length; i++) { + var f = fixture(); + var msg = f.message(); + mutations[i](f, msg); + assert.equal(f.handler.handleMessage(f.ws, msg), true); + assert.equal(f.closes.length, 0); + assert.equal(f.sent[0].message.ok, false); + } +}); + +test("a replay after the selected Worker leaves the group is rejected without a second mutation", function () { + var f = fixture(); + var msg = f.message(); + assert.equal(f.handler.handleMessage(f.ws, msg), true); + f.group.members = [1, 3]; + f.group.pair.workerIds = [3]; + assert.equal(f.handler.handleMessage(f.ws, msg), true); + assert.equal(f.closes.length, 1); + assert.equal(f.sent[1].message.ok, false); + assert.match(f.sent[1].message.error, /membership changed|selected Worker changed/); +}); + +test("a Worker socket cannot close its peer", function () { + var f = fixture(); + f.ws._clayActiveSession = 2; + var msg = f.message(); + msg.driverId = 2; + msg.driverOriginId = "origin-worker-a"; + msg.workerId = 3; + msg.workerOriginId = "origin-worker-b"; + msg.generation = 5; + assert.equal(f.handler.handleMessage(f.ws, msg), true); + assert.equal(f.closes.length, 0); + assert.match(f.sent[0].message.error, /configured Driver|roles changed/); +}); + +test("Mate contexts and server-disabled contexts reject the close contract", function () { + var f = fixture(); + f.handler = attachPairWorkerClose({ + sm: { sessions: new Map() }, splitStore: {}, multiWorkerFeature: null, isMate: true, + projectSlug: "project-a", hasSocket: function () { return true; }, + sendTo: function (socket, message) { f.sent.push({ socket: socket, message: message }); }, + closeWorker: function () { f.closes.push(true); }, + }); + assert.equal(f.handler.handleMessage(f.ws, f.message()), true); + assert.equal(f.closes.length, 0); + assert.match(f.sent[f.sent.length - 1].message.error, /unavailable/); +}); + +["false", "throw"].forEach(function (failureMode) { + test("real close transaction leaves task, queue, permission, and peer unchanged on persistence " + failureMode, async function (t) { + var f = await realFixture(t); + var groupBefore = JSON.parse(JSON.stringify(f.store.groups[0])); + var token = f.workerA._pairDelegation; + var tokenBefore = JSON.parse(JSON.stringify(token)); + var queue = f.workerA._pairFollowups; + var queueBefore = JSON.parse(JSON.stringify(queue)); + var registryBefore = JSON.parse(JSON.stringify(f.workerA._pairTaskIds)); + var peerBefore = JSON.parse(JSON.stringify({ provenance: f.workerB.sessionProvenance, + generation: f.workerB._pairGeneration, processing: f.workerB.isProcessing, + delegation: f.workerB._pairDelegation || null, queue: f.workerB._pairFollowups || null, + stopRequested: !!f.workerB.taskStopRequested, historyLength: f.workerB.history.length })); + f.setMode(failureMode); + assert.equal(f.handler.handleMessage(f.ws, f.request), true); + assert.equal(f.sent[f.sent.length - 1].ok, false); + assert.deepEqual(f.store.groups[0], groupBefore); + assert.equal(f.workerA._pairDelegation, token); + assert.deepEqual(JSON.parse(JSON.stringify(token)), tokenBefore); + assert.equal(f.workerA._pairFollowups, queue); + assert.deepEqual(JSON.parse(JSON.stringify(queue)), queueBefore); + assert.deepEqual(JSON.parse(JSON.stringify(f.workerA._pairTaskIds)), registryBefore); + assert.equal(f.pair.workerPermission.pendingCountFor(f.workerA), 1); + assert.equal(f.aborts(), 0); + assert.equal(!!f.workerA.taskStopRequested, false); + assert.deepEqual(JSON.parse(JSON.stringify({ provenance: f.workerB.sessionProvenance, + generation: f.workerB._pairGeneration, processing: f.workerB.isProcessing, + delegation: f.workerB._pairDelegation || null, queue: f.workerB._pairFollowups || null, + stopRequested: !!f.workerB.taskStopRequested, historyLength: f.workerB.history.length })), peerBefore); + assert.equal(f.outbox.get(token.outboxKey).outcome, null); + assert.equal(f.workerStarts(), 1); + }); +}); + +test("real client payload closes only its exact Worker and retains one durable outcome without replay", async function (t) { + var f = await realFixture(t); + var token = f.workerA._pairDelegation; + var peerGroup = f.store.groups[0]; + var peerBefore = JSON.parse(JSON.stringify({ provenance: f.workerB.sessionProvenance, + generation: f.workerB._pairGeneration, processing: f.workerB.isProcessing, + delegation: f.workerB._pairDelegation || null, queue: f.workerB._pairFollowups || null, + stopRequested: !!f.workerB.taskStopRequested, historyLength: f.workerB.history.length })); + assert.equal(f.request.driverOriginId, f.driver.sessionOriginId); + assert.equal(f.request.workerOriginId, f.workerA.sessionOriginId); + assert.equal(f.request.generation, f.workerA.sessionProvenance.generation); + assert.deepEqual(f.request.expectedWorkerIds, [f.workerA.localId, f.workerB.localId]); + assert.equal(f.handler.handleMessage(f.ws, f.request), true); + assert.equal(f.sent[f.sent.length - 1].ok, true); + assert.equal(f.store.groups[0], peerGroup); + assert.deepEqual(f.store.groups[0].pair.workerIds, [f.workerB.localId]); + assert.equal(f.store.groupForMember(f.workerA.localId), null); + assert.equal(f.store.groupForMember(f.workerB.localId), peerGroup); + assert.equal(f.workerB._pairDelegation, undefined); + assert.deepEqual(JSON.parse(JSON.stringify({ provenance: f.workerB.sessionProvenance, + generation: f.workerB._pairGeneration, processing: f.workerB.isProcessing, + delegation: f.workerB._pairDelegation || null, queue: f.workerB._pairFollowups || null, + stopRequested: !!f.workerB.taskStopRequested, historyLength: f.workerB.history.length })), peerBefore); + assert.equal(f.aborts(), 1); + assert.equal(f.pair.workerPermission.pendingCountFor(f.workerA), 0); + await f.permissionPromise; + var record = f.outbox.get(token.outboxKey); + assert.equal(record.outcome.status, "interrupted"); + assert.equal(record.taskId, "task-close-a"); + assert.equal(f.workerStarts(), 1, "the retained result never replays Worker execution"); + var durable = JSON.parse(fs.readFileSync(path.join(f.sm.sessionsDir, "pair-result-outbox.json"), "utf8")); + assert.equal(durable.records[token.outboxKey].outcome.status, "interrupted"); + var reloadedStore = createSplitGroupStore({ sessions: f.sm.sessions, sessionsDir: f.sm.sessionsDir, + usersModule: null, multiWorkerFeature: multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }) }); + assert.deepEqual(reloadedStore.groups[0].pair.workerIds, [f.workerB.localId]); + var reloadedOutbox = attachPairResultOutbox({ storageDir: f.sm.sessionsDir, projectSlug: "project-a" }); + assert.equal(reloadedOutbox.get(token.outboxKey).outcome.status, "interrupted"); +}); + +test("successful idle close does not drain the selected Worker's retained queue", async function (t) { + var f = await realFixture(t); + var token = f.workerA._pairDelegation; + var startsBefore = f.workerStarts(); + f.workerA.isProcessing = false; + assert.equal(f.handler.handleMessage(f.ws, f.request), true); + assert.equal(f.sent[f.sent.length - 1].ok, true); + assert.equal(f.workerStarts(), startsBefore); + assert.equal(f.workerA._pairDelegation, undefined); + assert.equal(f.workerA._pairFollowups.length, 1); + assert.equal(f.workerA._pairFollowups[0].taskId, "task-queued-a"); + assert.equal(f.workerA._pairFollowups[0].status, "queued"); + assert.equal(f.aborts(), 0); + assert.equal(f.outbox.get(token.outboxKey).outcome.status, "completed"); +}); diff --git a/test/session-pair-factory-wiring.test.js b/test/session-pair-factory-wiring.test.js index 942ba9c3..aad751b8 100644 --- a/test/session-pair-factory-wiring.test.js +++ b/test/session-pair-factory-wiring.test.js @@ -410,7 +410,7 @@ test("the wiring passes the whole context, so every dependency resolves", functi var match; while ((match = re.exec(factorySource)) !== null) reads[match[1]] = true; var names = Object.keys(reads).sort(); - assert.deepEqual(names, ["isMate", "sendTo", "splitStore", "sm", "usersModule"].sort(), + assert.deepEqual(names, ["isMate", "multiWorkerFeature", "sendTo", "splitStore", "sm", "usersModule"].sort(), "the factory's exact dependency list"); for (var i = 0; i < names.length; i++) { assert.match(pairSource, new RegExp("ctx\\." + names[i] + "\\b|var \\w+ = ctx\\." + names[i]), diff --git a/test/session-pair-target.test.js b/test/session-pair-target.test.js new file mode 100644 index 00000000..ae160c77 --- /dev/null +++ b/test/session-pair-target.test.js @@ -0,0 +1,95 @@ +var test = require("node:test"); +var assert = require("node:assert/strict"); +var pairTarget = require("../lib/session-pair-target"); +var multiWorkerFeature = require("../lib/multi-worker-feature"); +var FEATURE = multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }); +var mcp = require("../lib/session-pair-mcp-server"); +var attachPairLifecycle = require("../lib/project-pair-lifecycle").attachPairLifecycle; +var attachTurnControl = require("../lib/session-pair-turn-control").attachPairTurnControl; + +function makeSessions() { + return new Map([ + [1, { localId: 1, ownerId: "owner", history: [], isProcessing: false }], + [2, { localId: 2, ownerId: "owner", title: "Worker A", history: [], isProcessing: false }], + [3, { localId: 3, ownerId: "owner", title: "Worker B", history: [], isProcessing: false }], + ]); +} + +function versionedGroup() { + return { id: "v2", members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }; +} + +test("exact target resolution supports addressed reads and gates ambiguous or mutating V2 calls", function () { + var sessions = makeSessions(); + var sm = { sessions: sessions }; + var group = versionedGroup(); + var driver = sessions.get(1); + assert.equal(pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 2 }, { readOnly: true }).partner.localId, 2); + assert.equal(pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 3 }, { readOnly: true }).partner.localId, 3); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, driver, {}, { readOnly: true }); }, /workerId is required/); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 9 }, { readOnly: true }); }, /exact configured Worker/); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 2 }, { mutation: true }); }, /multi-Worker mutations remain gated/); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 2 }); }, /multi-Worker mutations remain gated/); + assert.equal(pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 2 }, { + mutation: true, multiWorkerFeature: FEATURE, + }).workerId, 2); + assert.equal(pairTarget.resolveGroupTarget(sm, group, driver, { workerId: 2 }, { + mutation: true, structural: true, multiWorkerFeature: FEATURE, + }).workerId, 2); + assert.deepEqual(group.members, [1, 2, 3], "rejected targeting does not mutate the group"); + var stale = Object.assign({}, driver); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, stale, { workerId: 2 }, { readOnly: true }); }, /no longer live/); + var otherOwner = Object.assign({}, driver, { ownerId: "other" }); + sessions.set(1, otherOwner); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, group, otherOwner, { workerId: 2 }, { readOnly: true }); }, /access denied/); +}); + +test("legacy and ad-hoc target omission remains compatible", function () { + var sessions = makeSessions(); + var sm = { sessions: sessions }; + var driver = sessions.get(1); + var legacy = { id: "legacy", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + assert.equal(pairTarget.resolveGroupTarget(sm, legacy, driver, {}, { mutation: true }).workerId, 2); + var adhoc = { id: "adhoc", members: [1, 2], pair: null }; + assert.equal(pairTarget.resolveGroupTarget(sm, adhoc, driver, {}, { mutation: true }).workerId, 2); + assert.throws(function () { pairTarget.resolveGroupTarget(sm, adhoc, driver, { workerId: 3 }, { mutation: true }); }, /exact split partner/); +}); + +test("lifecycle status reads the addressed Worker and rejects omitted V2 target", async function () { + var sessions = makeSessions(); + sessions.get(2).model = "worker-a"; + sessions.get(3).model = "worker-b"; + var group = versionedGroup(); + var store = { groupForMember: function () { return group; } }; + var turnControl = attachTurnControl({ sm: { sessions: sessions }, splitStore: store }); + var lifecycle = attachPairLifecycle({ + sm: { sessions: sessions, saveSessionFile: function () {} }, splitStore: store, turnControl: turnControl, + preflightWorkerForDriver: function () {}, sendToPartner: function () { return Promise.resolve({}); }, + createWorkerForDriver: function () { throw new Error("not reached"); }, + }); + var status = lifecycle.toolHandlers(sessions.get(1)).status; + var addressed = JSON.parse((await status({ workerId: 3 })).content[0].text); + assert.equal(addressed.worker.sessionId, 3); + assert.equal(addressed.configuration.model, "worker-b"); + var ambiguous = await status({}); + assert.equal(ambiguous.isError, true); + assert.match(ambiguous.content[0].text, /workerId is required/); + var wrong = await status({ workerId: 99 }); + assert.equal(wrong.isError, true); + assert.match(wrong.content[0].text, /exact configured Worker/); + var blocked = await lifecycle.toolHandlers(sessions.get(1)).replace({ workerId: 2, message: "must not mutate" }); + assert.equal(blocked.isError, true); + assert.match(blocked.content[0].text, /multi-Worker mutations remain gated/); + assert.deepEqual(group.members, [1, 2, 3], "blocked mutation leaves the group unchanged"); +}); + +test("pair MCP schemas forward optional workerId without making it required", function () { + var defs = mcp.getToolDefs({}, { lifecycle: true }); + var names = ["send_to_partner", "read_partner", "message_partner", "partner_status", "replace_partner", "record_partner_evaluation"]; + for (var i = 0; i < names.length; i++) { + var found = defs.find(function (entry) { return entry.name === names[i]; }); + assert.ok(found, names[i] + " is present"); + assert.ok(found.inputSchema.workerId, names[i] + " forwards workerId"); + assert.equal(found.inputSchema.workerId.isOptional(), true, names[i] + " keeps workerId optional"); + } +}); diff --git a/test/session-pair.test.js b/test/session-pair.test.js index 6c97f21e..3778492b 100644 --- a/test/session-pair.test.js +++ b/test/session-pair.test.js @@ -4,7 +4,10 @@ var fs = require("node:fs"); var os = require("node:os"); var path = require("node:path"); var pairModule = require("../lib/project-session-pair"); +var pairTarget = require("../lib/session-pair-target"); +var multiWorkerFeature = require("../lib/multi-worker-feature"); var attachOutbox = require("../lib/project-pair-result-outbox").attachPairResultOutbox; +var attachRecovery = require("../lib/project-pair-result-recovery").attachPairResultRecovery; function parseToolResult(result) { return JSON.parse(result.content[0].text); @@ -24,9 +27,11 @@ function fixture(configured, options) { var driver = { localId: 1, sessionOriginId: "driver-origin", ownerId: null, title: "Planner", vendor: "claude", model: options.driverModel || "claude-fable-5", history: [], isProcessing: false }; var worker = { localId: 2, sessionOriginId: "worker-origin", ownerId: null, title: "Builder", vendor: "codex", history: [], isProcessing: false }; if (options.driverQueryReady) driver.queryInstance = {}; + var workerB = { localId: 3, sessionOriginId: "worker-b-origin", ownerId: null, title: "Reviewer", vendor: "codex", model: "gpt-5.6-terra", history: [], isProcessing: false }; var sessions = new Map([[1, driver], [2, worker]]); - var group = options.ungrouped ? null : { id: "sg_pair", members: [1, 2] }; - if (configured && group) group.pair = { driverId: 1, workerId: 2 }; + if (options.versioned) sessions.set(3, workerB); + var group = options.ungrouped ? null : { id: "sg_pair", members: options.versioned ? [1, 2, 3] : [1, 2] }; + if (configured && group) group.pair = options.versioned ? { version: 2, driverId: 1, workerIds: [2, 3] } : { driverId: 1, workerId: 2 }; var events = []; var starts = []; var driverPushes = []; @@ -49,12 +54,13 @@ function fixture(configured, options) { sendToSession: function (session, message) { if (message.type === "pair_session_created") pairMessages.push(message); }, broadcastSessionList: function () {}, createSessionRaw: function (spec) { - worker.ownerId = spec.ownerId || null; - worker.vendor = spec.vendor; - worker.model = spec.model || null; - worker.effort = spec.effort || null; - sessions.set(worker.localId, worker); - return worker; + var created = options.allocateSecondWorker ? workerB : worker; + created.ownerId = spec.ownerId || null; + created.vendor = spec.vendor; + created.model = spec.model || null; + created.effort = spec.effort || null; + sessions.set(created.localId, created); + return created; }, }; var sdk = { @@ -70,7 +76,7 @@ function fixture(configured, options) { startQuery: function (session, text, images, linuxUser, beforePush, onAccepted) { starts.push({ session: session, text: text }); session._queryGeneration = (session._queryGeneration || 0) + 1; - if (session !== worker) { + if (session !== worker && session !== workerB) { if (options.onDriverStart) options.onDriverStart(session); if (options.driverStartReject) return Promise.reject(new Error("driver start failed")); if (typeof beforePush === "function") { @@ -101,29 +107,51 @@ function fixture(configured, options) { session.history.push({ type: "done", code: 0 }); session._lastTurnInterrupted = true; } else { - session.history.push({ type: "delta", text: "Partner result" }); + session.history.push({ type: "delta", text: session === workerB ? "Partner B result" : "Partner result" }); } session.isProcessing = false; if (options.autoTurnDone !== false && !options.workerInterrupted) attached.handleTurnDone(session); - }, options.workerDelay || 20); + }, session === workerB ? (options.workerBDelay || options.workerDelay || 20) : (options.workerDelay || 20)); return Promise.resolve(); }, }; + var splitStore = { + groupForMember: function (id) { return !group || group.members.indexOf(id) === -1 ? null : group; }, + create: function (ws, msg) { + group = { id: "sg_created", members: msg.members.slice(), pair: msg.pair }; + return { ok: true, group: group }; + }, + dissolve: function (ws, msg) { + if (!group || group.id !== msg.id) return { ok: false, error: "Split group not found" }; + var removed = group; + group = null; + return { ok: true, group: removed }; + }, + removeWorker: function (ownerId, msg, capability) { + if (!multiWorkerFeature.isEnabled(capability) || !group || + group.id !== msg.groupId || group.pair.workerIds.indexOf(msg.workerId) === -1) { + return { ok: false, error: "The selected Worker changed" }; + } + var remaining = group.pair.workerIds.filter(function (id) { return id !== msg.workerId; }); + group.members = [group.pair.driverId].concat(remaining); + group.pair = { version: 2, driverId: group.pair.driverId, workerIds: remaining }; + return { ok: true, group: group }; + }, + addWorker: function (ownerId, msg, capability) { + if (!multiWorkerFeature.isEnabled(capability) || !group || msg.expectedGroup !== group || + group.id !== msg.groupId || !sessions.has(msg.newWorkerId)) return { ok: false, error: "The exact split group changed" }; + var existing = group.pair.workerIds ? group.pair.workerIds.slice() : [group.pair.workerId]; + if (existing.length >= 2 || existing.join(",") !== msg.expectedWorkerIds.join(",")) { + return { ok: false, error: "The configured Worker membership changed" }; + } + group.members = [group.pair.driverId].concat(existing, [msg.newWorkerId]); + group.pair = { version: 2, driverId: group.pair.driverId, workerIds: existing.concat([msg.newWorkerId]) }; + return { ok: true, group: group }; + }, + }; attached = pairModule.attachSessionPair({ sm: sm, - splitStore: { - groupForMember: function (id) { return !group || group.members.indexOf(id) === -1 ? null : group; }, - create: function (ws, msg) { - group = { id: "sg_created", members: msg.members.slice(), pair: msg.pair }; - return { ok: true, group: group }; - }, - dissolve: function (ws, msg) { - if (!group || group.id !== msg.id) return { ok: false, error: "Split group not found" }; - var removed = group; - group = null; - return { ok: true, group: removed }; - }, - }, + splitStore: splitStore, getSdk: function () { sdkLookups++; return options.disableResultSdk && sdkLookups > 1 ? null : sdk; }, send: function (message) { events.push(message); }, sendTo: function () {}, @@ -133,8 +161,11 @@ function fixture(configured, options) { onPartnerResult: function () { partnerResults++; if (options.throwPartnerResult) throw new Error("result hook failed"); }, resultOutbox: options.resultOutbox, acceptanceTimeoutMs: options.acceptanceTimeoutMs, + multiWorkerFeature: options.enableMultiWorkerRuntime ? multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }) : null, }); - return { attached: attached, driver: driver, worker: worker, sessions: sessions, group: group, getGroup: function () { return group; }, events: events, starts: starts, driverPushes: driverPushes, pairMessages: pairMessages, partnerResults: function () { return partnerResults; } }; + return { attached: attached, driver: driver, worker: worker, workerB: workerB, sessions: sessions, sm: sm, splitStore: splitStore, + group: group, getGroup: function () { return group; }, events: events, starts: starts, driverPushes: driverPushes, + pairMessages: pairMessages, partnerResults: function () { return partnerResults; } }; } test("configured pairs expose partner tools only to the Driver", function () { @@ -155,12 +186,322 @@ test("configured pairs expose partner tools only to the Driver", function () { assert.match(f.attached.getSystemPrompt(f.worker), /report_partner_outcome/); }); +test("the real coordinator accepts one add card while the existing Worker remains active", async function () { + var f = fixture(true, { enableMultiWorkerRuntime: true, allocateSecondWorker: true, workerDelay: 20 }); + f.worker.isProcessing = true; + f.worker._pairGeneration = 4; + var propose = f.attached.getToolDefs(f.driver).find(function (item) { return item.name === "propose_worker"; }); + assert.ok(propose); + assert.deepStrictEqual(f.attached.getToolDefs(f.worker).map(function (item) { return item.name; }), ["report_partner_outcome"]); + var posted = parseToolResult(await propose.handler({ + summary: "A separate verification Worker can proceed independently.", + plan: "1. Preserve the active Worker\n2. Delegate separate verification files", + message: "Inspect the independent verification scope and report the result.", + recommendedVendor: "codex", recommendedModel: "gpt-5.6-sol", recommendedEffort: "medium", + recommendationRationale: "The installed Codex runtime fits this isolated verification task.", + })); + assert.equal(posted.status, "posted"); + var accepted = await f.attached.respondToWorkerProposal({ _clayActiveSession: 1 }, { + proposalId: posted.proposalId, accepted: true, vendor: "codex", model: "gpt-5.6-sol", effort: "medium", + }); + assert.equal(accepted.ok, true); + assert.deepEqual(f.getGroup().pair.workerIds, [2, 3]); + assert.equal(f.worker.isProcessing, true, "adding Worker B does not interrupt Worker A"); + assert.equal(f.starts.filter(function (start) { return start.session === f.workerB; }).length, 1); + assert.deepStrictEqual(f.attached.getToolDefs(f.workerB).map(function (item) { return item.name; }), ["report_partner_outcome"]); + assert.match(f.attached.getSystemPrompt(f.driver), /workerId to every targeted/); + await assert.rejects(f.attached.respondToWorkerProposal({ _clayActiveSession: 1 }, { + proposalId: posted.proposalId, accepted: true, + }), /already been resolved/); +}); + test("ad-hoc splits expose partner tools to both sessions", function () { var f = fixture(false); assert.strictEqual(f.attached.getToolDefs(f.driver).length, 4); assert.strictEqual(f.attached.getToolDefs(f.worker).length, 4); }); +test("ad-hoc delegation still completes through blocking wait", async function () { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-adhoc-result-")); + try { + var outbox = attachOutbox({ storageDir: dir }); + var f = fixture(false, { resultOutbox: outbox }); + var send = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "send_to_partner"; }); + var completed = parseToolResult(await send.handler({ message: "Ad-hoc wait", wait: true, timeoutSeconds: 2 })); + assert.equal(completed.status, "complete"); + assert.equal(completed.response, "Partner result"); + assert.equal(f.worker._pairDelegation, undefined); + assert.equal(outbox.list()[0].finalized, true); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("ad-hoc detached delegation uses the fallback callback once", async function () { + var f = fixture(false, { workerDelay: 15 }); + var send = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "send_to_partner"; }); + await send.handler({ message: "Ad-hoc detached", wait: false }); + await new Promise(function (resolve) { setTimeout(resolve, 70); }); + assert.equal(f.driverPushes.length, 1); + assert.match(f.driverPushes[0], /Partner result/); + assert.equal(f.worker._pairDelegation, undefined); +}); + +test("real pair handlers address separate V2 Workers and reject ambiguous targets", async function () { + var f = fixture(true, { versioned: true }); + f.worker.history.push({ type: "user_message", text: "Worker A task" }); + f.workerB.history.push({ type: "user_message", text: "Worker B task" }); + var tools = f.attached.getToolDefs(f.driver); + var read = tools.find(function (tool) { return tool.name === "read_partner"; }); + var status = tools.find(function (tool) { return tool.name === "partner_status"; }); + var a = parseToolResult(await read.handler({ workerId: 2, lastTurns: 1 })); + var b = parseToolResult(await read.handler({ workerId: 3, lastTurns: 1 })); + assert.equal(a.partnerId, 2); + assert.equal(b.partnerId, 3); + assert.equal(a.turns[0].user, "Worker A task"); + assert.equal(b.turns[0].user, "Worker B task"); + var ambiguous = await read.handler({ lastTurns: 0 }); + assert.equal(ambiguous.isError, true); + assert.match(ambiguous.content[0].text, /workerId is required/); + var wrong = await status.handler({ workerId: 99 }); + assert.equal(wrong.isError, true); + assert.equal(f.getGroup().members.join(","), "1,2,3"); +}); + +test("server-gated V2 runtime delegates concurrently and completes each exact Worker independently", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, workerDelay: 15 }); + var send = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "send_to_partner"; }); + var first = parseToolResult(await send.handler({ workerId: 2, taskId: "task-a", message: "Build A", wait: false })); + var second = parseToolResult(await send.handler({ workerId: 3, taskId: "task-b", message: "Build B", wait: false })); + assert.equal(first.partnerId, 2); + assert.equal(second.partnerId, 3); + assert.equal(f.worker._pairDelegation.taskId, "task-a"); + assert.equal(f.workerB._pairDelegation.taskId, "task-b"); + await new Promise(function (resolve) { setTimeout(resolve, 80); }); + assert.equal(f.worker._pairDelegation, undefined); + assert.equal(f.workerB._pairDelegation, undefined); + assert.equal(f.starts.filter(function (entry) { return entry.session === f.worker; }).length, 1); + assert.equal(f.starts.filter(function (entry) { return entry.session === f.workerB; }).length, 1); + assert.equal(f.worker._lastPairOutcome.taskId, "task-a"); + assert.equal(f.workerB._lastPairOutcome.taskId, "task-b"); +}); + +test("durable V2 sender persists and accepts out-of-order results without Worker replay", async function () { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-outbox-")); + try { + var outbox = attachOutbox({ storageDir: dir, projectSlug: "project" }); + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, driverQueryReady: true, + resultOutbox: outbox, workerDelay: 25, workerBDelay: 110 }); + var send = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "send_to_partner"; }); + await send.handler({ workerId: 2, taskId: "durable-a", message: "Durable A" }); + await send.handler({ workerId: 3, taskId: "durable-b", message: "Durable B" }); + await new Promise(function (resolve) { setTimeout(resolve, 55); }); + var early = outbox.list(); + var a = early.find(function (record) { return record.taskId === "durable-a"; }); + var b = early.find(function (record) { return record.taskId === "durable-b"; }); + assert.equal(a.deliveryState, "accepted"); + assert.equal(a.finalized, true); + assert.equal(b.outcome, null); + assert.equal(f.workerB.isProcessing, true, "Worker A delivery is independent of running Worker B"); + await new Promise(function (resolve) { setTimeout(resolve, 100); }); + var records = outbox.list(); + assert.equal(records.length, 2); + assert.equal(records.filter(function (record) { return record.deliveryState === "accepted" && record.finalized; }).length, 2); + assert.notEqual(records[0].workerOriginId, records[1].workerOriginId); + assert.equal(f.starts.filter(function (entry) { return entry.session === f.worker || entry.session === f.workerB; }).length, 2, + "result delivery never redispatches a Worker"); + assert.equal(f.driverPushes.length, 2); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("V2 Stop retains both durable results until an actual human turn", async function () { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-stop-")); + try { + var outbox = attachOutbox({ storageDir: dir }); + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, driverQueryReady: true, + resultOutbox: outbox, workerDelay: 20, workerBDelay: 30 }); + f.worker._pairGeneration = 1; + f.worker.sessionProvenance = { kind: "worker", parentSessionOriginId: f.driver.sessionOriginId, generation: 1 }; + f.workerB._pairGeneration = 2; + f.workerB.sessionProvenance = { kind: "worker", parentSessionOriginId: f.driver.sessionOriginId, generation: 2 }; + var recovery = attachRecovery({ sm: f.sm, store: f.splitStore, outbox: outbox, resultCapture: f.attached.resultCapture, + wake: f.attached.wakePartnerResult, projectSlug: null, blockedReason: f.attached.stopBarrier, + sendTo: function () {}, getClients: function () { return []; }, isMultiUser: function () { return false; } }); + var send = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "send_to_partner"; }); + await send.handler({ workerId: 2, taskId: "stopped-a", message: "Stopped A" }); + await send.handler({ workerId: 3, taskId: "stopped-b", message: "Stopped B" }); + f.attached.handleHumanStop(f.driver); + await new Promise(function (resolve) { setTimeout(resolve, 70); }); + assert.equal(outbox.list().filter(function (record) { return record.state === "blocked" && record.outcome; }).length, 2); + assert.equal(outbox.list().filter(function (record) { return record.finalized; }).length, 0); + assert.equal(f.driverPushes.length, 0); + assert.equal(f.attached.beginHumanTurn(f.driver), true); + recovery.beginHumanTurn(f.driver); + await new Promise(function (resolve) { setImmediate(resolve); }); + assert.equal(outbox.list().filter(function (record) { return record.deliveryState === "accepted" && record.finalized; }).length, 2); + assert.equal(f.driverPushes.length, 2, "each retained result is accepted by the Driver once"); + assert.equal(f.starts.filter(function (entry) { return entry.session === f.worker || entry.session === f.workerB; }).length, 2); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("generation changes fence delayed durable capture and sender acceptance", async function () { + var captureDir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-stale-capture-")); + var acceptanceDir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-stale-acceptance-")); + try { + var calls = 0; + var captureOutbox = attachOutbox({ storageDir: captureDir, persist: function (target, serialized) { + calls++; + if (calls === 2) return false; + atomicPersistForTest(target, serialized); + return true; + } }); + var captureFixture = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, resultOutbox: captureOutbox, workerDelay: 10 }); + captureFixture.worker._pairGeneration = 7; + await captureFixture.attached.getToolDefs(captureFixture.driver)[0].handler({ workerId: 2, taskId: "stale-capture", message: "Capture later" }); + await new Promise(function (resolve) { setTimeout(resolve, 20); }); + captureFixture.worker._pairGeneration = 8; + var newerCapture = { taskId: "newer-capture", workerSessionId: 2, generation: 8 }; + captureFixture.worker._pairDelegation = newerCapture; + await new Promise(function (resolve) { setTimeout(resolve, 90); }); + assert.equal(captureOutbox.list()[0].outcome, null); + assert.equal(captureOutbox.list()[0].finalized, false); + assert.equal(captureFixture.worker._pairDelegation, newerCapture); + + var acceptanceOutbox = attachOutbox({ storageDir: acceptanceDir }); + var acceptanceFixture = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, resultOutbox: acceptanceOutbox, + workerDelay: 10, driverStartAcceptedDelay: 90, acceptanceTimeoutMs: 200 }); + acceptanceFixture.worker._pairGeneration = 11; + await acceptanceFixture.attached.getToolDefs(acceptanceFixture.driver)[0].handler({ workerId: 2, taskId: "stale-acceptance", message: "Accept later" }); + await new Promise(function (resolve) { setTimeout(resolve, 35); }); + acceptanceFixture.worker._pairGeneration = 12; + var newerAcceptance = { taskId: "newer-acceptance", workerSessionId: 2, generation: 12 }; + acceptanceFixture.worker._pairDelegation = newerAcceptance; + await new Promise(function (resolve) { setTimeout(resolve, 100); }); + assert.notEqual(acceptanceOutbox.list()[0].deliveryState, "accepted"); + assert.equal(acceptanceOutbox.list()[0].finalized, false); + assert.equal(acceptanceFixture.worker._pairDelegation, newerAcceptance); + } finally { + fs.rmSync(captureDir, { recursive: true, force: true }); + fs.rmSync(acceptanceDir, { recursive: true, force: true }); + } +}); + +test("out-of-order V2 completion and follow-up queues stay isolated by Worker", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, workerDelay: 90, workerBDelay: 15 }); + var tools = f.attached.getToolDefs(f.driver); + var send = tools.find(function (tool) { return tool.name === "send_to_partner"; }); + var queue = tools.find(function (tool) { return tool.name === "queue_partner_followup"; }); + var inspect = tools.find(function (tool) { return tool.name === "inspect_partner_followups"; }); + await send.handler({ workerId: 2, taskId: "active-a", message: "Active A" }); + await send.handler({ workerId: 3, taskId: "active-b", message: "Active B" }); + await queue.handler({ workerId: 2, taskId: "queued-a", message: "Queued A" }); + await queue.handler({ workerId: 3, taskId: "queued-b", message: "Queued B" }); + var a = parseToolResult(await inspect.handler({ workerId: 2 })); + var b = parseToolResult(await inspect.handler({ workerId: 3 })); + assert.deepEqual(a.queued.map(function (task) { return task.taskId; }), ["queued-a"]); + assert.deepEqual(b.queued.map(function (task) { return task.taskId; }), ["queued-b"]); + await new Promise(function (resolve) { setTimeout(resolve, 35); }); + assert.equal(f.workerB._lastPairOutcome.taskId, "queued-b"); + assert.equal(f.worker._lastPairOutcome, undefined, "Worker B completion does not complete Worker A"); + await new Promise(function (resolve) { setTimeout(resolve, 180); }); + assert.equal(f.worker._lastPairOutcome.taskId, "queued-a"); + assert.equal(f.workerB._lastPairOutcome.taskId, "queued-b"); +}); + +test("V2 Worker permissions bind each request to its exact live Worker", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, driverQueryReady: true }); + var pendingA = f.attached.workerPermission.routeIfWorker(f.worker, { toolName: "Write", input: { path: "a" } }); + var pendingB = f.attached.workerPermission.routeIfWorker(f.workerB, { toolName: "Write", input: { path: "b" } }); + var requests = f.driver.history.filter(function (entry) { return entry.workerPermissionRequest; }); + assert.deepEqual(requests.map(function (entry) { return entry.workerSessionId; }), [2, 3]); + await f.attached.workerPermission.handleDriverResponse({ requestId: requests[0].requestId, decision: "allow" }, f.driver); + await f.attached.workerPermission.handleDriverResponse({ requestId: requests[1].requestId, decision: "deny", reason: "B only" }, f.driver); + assert.deepEqual(await pendingA, { behavior: "allow", updatedInput: { path: "a" } }); + assert.match((await pendingB).message, /B only/); +}); + +test("a stale Worker generation cannot report or capture a newer generation result", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, workerDelay: 15 }); + f.worker._pairGeneration = 4; + var tools = f.attached.getToolDefs(f.driver); + await tools.find(function (tool) { return tool.name === "send_to_partner"; }).handler({ workerId: 2, taskId: "generation-four", message: "Old generation" }); + f.worker._pairGeneration = 5; + var report = f.attached.getToolDefs(f.worker)[0]; + var rejected = parseToolResult(await report.handler({ taskId: "generation-four", outcome: "completed" })); + assert.equal(rejected.status, "rejected"); + assert.match(rejected.reason, /no longer matches/); + await new Promise(function (resolve) { setTimeout(resolve, 40); }); + assert.equal(f.worker._lastPairOutcome, undefined); + assert.equal(f.worker._pairDelegation.taskId, "generation-four", "uncertain stale work stays retained"); +}); + +test("V2 per-Worker interrupt is exact while human Stop blocks and aborts both Workers", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true, workerDelay: 200 }); + var tools = f.attached.getToolDefs(f.driver); + var send = tools.find(function (tool) { return tool.name === "send_to_partner"; }); + var interrupt = tools.find(function (tool) { return tool.name === "interrupt_partner"; }); + var abortedA = 0, abortedB = 0; + f.worker.abortController = { abort: function () { abortedA++; } }; + f.workerB.abortController = { abort: function () { abortedB++; } }; + await send.handler({ workerId: 2, taskId: "stop-a", message: "Build A" }); + await send.handler({ workerId: 3, taskId: "stop-b", message: "Build B" }); + await interrupt.handler({ workerId: 2, reason: "Only A" }); + assert.equal(abortedA, 1); + assert.equal(abortedB, 0); + assert.equal(f.workerB.taskStopRequested, undefined); + assert.equal(f.attached.handleHumanStop(f.driver), true); + assert.equal(abortedA, 2); + assert.equal(abortedB, 1); + var blocked = await send.handler({ workerId: 3, message: "Must wait" }); + assert.equal(blocked.isError, true); + assert.match(blocked.content[0].text, /human stopped/); + assert.equal(f.attached.beginHumanTurn(f.driver), true); +}); + +test("server-gated V2 structural close removes only the selected Worker", async function () { + var f = fixture(true, { versioned: true, enableMultiWorkerRuntime: true }); + var close = f.attached.getToolDefs(f.driver).find(function (tool) { return tool.name === "close_partner"; }); + var result = await close.handler({ workerId: 2 }); + assert.equal(result.isError, undefined); + assert.deepEqual(f.getGroup().members, [1, 3]); + assert.deepEqual(f.getGroup().pair.workerIds, [3]); +}); + +test("task-control schemas parse workerId and handlers preserve exact V2 targets", async function () { + var f = fixture(true, { versioned: true }); + var tools = f.attached.getToolDefs(f.driver); + var names = ["queue_partner_followup", "inspect_partner_followups", "cancel_partner_followup", "replace_partner_task", "resume_partner_task"]; + var z = require("zod"); + for (var i = 0; i < names.length; i++) { + var tool = tools.find(function (entry) { return entry.name === names[i]; }); + assert.ok(tool.inputSchema.workerId, names[i] + " exposes workerId"); + assert.equal(tool.inputSchema.workerId.isOptional(), true); + var sample = { workerId: 3 }; + if (names[i] === "queue_partner_followup") sample.message = "queued"; + if (names[i] === "cancel_partner_followup") sample.taskId = "missing"; + if (names[i] === "replace_partner_task") { sample.targetTaskId = "missing"; sample.message = "replacement"; } + if (names[i] === "resume_partner_task") sample.message = "resume"; + assert.doesNotThrow(function () { z.object(tool.inputSchema).parse(sample); }, names[i] + " accepts an integer workerId"); + } + f.workerB._pairFollowups = [{ taskId: "b-task", status: "queued", message: "B work", workerId: 3 }]; + var inspect = tools.find(function (tool) { return tool.name === "inspect_partner_followups"; }); + var inspected = parseToolResult(await inspect.handler({ workerId: 3 })); + assert.equal(inspected.current, null); + assert.equal(inspected.queued[0].workerSessionId, undefined); + assert.equal(inspected.queued[0].taskId, "b-task"); + var queue = tools.find(function (tool) { return tool.name === "queue_partner_followup"; }); + var blocked = await queue.handler({ workerId: 3, message: "must remain gated" }); + assert.match(blocked.content[0].text, /multi-Worker mutations remain gated/); + assert.equal(f.workerB._pairFollowups.length, 1, "blocked V2 queue does not mutate the selected Worker"); + var wrong = await queue.handler({ workerId: 99, message: "wrong target" }); + assert.match(wrong.content[0].text, /exact configured Worker/); + assert.equal(f.workerB._pairFollowups.length, 1, "wrong target does not mutate any queue"); +}); + test("send_to_partner records attribution and returns the response", async function () { var f = fixture(true); var tool = f.attached.getToolDefs(f.driver)[0]; diff --git a/test/session-provenance.test.js b/test/session-provenance.test.js index 8b29746a..85f85a28 100644 --- a/test/session-provenance.test.js +++ b/test/session-provenance.test.js @@ -106,7 +106,8 @@ test("manager reload reconciles Worker origins before the first evaluation and c var restoredDriver = restored.filter(function (s) { return s.cliSessionId === "origin-driver"; })[0]; var restoredWorker = restored.filter(function (s) { return s.cliSessionId === "origin-worker"; })[0]; assert.notEqual(restoredWorker.localId, worker.localId, "reload reassigned local IDs"); - var group = { id: "reloaded-group", pair: { driverId: restoredDriver.localId, workerId: restoredWorker.localId } }; + var group = { id: "reloaded-group", members: [restoredDriver.localId, restoredWorker.localId], + pair: { driverId: restoredDriver.localId, workerId: restoredWorker.localId } }; var lifecycle = attachPairLifecycle({ sm: second, splitStore: { groupForMember: function (id) { return id === restoredDriver.localId || id === restoredWorker.localId ? group : null; } }, diff --git a/test/session-split-group-roles.test.js b/test/session-split-group-roles.test.js new file mode 100644 index 00000000..0b151564 --- /dev/null +++ b/test/session-split-group-roles.test.js @@ -0,0 +1,161 @@ +var test = require("node:test"); +var assert = require("node:assert"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var roles = require("../lib/session-split-group-roles"); +var anchors = require("../lib/session-split-group-anchors"); + +function sessions() { + return new Map([ + [1, { localId: 1, cliSessionId: "cli-driver", sessionOriginId: "origin-driver" }], + [2, { localId: 2, cliSessionId: "cli-worker-a", sessionOriginId: "origin-worker-a" }], + [3, { localId: 3, cliSessionId: "cli-worker-b", sessionOriginId: "origin-worker-b" }], + ]); +} + +test("role normalization preserves legacy pairs and supports a bounded version 2 shape", function () { + var legacy = roles.normalizePair({ driverId: 1, workerId: 2 }, [1, 2]); + assert.deepStrictEqual(legacy, { ok: true, kind: "legacy", version: null, driverId: 1, workerIds: [2] }); + var versioned = roles.normalizePair({ version: 2, driverId: 1, workerIds: [2, 3] }, [1, 2, 3]); + assert.deepStrictEqual(versioned, { ok: true, kind: "versioned", version: 2, driverId: 1, workerIds: [2, 3] }); + assert.deepStrictEqual(roles.resolveWorkerTarget(legacy, null), { ok: true, workerId: 2 }); + assert.deepStrictEqual(roles.resolveWorkerTarget(versioned, 3), { ok: true, workerId: 3 }); + assert.match(roles.resolveWorkerTarget(versioned, null).error, /workerId is required/); + assert.match(roles.resolveWorkerTarget(versioned, 9).error, /exact configured Worker/); +}); + +test("live target matching preserves ad-hoc pairs and fences Worker generations", function () { + var current = sessions(); + current.get(1).ownerId = "owner"; current.get(2).ownerId = "owner"; current.get(3).ownerId = "owner"; + var sm = { sessions: current }; + var adhoc = { id: "adhoc", members: [1, 2] }; + assert.equal(roles.matchesLiveTarget(sm, adhoc, current.get(1), current.get(2)), true); + assert.equal(roles.matchesLiveTarget(sm, adhoc, current.get(2), current.get(1)), true); + current.get(2)._pairGeneration = 7; + assert.equal(roles.tokenMatchesWorker(current.get(2), { workerSessionId: 2, generation: 7 }), true); + assert.equal(roles.tokenMatchesWorker(current.get(2), { workerSessionId: 2, generation: 6 }), false); + current.get(2).ownerId = "other"; + assert.equal(roles.matchesLiveTarget(sm, adhoc, current.get(1), current.get(2)), false); +}); + +test("role normalization rejects conflicting, incomplete, duplicate, extra, and unknown forms", function () { + var invalid = [ + [{ version: 1, driverId: 1, workerId: 2 }, [1, 2]], + [{ version: 2, driverId: 1, workerId: 2, workerIds: [2] }, [1, 2]], + [{ version: 2, driverId: 1 }, [1, 2]], + [{ version: 2, driverId: 1, workerIds: [] }, [1, 2]], + [{ version: 2, driverId: 1, workerIds: [2, 2] }, [1, 2, 3]], + [{ version: 2, driverId: 1, workerIds: [2] }, [1, 2, 3]], + [{ driverId: 1, workerId: 2 }, [1, 2, 3]], + ]; + for (var i = 0; i < invalid.length; i++) assert.strictEqual(roles.normalizePair(invalid[i][0], invalid[i][1]).ok, false); + assert.match(roles.validateStorePair({ version: 2, driverId: 1, workerIds: [2, 3] }, [1, 2, 3]).error, /reserved/); + assert.match(roles.normalizePair(null, [1, 2, 3]).error, /exactly two/); +}); + +test("versioned anchors resolve roles by stable CLI and origin identities after reorder and renumber", function () { + var current = sessions(); + var group = { members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }; + var memberAnchors = anchors.memberAnchors(group, current); + var pairAnchors = anchors.pairAnchors(group, current); + assert.deepStrictEqual(anchors.resolveVersionedMembers(new Map([ + [11, { localId: 11, cliSessionId: "cli-worker-b", sessionOriginId: "origin-worker-b" }], + [12, { localId: 12, cliSessionId: "cli-driver", sessionOriginId: "origin-driver" }], + [13, { localId: 13, cliSessionId: "cli-worker-a", sessionOriginId: "origin-worker-a" }], + ]), memberAnchors).map(function (session) { return session.sessionOriginId; }), + ["origin-driver", "origin-worker-a", "origin-worker-b"]); + var resolved = anchors.resolveVersionedPair(new Map([ + [11, { localId: 11, cliSessionId: "cli-worker-b", sessionOriginId: "origin-worker-b" }], + [12, { localId: 12, cliSessionId: "cli-driver", sessionOriginId: "origin-driver" }], + [13, { localId: 13, cliSessionId: "cli-worker-a", sessionOriginId: "origin-worker-a" }], + ]), pairAnchors); + assert.strictEqual(resolved.driver.localId, 12); + assert.deepStrictEqual(resolved.workers.map(function (session) { return session.localId; }), [13, 11]); + var reordered = { members: [3, 1, 2], pair: group.pair }; + assert.deepStrictEqual(anchors.versionedMemberAnchors(reordered, current).members.map(function (anchor) { return anchor.origin; }), + ["origin-worker-b", "origin-driver", "origin-worker-a"]); + assert.strictEqual(anchors.resolveVersionedMembers(new Map([[1, { localId: 1 }]]), memberAnchors), null, + "versioned anchors never fall back to local ids"); +}); + +test("anchor boundaries reject malformed roles, invalid counts, disagreement, ambiguity, and missing anchors", function () { + var current = sessions(); + var valid = { members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }; + assert.ok(anchors.memberAnchors(valid, current)); + assert.ok(anchors.pairAnchors(valid, current)); + var invalidGroups = [ + { members: [1, 2, 3], pair: { version: 3, driverId: 1, workerIds: [2, 3] } }, + { members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 2] } }, + { members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3, 4] } }, + { members: [1, 2], pair: { version: 2, driverId: 1, workerIds: [1] } }, + ]; + for (var i = 0; i < invalidGroups.length; i++) { + assert.strictEqual(anchors.memberAnchors(invalidGroups[i], current), null); + assert.strictEqual(anchors.pairAnchors(invalidGroups[i], current), null); + } + var incomplete = new Map(current); + incomplete.delete(3); + assert.strictEqual(anchors.memberAnchors(valid, incomplete), null); + assert.strictEqual(anchors.pairAnchors(valid, incomplete), null); + assert.strictEqual(anchors.findSession(current, "cli-driver", "origin-worker-a"), null); + var ambiguous = new Map([ + [4, { localId: 4, cliSessionId: "same-cli", sessionOriginId: "origin-4" }], + [5, { localId: 5, cliSessionId: "same-cli", sessionOriginId: "origin-5" }], + ]); + assert.strictEqual(anchors.findSession(ambiguous, "same-cli", null), null); + assert.strictEqual(anchors.resolveVersionedPair(ambiguous, { + version: 2, driver: { cli: "same-cli", origin: null }, workers: [{ cli: "originless", origin: "origin-4" }], + }), null); + assert.strictEqual(anchors.resolveVersionedPair(current, { + version: 2, driver: { cli: "cli-driver", origin: "origin-driver" }, workers: [], + }), null); +}); + +test("legacy anchor arrays remain position-compatible and resolve exact identities", function () { + var current = sessions(); + var group = { members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + assert.deepStrictEqual(anchors.memberAnchors(group, current), { + cli: ["cli-driver", "cli-worker-a"], origin: ["origin-driver", "origin-worker-a"], + }); + assert.deepStrictEqual(anchors.pairAnchors(group, current), { + cli: ["cli-driver", "cli-worker-a"], origin: ["origin-driver", "origin-worker-a"], + }); + assert.strictEqual(anchors.findSession(current, "cli-worker-a", "origin-worker-a").localId, 2); +}); + +test("the production split store rejects versioned multi-member admission", function () { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-role-gate-")); + try { + var current = sessions(); + var store = require("../lib/session-split-groups").createSplitGroupStore({ sessions: current, sessionsDir: dir, usersModule: null }); + assert.strictEqual(store.create(null, { members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }).ok, false); + fs.writeFileSync(path.join(dir, "split-groups.json"), JSON.stringify([{ id: "v2", members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }])); + var reloaded = require("../lib/session-split-groups").createSplitGroupStore({ sessions: current, sessionsDir: dir, usersModule: null }); + assert.deepStrictEqual(reloaded.groups, []); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("loading and mutating preserves unsupported future-format records on disk", function () { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-role-preserve-")); + try { + var current = sessions(); + var future = { + id: "future-format", members: [1, 2, 3], + pair: { version: 2, driverId: 1, workerIds: [2, 3] }, + futureField: { preserved: true }, + }; + var file = path.join(dir, "split-groups.json"); + fs.writeFileSync(file, JSON.stringify([future], null, 2) + "\n"); + var store = require("../lib/session-split-groups").createSplitGroupStore({ sessions: current, sessionsDir: dir, usersModule: null }); + assert.deepStrictEqual(store.groups, []); + assert.deepStrictEqual(JSON.parse(fs.readFileSync(file, "utf8")), [future]); + assert.strictEqual(store.create(null, { members: [1, 2] }).ok, true); + var persisted = JSON.parse(fs.readFileSync(file, "utf8")); + assert.deepStrictEqual(persisted.find(function (group) { return group.id === future.id; }), future); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/session-split-group-v2-store.test.js b/test/session-split-group-v2-store.test.js new file mode 100644 index 00000000..3466f0e9 --- /dev/null +++ b/test/session-split-group-v2-store.test.js @@ -0,0 +1,283 @@ +var test = require("node:test"); +var assert = require("node:assert"); +var fs = require("fs"); +var os = require("os"); +var path = require("path"); +var createStore = require("../lib/session-split-groups").createSplitGroupStore; +var multiWorkerFeature = require("../lib/multi-worker-feature"); +var attachFactory = require("../lib/session-pair-factory").attachPairFactory; +var attachLifecycle = require("../lib/project-pair-lifecycle").attachPairLifecycle; +var attachTurnControl = require("../lib/session-pair-turn-control").attachPairTurnControl; +var structuralClose = require("../lib/project-pair-structural-close"); + +var CAPABILITY = multiWorkerFeature.fromServerConfig({ multiWorkerRuntimeEnabled: true }); + +function session(id, origin, generation) { + return { localId: id, title: origin, ownerId: null, cliSessionId: "cli-" + origin, + sessionOriginId: "origin-" + origin, _pairGeneration: generation || null }; +} + +function fixture(t, persistGroups) { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-store-")); + t.after(function () { fs.rmSync(dir, { recursive: true, force: true }); }); + var sessions = new Map([[1, session(1, "driver")], [2, session(2, "a", 4)], [3, session(3, "b", 5)], [4, session(4, "c", 6)]]); + var store = createStore({ sessions: sessions, sessionsDir: dir, usersModule: null, + multiWorkerFeature: CAPABILITY, persistGroups: persistGroups }); + var created = store.create(null, { members: [1, 2], pair: { driverId: 1, workerId: 2 } }); + assert.equal(created.ok, true); + return { dir: dir, sessions: sessions, store: store, group: created.group }; +} + +function identity(f, workerId, newWorkerId) { + var worker = f.sessions.get(workerId); + var added = f.sessions.get(newWorkerId); + return { groupId: f.group.id, driverId: 1, driverOriginId: "origin-driver", + workerId: workerId, workerOriginId: worker.sessionOriginId, generation: worker._pairGeneration, + newWorkerId: newWorkerId, newWorkerOriginId: added.sessionOriginId }; +} + +test("server-gated add persists V2 anchors and reloads renumbered exact identities", function (t) { + var f = fixture(t); + var added = f.store.addWorker(null, identity(f, 2, 3), CAPABILITY); + assert.equal(added.ok, true); + assert.deepEqual(f.group.pair, { version: 2, driverId: 1, workerIds: [2, 3] }); + assert.equal(f.group.memberAnchors.version, 2); + assert.equal(f.store.addWorker(null, identity(f, 2, 4), CAPABILITY).ok, false, "two Workers is the hard maximum"); + + var renumbered = new Map([[11, session(11, "b", 5)], [12, session(12, "driver")], [13, session(13, "a", 4)]]); + var reloaded = createStore({ sessions: renumbered, sessionsDir: f.dir, usersModule: null, multiWorkerFeature: CAPABILITY }); + assert.deepEqual(reloaded.groups[0].members, [12, 13, 11]); + assert.deepEqual(reloaded.groups[0].pair, { version: 2, driverId: 12, workerIds: [13, 11] }); +}); + +test("malformed stored records are pruned without startup crashes while future formats remain intact", function (t) { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-malformed-")); + t.after(function () { fs.rmSync(dir, { recursive: true, force: true }); }); + var future = { id: "future", members: [1, 2], pair: { version: 99, driverId: 1, workerIds: [2] }, + diagnostic: { keep: true } }; + fs.writeFileSync(path.join(dir, "split-groups.json"), JSON.stringify([ + null, 7, {}, { id: "missing-members" }, + { id: "invalid-pair", members: [1, 2], pair: { driverId: 1, workerId: 1 } }, future, + ], null, 2) + "\n"); + var sessions = new Map([[1, session(1, "driver")], [2, session(2, "a")]]); + var store; + assert.doesNotThrow(function () { + store = createStore({ sessions: sessions, sessionsDir: dir, usersModule: null }); + }); + assert.deepEqual(store.groups, []); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, "split-groups.json"), "utf8")), [future]); + assert.equal(store.create(null, { members: [1, 2], pair: { driverId: 1, workerId: 2 } }).ok, true); +}); + +test("inconsistent V2 role anchors stay diagnostic while a valid neighboring group reloads and rewrites", function (t) { + var dir = fs.mkdtempSync(path.join(os.tmpdir(), "clay-v2-inconsistent-")); + t.after(function () { fs.rmSync(dir, { recursive: true, force: true }); }); + var original = new Map([[1, session(1, "driver")], [2, session(2, "a")], [3, session(3, "b")], + [5, session(5, "neighbor-a")], [6, session(6, "neighbor-b")]]); + var initial = createStore({ sessions: original, sessionsDir: dir, usersModule: null, multiWorkerFeature: CAPABILITY }); + var versioned = initial.create(null, { members: [1, 2], pair: { driverId: 1, workerId: 2 } }).group; + assert.equal(initial.addWorker(null, { groupId: versioned.id, driverId: 1, driverOriginId: "origin-driver", + newWorkerId: 3, newWorkerOriginId: "origin-b" }, CAPABILITY).ok, true); + var neighbor = initial.create(null, { members: [5, 6], pair: { driverId: 5, workerId: 6 } }).group; + var file = path.join(dir, "split-groups.json"); + var stored = JSON.parse(fs.readFileSync(file, "utf8")); + var corrupt = stored.find(function (group) { return group.id === versioned.id; }); + corrupt.pairAnchors.workers = corrupt.pairAnchors.workers.slice(0, 1); + var diagnostic = JSON.parse(JSON.stringify(corrupt)); + fs.writeFileSync(file, JSON.stringify(stored, null, 2) + "\n"); + + var renumbered = new Map([[1, session(1, "driver")], [2, session(2, "a")], [3, session(3, "b")], + [50, session(50, "neighbor-a")], [60, session(60, "neighbor-b")]]); + var reloaded; + assert.doesNotThrow(function () { + reloaded = createStore({ sessions: renumbered, sessionsDir: dir, usersModule: null, multiWorkerFeature: CAPABILITY }); + }); + assert.equal(reloaded.groups.length, 1); + assert.equal(reloaded.groups[0].id, neighbor.id); + assert.deepEqual(reloaded.groups[0].members, [50, 60]); + var rewritten = JSON.parse(fs.readFileSync(file, "utf8")); + assert.deepEqual(rewritten.find(function (group) { return group.id === versioned.id; }), diagnostic); + assert.deepEqual(rewritten.find(function (group) { return group.id === neighbor.id; }).members, [50, 60]); +}); + +test("replace and remove affect only the selected Worker and reject stale identity or generation", function (t) { + var f = fixture(t); + assert.equal(f.store.addWorker(null, identity(f, 2, 3), CAPABILITY).ok, true); + var stale = identity(f, 2, 4); + stale.generation = 3; + assert.match(f.store.replaceWorker(null, stale, CAPABILITY).error, /identity or generation changed/); + assert.equal(f.store.replaceWorker(null, identity(f, 2, 4), CAPABILITY).ok, true); + assert.deepEqual(f.group.pair.workerIds, [4, 3]); + assert.equal(f.store.groupForMember(3), f.group, "the other Worker remains attached"); + assert.equal(f.store.groupForMember(2), null, "the replaced Worker is detached"); + var remove = identity(f, 3, 2); + delete remove.newWorkerId; delete remove.newWorkerOriginId; + assert.equal(f.store.removeWorker(null, remove, CAPABILITY).ok, true); + assert.deepEqual(f.group.pair, { version: 2, driverId: 1, workerIds: [4] }); +}); + +test("false and throwing persistence roll back memberships and anchors without disturbing the other Worker", function (t) { + var mode = "ok"; + var f = fixture(t, function () { + if (mode === "false") return false; + if (mode === "throw") throw new Error("disk unavailable"); + return true; + }); + var before = JSON.stringify(f.group); + mode = "false"; + assert.equal(f.store.addWorker(null, identity(f, 2, 3), CAPABILITY).ok, false); + assert.equal(JSON.stringify(f.group), before); + assert.equal(f.store.groupForMember(3), null); + mode = "throw"; + assert.equal(f.store.addWorker(null, identity(f, 2, 3), CAPABILITY).ok, false); + assert.equal(JSON.stringify(f.group), before); +}); + +test("production admission stays off and scheduled Driver transfer explicitly rejects V2", function (t) { + var f = fixture(t); + assert.equal(f.store.addWorker(null, identity(f, 2, 3), CAPABILITY).ok, true); + assert.match(f.store.beginOwnedDriverTransfer(null, { id: f.group.id, sourceDriverId: 1, targetDriverId: 4, workerId: 2 }).error, + /Version 2 Driver transfer is not enabled/); + var denied = f.store.replaceWorker(null, identity(f, 2, 4), null); + assert.match(denied.error, /server multi-Worker feature/); +}); + +test("the real factory adds and replaces one selected Worker without changing the other", function (t) { + var f = fixture(t); + var driver = f.sessions.get(1); + driver.vendor = "claude"; driver.model = "claude-model"; driver.history = []; + var nextId = 10; + var sm = { + sessions: f.sessions, + installedVendors: ["claude", "codex"], + modelsByVendor: { claude: [{ value: "claude-model" }], codex: [{ value: "codex-model" }] }, + createSessionRaw: function (spec) { + var created = session(nextId++, "created-" + nextId); + created.ownerId = spec.ownerId || null; created.vendor = spec.vendor; created.model = spec.model; created.effort = spec.effort; + f.sessions.set(created.localId, created); + return created; + }, + deleteSessionQuiet: function (id) { f.sessions.delete(id); }, + saveSessionFile: function () {}, broadcastSessionList: function () {}, sendToSession: function () {}, + }; + var factory = attachFactory({ sm: sm, splitStore: f.store, isMate: false, + usersModule: { isMultiUser: function () { return false; } }, sendTo: function () {}, multiWorkerFeature: CAPABILITY }); + var added = factory.addWorkerForDriver(driver, { workerVendor: "codex", workerModel: "codex-model" }); + added.worker._pairGeneration = 7; + assert.deepEqual(f.group.pair.workerIds, [2, added.worker.localId]); + var replaced = factory.replaceWorkerForDriver(driver, f.sessions.get(2), { + workerVendor: "codex", workerModel: "codex-model", + }); + assert.deepEqual(f.group.pair.workerIds, [replaced.worker.localId, added.worker.localId]); + assert.equal(f.store.groupForMember(added.worker.localId), f.group); + assert.equal(f.sessions.has(2), true, "replaced history remains available"); +}); + +test("lifecycle replacement and close route through the selected Worker while its peer stays attached", async function (t) { + var f = fixture(t); + var driver = f.sessions.get(1); + var workerA = f.sessions.get(2); + driver.vendor = "claude"; driver.model = "claude-model"; driver.history = []; + workerA.vendor = "codex"; workerA.model = "codex-model"; workerA.history = []; + var nextId = 20; + var sm = { sessions: f.sessions, installedVendors: ["claude", "codex"], + modelsByVendor: { claude: [{ value: "claude-model" }], codex: [{ value: "codex-model" }] }, + createSessionRaw: function (spec) { + var created = session(nextId++, "lifecycle-" + nextId); + created.ownerId = spec.ownerId || null; created.vendor = spec.vendor; created.model = spec.model; + created.effort = spec.effort; created.history = []; f.sessions.set(created.localId, created); return created; + }, + deleteSessionQuiet: function (id) { f.sessions.delete(id); }, saveSessionFile: function () {}, + broadcastSessionList: function () {}, sendToSession: function () {}, + }; + var factory = attachFactory({ sm: sm, splitStore: f.store, isMate: false, + usersModule: { isMultiUser: function () { return false; } }, sendTo: function () {}, multiWorkerFeature: CAPABILITY }); + var workerB = factory.addWorkerForDriver(driver, { workerVendor: "codex", workerModel: "codex-model" }).worker; + var turnControl = attachTurnControl({ sm: sm, splitStore: f.store }); + var lifecycle = attachLifecycle({ sm: sm, splitStore: f.store, turnControl: turnControl, + multiWorkerFeature: CAPABILITY, + preflightWorkerForDriver: factory.preflightWorkerForDriver, + replaceWorkerForDriver: factory.replaceWorkerForDriver, + createWorkerForDriver: factory.createWorkerForDriver, + cancelWorkerPermissions: function () {}, markInterruption: function () {}, + sendToPartner: function () { throw new Error("not reached"); }, + }); + lifecycle.recordGenerationStart(driver, workerA); + lifecycle.recordGenerationStart(driver, workerB); + var replaced = await lifecycle.replacePartner({ workerId: workerA.localId, workerVendor: "codex", + workerModel: "codex-model", transactionId: "replace-worker-a" }, driver); + var replacement = f.sessions.get(replaced.workerSessionId); + assert.deepEqual(f.group.pair.workerIds, [replacement.localId, workerB.localId]); + assert.equal(f.store.groupForMember(workerB.localId), f.group); + var closed = structuralClose.removeSelectedWorker(f.store, f.group, driver, replacement, CAPABILITY); + assert.equal(closed.ok, true); + assert.deepEqual(f.group.pair.workerIds, [workerB.localId]); + assert.equal(f.store.groupForMember(workerB.localId), f.group); +}); + +test("factory cleanup removes a newly allocated Worker when the structural transaction is rejected", function () { + var driver = session(1, "driver"); + driver.vendor = "claude"; driver.model = "claude-model"; driver.history = []; + var sessions = new Map([[1, driver]]); + var group = { id: "pair", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + var nextId = 9; + var sm = { sessions: sessions, installedVendors: ["claude", "codex"], + modelsByVendor: { codex: [{ value: "codex-model" }] }, + createSessionRaw: function (spec) { + var created = session(nextId++, "orphan-candidate"); created.vendor = spec.vendor; + sessions.set(created.localId, created); return created; + }, + deleteSessionQuiet: function (id) { sessions.delete(id); }, saveSessionFile: function () {}, + broadcastSessionList: function () {}, sendToSession: function () {}, + }; + var factory = attachFactory({ sm: sm, isMate: false, multiWorkerFeature: CAPABILITY, + usersModule: { isMultiUser: function () { return false; } }, sendTo: function () {}, + splitStore: { groupForMember: function () { return group; }, addWorker: function () { + return { ok: false, error: "stale group" }; + } }, + }); + assert.throws(function () { + factory.addWorkerForDriver(driver, { workerVendor: "codex", workerModel: "codex-model" }); + }, /stale group/); + assert.deepEqual(Array.from(sessions.keys()), [1]); +}); + +test("factory persistence false or throw prevents structural mutation and cleans up the candidate", function () { + ["driver-false", "worker-false", "worker-throw"].forEach(function (mode) { + var driver = session(1, "driver"); + driver.vendor = "claude"; driver.model = "claude-model"; driver.history = []; + var peer = session(2, "peer", 1); + var sessions = new Map([[1, driver], [2, peer]]); + var group = { id: "pair-" + mode, ownerId: null, members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + var before = JSON.stringify(group); + var storeMutations = 0; + var sm = { sessions: sessions, installedVendors: ["claude", "codex"], + modelsByVendor: { codex: [{ value: "codex-model" }] }, + createSessionRaw: function (spec) { + var created = session(9, "candidate"); created.vendor = spec.vendor; created.history = []; + sessions.set(created.localId, created); return created; + }, + deleteSessionQuiet: function (id) { sessions.delete(id); }, + saveSessionFile: function (target) { + if (mode === "driver-false" && target === driver) return false; + if (target !== driver && mode === "worker-false") return false; + if (target !== driver && mode === "worker-throw") throw new Error("session disk unavailable"); + return true; + }, + broadcastSessionList: function () {}, sendToSession: function () {}, + }; + var factory = attachFactory({ sm: sm, isMate: false, multiWorkerFeature: CAPABILITY, + usersModule: { isMultiUser: function () { return false; } }, sendTo: function () {}, + splitStore: { groupForMember: function () { return group; }, addWorker: function () { + storeMutations++; + return { ok: true, group: group }; + } }, + }); + assert.throws(function () { + factory.addWorkerForDriver(driver, { workerVendor: "codex", workerModel: "codex-model" }); + }, /persistence returned false|session disk unavailable/); + assert.equal(storeMutations, 0, mode + " must fail before group mutation"); + assert.equal(JSON.stringify(group), before); + assert.deepEqual(Array.from(sessions.keys()), [1, 2]); + }); +}); diff --git a/test/split-boundary-regression.test.js b/test/split-boundary-regression.test.js index 0874702d..eda32dd3 100644 --- a/test/split-boundary-regression.test.js +++ b/test/split-boundary-regression.test.js @@ -55,6 +55,7 @@ function buildHarness() { } load("store.js"); + load("split-group-helpers.js"); load("project-activation.js"); load("split-session-boundary.js"); load("app-projects.js"); diff --git a/test/split-groups.test.js b/test/split-groups.test.js index 8739fe3d..d7539abd 100644 --- a/test/split-groups.test.js +++ b/test/split-groups.test.js @@ -527,12 +527,12 @@ test("pane connections receive pair roles for Worker chrome enforcement", functi assert.deepStrictEqual(sent[0].message.groups[0].pair, { driverId: 1, workerId: 2 }); }); -test("client helpers derive grouped ids and preserve stored member order", async function () { +test("client helpers derive grouped ids and match group membership independent of stored order", async function () { var helpers = await loadClientHelpers(); var groups = [{ id: "a", members: [2, 1] }, { id: "b", members: [3, 4] }]; assert.deepStrictEqual(Array.from(helpers.groupedSessionIds(groups)), [2, 1, 3, 4]); assert.strictEqual(helpers.findSplitGroup(groups, [2, 1]).id, "a"); - assert.strictEqual(helpers.findSplitGroup(groups, [1, 2]), null); + assert.strictEqual(helpers.findSplitGroup(groups, [1, 2]).id, "a"); assert.strictEqual(helpers.isConfiguredWorker([ { id: "pair", members: [1, 2], pair: { driverId: 1, workerId: 2 } }, ], 2), true); diff --git a/test/split-view.test.js b/test/split-view.test.js index 64892c81..82f360db 100644 --- a/test/split-view.test.js +++ b/test/split-view.test.js @@ -6,6 +6,7 @@ var path = require("path"); var paneHelperPromise = null; var projectActivationPromise = null; +var splitGroupPromise = null; function loadPaneHelpers() { if (!paneHelperPromise) { var file = path.join(__dirname, "../lib/public/modules/pane-session.js"); @@ -24,6 +25,64 @@ function loadProjectActivationHelpers() { return projectActivationPromise; } +function loadSplitGroupHelpers() { + if (!splitGroupPromise) { + var file = path.join(__dirname, "../lib/public/modules/split-group-helpers.js"); + splitGroupPromise = import("data:text/javascript;base64," + Buffer.from(fs.readFileSync(file, "utf8")).toString("base64")); + } + return splitGroupPromise; +} + +test("split group projection follows explicit legacy and v2 roles, not member order", async function () { + var helpers = await loadSplitGroupHelpers(); + var legacy = { members: [22, 11], pair: { driverId: 11, workerId: 22 } }; + assert.deepStrictEqual(helpers.splitGroupMemberIds(legacy), [11, 22]); + assert.equal(helpers.isConfiguredWorker([legacy], 22), true); + assert.equal(helpers.isConfiguredWorker([legacy], 11), false); + var v2 = { members: [33, 11, 22], pair: { version: 2, driverId: 11, workerIds: [22, 33] } }; + assert.deepStrictEqual(helpers.splitGroupMemberIds(v2), [11, 22, 33]); + assert.deepStrictEqual(helpers.splitGroupRoles(v2).workerIds, [22, 33]); + assert.equal(helpers.findSplitGroup([v2], [33, 11, 22]), v2); + assert.equal(helpers.splitGroupActiveAnchor(v2, 33), 11); + assert.equal(helpers.splitGroupActiveAnchor({ members: [44, 55] }, 55), 55); + assert.equal(helpers.splitGroupActiveAnchor({ members: [44, 55] }, 66), 44); + var splitSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); + assert.match(splitSource, /var anchorId = splitGroupActiveAnchor\(group, activeId\);/); + assert.match(splitSource, /switch_session", id: anchorId/); +}); + +test("targeted Worker close payload carries stable Driver, group, Worker, and generation identity", async function () { + var helpers = await loadSplitGroupHelpers(); + var group = { + id: "group-a", + members: [11, 22, 33], + pair: { version: 2, driverId: 11, workerIds: [22, 33] }, + pairAnchors: { + version: 2, + driver: { origin: "origin-driver" }, + workers: [{ origin: "origin-a" }, { origin: "origin-b" }], + }, + }; + var request = helpers.splitWorkerCloseRequest(group, [ + { id: 11, workerGeneration: null }, + { id: 22, workerGeneration: 4 }, + { id: 33, workerGeneration: 5 }, + ], "project-a", 33, "request-a"); + assert.deepStrictEqual(request, { + type: "split_worker_close", + requestId: "request-a", + projectSlug: "project-a", + groupId: "group-a", + driverId: 11, + driverOriginId: "origin-driver", + workerId: 33, + workerOriginId: "origin-b", + generation: 5, + expectedWorkerIds: [22, 33], + }); + assert.equal(helpers.splitWorkerCloseRequest(group, [{ id: 33, workerGeneration: null }], "project-a", 33, "missing"), null); +}); + test("split session selection is scoped to the hydrated project socket", async function () { var helpers = await loadProjectActivationHelpers(); var socket = { readyState: 1 }; @@ -215,33 +274,47 @@ test("configured Split Workers preserve direct human messaging and stopping", fu assert.doesNotMatch(paneCss, /worker-controlled/); }); -test("dissolving a pair closes the split UI back to the Driver session", function () { +test("membership updates reconcile the split instead of collapsing it", function () { + var splitSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); + + assert.match(splitSource, /var projectedPanes = projectedIds\.map/); + assert.match(splitSource, /store\.set\(\{ splitPanes: \{ groupId: split\.groupId, panes: projectedPanes \} \}\)/); + assert.doesNotMatch(splitSource, /!hasCurrentSplitGroup\(split, state\.splitGroups\)/); +}); + +test("stacked Worker close uses the targeted backend contract instead of disabling the control", function () { var splitSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); + var rendererSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-pane-renderer.js"), "utf8"); + assert.match(splitSource, /splitWorkerCloseRequest/); + assert.match(splitSource, /JSON\.stringify\(request\)/); + assert.doesNotMatch(rendererSource, /Worker removal requires backend integration/); +}); - assert.match(splitSource, /if \(!hasCurrentSplitGroup\(split, state\.splitGroups\)\) switchNativeSession\(split\.panes\[0\]\.sessionId, state\.currentSlug\)/); +test("split pane reconciliation is keyed and does not rebuild existing pane nodes", function () { + var reconcilerSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-pane-reconciler.js"), "utf8"); + assert.match(reconcilerSource, /splitPaneIdentity/); + assert.match(reconcilerSource, /byKey\.get\(key\)/); + assert.doesNotMatch(reconcilerSource, /host\.innerHTML/); + assert.doesNotMatch(reconcilerSource, /moveBefore|insertBefore/); }); test("split pane permission control is anchored beside the session title", function () { var splitSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); + var rendererSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-pane-renderer.js"), "utf8"); var paneCss = fs.readFileSync(path.join(__dirname, "../lib/public/css/pane.css"), "utf8"); - assert.match(splitSource, /header\.insertBefore\(fullAccess, ctxChip\)/); + assert.match(rendererSource, /header\.insertBefore\(fullAccess, ctxChip\)/); assert.match(paneCss, /\.split-pane-title\s*\{[^}]*flex:\s*0 1 auto/s); assert.match(paneCss, /\.split-pane-context\s*\{[^}]*margin-left:\s*auto/s); }); test("configured Split Workers never show the outer Skip Permissions control", function () { var splitSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-view.js"), "utf8"); - var stateStart = splitSource.indexOf("function panePermissionControlState"); - var stateEnd = splitSource.indexOf("function updatePaneFullAccessButton", stateStart); - var stateSource = splitSource.slice(stateStart, stateEnd); - var updateStart = stateEnd; - var updateEnd = splitSource.indexOf("function createPane", updateStart); - var updateSource = splitSource.slice(updateStart, updateEnd); - - assert.match(stateSource, /var worker = !!session && isConfiguredWorker\(store\.get\('splitGroups'\), session\.id\)/); - assert.match(stateSource, /effectivePermissionMode: session && session\.effectivePermissionMode \|\| null/); - assert.match(stateSource, /visible: !!session && !worker && mode === "gui"/); - assert.match(stateSource, /locked: worker/); - assert.match(updateSource, /renderPermissionControl\(button, panePermissionControlState\(session\)\)/); + var rendererSource = fs.readFileSync(path.join(__dirname, "../lib/public/modules/split-pane-renderer.js"), "utf8"); + + assert.match(rendererSource, /var worker = !!session && isConfiguredWorker\(store\.get\('splitGroups'\), session\.id\)/); + assert.match(rendererSource, /effectivePermissionMode: session && session\.effectivePermissionMode \|\| null/); + assert.match(rendererSource, /visible: !!session && !worker && mode === "gui"/); + assert.match(rendererSource, /locked: worker/); + assert.match(rendererSource, /renderPermissionControl\(button, permissionState\(session\)\)/); }); diff --git a/test/worker-permission-routing.test.js b/test/worker-permission-routing.test.js index 03834d4a..c9188062 100644 --- a/test/worker-permission-routing.test.js +++ b/test/worker-permission-routing.test.js @@ -13,6 +13,7 @@ var path = require("node:path"); var root = path.join(__dirname, ".."); var { attachWorkerPermission, isUserInputTool, USER_INPUT_TOOLS } = require("../lib/project-worker-permission"); +var closeTransactionSource = fs.readFileSync(path.join(root, "lib/project-pair-close-transaction.js"), "utf8"); // --- Harness -------------------------------------------------------------- @@ -511,15 +512,15 @@ test("the Driver tool is mounted alongside the existing pair tools", function () assert.match(pairSource, /workerPermission\.getToolDefs\(boundSession, \{ dormantDriver: false \}\)/, "a paired Driver retains the permission decision tool"); // A configured Worker receives only its exact outcome-report surface. - assert.match(pairSource, /group && group\.pair && group\.pair\.driverId !== boundSession\.localId\) return taskControl\.workerToolDefs\(boundSession\);/); + assert.match(pairSource, /boundRoles && boundRoles\.workers\.indexOf\(boundSession\) !== -1\) return taskControl\.workerToolDefs\(boundSession\);/); }); -test("closing the pair cancels pending requests immediately", function () { - var fn = pairSource.slice(pairSource.indexOf("function closePartner(args, caller)")); - fn = fn.slice(0, fn.indexOf("\n function getToolDefs")); - assert.match(fn, /workerPermission\.cancelForSession\(partner, "The Driver closed the Split Worker pair\."\);/); - assert.ok(fn.indexOf("cancelForSession") < fn.indexOf("store.dissolve"), - "cancelled before the group record disappears"); +test("closing the pair cancels pending requests only after structural persistence", function () { + assert.match(closeTransactionSource, /workerPermission\.cancelForSession\(partner, "The Driver closed the Split Worker pair\."\);/); + assert.ok(closeTransactionSource.indexOf("structuralClose.removeSelectedWorker") < + closeTransactionSource.indexOf("cancelForSession"), "persistence failure leaves pending requests untouched"); + assert.ok(closeTransactionSource.indexOf("resultCapture.prepare") < + closeTransactionSource.indexOf("structuralClose.removeSelectedWorker"), "the exact pair captures before structural removal"); }); test("a permission request detaches a waiting delegation through the existing path", function () { @@ -532,6 +533,8 @@ test("a permission request detaches a waiting delegation through the existing pa test("server conventions and module sizes hold", function () { var files = [ ["project-worker-permission.js", routerSource], + ["project-pair-global-stop.js", fs.readFileSync(path.join(root, "lib/project-pair-global-stop.js"), "utf8")], + ["project-pair-close-transaction.js", closeTransactionSource], ["project-session-pair.js", pairSource], ["session-pair-factory.js", fs.readFileSync(path.join(root, "lib/session-pair-factory.js"), "utf8")], ]; diff --git a/test/worker-proposal.test.js b/test/worker-proposal.test.js index 5bab3471..62348778 100644 --- a/test/worker-proposal.test.js +++ b/test/worker-proposal.test.js @@ -14,6 +14,7 @@ function fixture(options) { var opts = options || {}; var session = { localId: 1, + sessionOriginId: "driver-origin", ownerId: null, title: "Planner", vendor: "claude", @@ -24,10 +25,13 @@ function fixture(options) { isProcessing: false, }; var sessions = new Map([[session.localId, session]]); + var seededWorkers = opts.workers || []; + for (var wi = 0; wi < seededWorkers.length; wi++) sessions.set(seededWorkers[wi].localId, seededWorkers[wi]); var updates = []; var directEvents = []; var starts = []; var pairs = []; + var additions = []; var delegations = []; var adapters = {}; var sm = { @@ -68,12 +72,18 @@ function fixture(options) { group: { id: "sg_worker", members: [1, 2], pair: { driverId: 1, workerId: 2 } }, }; }, + addWorkerForDriver: function (driver, message) { + additions.push({ driver: driver, message: message }); + if (typeof opts.addWorkerForDriver === "function") return opts.addWorkerForDriver(driver, message, sessions); + throw new Error("addition was not configured"); + }, sendToPartner: function (args, target) { delegations.push({ args: args, session: target }); return Promise.resolve({ content: [{ type: "text", text: JSON.stringify({ status: "complete", response: "Implemented and tested." }) }], }); }, + multiWorkerFeature: opts.multiWorkerFeature, }); return { attached: attached, @@ -82,6 +92,7 @@ function fixture(options) { directEvents: directEvents, starts: starts, pairs: pairs, + additions: additions, delegations: delegations, adapters: adapters, sm: sm, @@ -147,7 +158,7 @@ test("a proposal cannot outlive its exact session while model catalogs load", as resolveCatalog([{ value: "gpt-5.6-sol", displayName: "GPT-5.6 Sol" }]); var result = parseToolResult(await pending); - assert.match(result.error, /session changed while Split Worker runtimes were loading/); + assert.match(result.error, /membership changed while runtimes were loading/); assert.strictEqual(oldSession.history.length, 0, "the stale session receives no audit record"); assert.strictEqual(replacement.history.length, 0, "the replacement session receives no forged record"); assert.strictEqual(f.pairs.length, 0); @@ -342,8 +353,9 @@ test("an interrupted Worker proposal stays interrupted and warns the Driver", as }); function replacementFixture(status) { - var group = { id: "existing-pair", pair: { driverId: 1, workerId: 2 } }; + var group = { id: "existing-pair", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; var f = fixture({ group: group }); + f.sm.sessions.set(2, { localId: 2, ownerId: null, sessionOriginId: "worker-origin", _pairGeneration: 1 }); f.session.history.push({ type: "worker_proposal", proposalId: "accepted-worker", status: status, workerId: 2, groupId: group.id }); return f; } @@ -386,6 +398,107 @@ test("concurrent replacement proposals create only one pending decision", async assert.strictEqual(f.session.history.length, 2); }); +test("a V2 replacement card cannot outlive another member's generation change", async function () { + var feature = require("../lib/multi-worker-feature").fromServerConfig({ multiWorkerRuntimeEnabled: true }); + var group = { id: "v2-pair", members: [1, 2, 3], + pair: { version: 2, driverId: 1, workerIds: [2, 3] } }; + var f = fixture({ group: group, multiWorkerFeature: feature }); + var workerA = { localId: 2, ownerId: null, sessionOriginId: "worker-a", _pairGeneration: 4 }; + var workerB = { localId: 3, ownerId: null, sessionOriginId: "worker-b", _pairGeneration: 5 }; + f.sm.sessions.set(2, workerA); f.sm.sessions.set(3, workerB); + var args = replacementArgs(); + args.workerId = 2; + var posted = parseToolResult(await f.attached.proposeReplacement(args, f.session)); + assert.equal(posted.status, "posted"); + var proposal = f.session.history[f.session.history.length - 1]; + workerB._pairGeneration = 6; + await assert.rejects(f.attached.respondToProposal(f.ws, { proposalId: proposal.proposalId, + accepted: true, vendor: "codex", model: "gpt-5.6-sol", effort: "medium" }), /pair changed/); + assert.equal(proposal.status, "pending"); + assert.equal(workerA._pairGeneration, 4, "the selected Worker is untouched"); +}); + +test("an authoritative configuration card adds one exact second Worker and delegates once", async function () { + var feature = require("../lib/multi-worker-feature").fromServerConfig({ multiWorkerRuntimeEnabled: true }); + var worker = { localId: 2, ownerId: null, sessionOriginId: "worker-a", _pairGeneration: 4, history: [] }; + var group = { id: "pair", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + var f = fixture({ group: group, workers: [worker], multiWorkerFeature: feature, + addWorkerForDriver: function (driver, args, sessions) { + assert.strictEqual(args.expectedGroup, group); + assert.deepStrictEqual(args.expectedWorkerIds, [2]); + var added = { localId: 3, ownerId: null, sessionOriginId: "worker-b", _pairGeneration: 5, history: [] }; + sessions.set(3, added); + group.members = [1, 2, 3]; + group.pair = { version: 2, driverId: 1, workerIds: [2, 3] }; + return { driver: driver, worker: added, group: group }; + } }); + var tool = f.attached.getToolDefs(f.session, { persistent: true, controlsOnly: true }).find(function (item) { return item.name === "propose_worker"; }); + assert.ok(tool, "the paired Driver receives the add proposal tool only through the enabled server feature"); + var posted = parseToolResult(await tool.handler({ summary: "Parallel independent verification is useful.", + plan: "1. Keep Worker A running\n2. Give Worker B separate files", message: "Verify the independent backend paths.", + recommendedVendor: "codex", recommendedModel: "gpt-5.6-sol", recommendedEffort: "medium", + recommendationRationale: "The installed Codex runtime fits this independent verification." })); + assert.strictEqual(posted.status, "posted"); + var proposal = f.session.history[f.session.history.length - 1]; + assert.strictEqual(proposal.action, "add"); + assert.deepStrictEqual(proposal.sourceWorkerIds, [2]); + assert.strictEqual(proposal.sourceWorkers[0].generation, 4); + var accepted = await f.attached.respondToProposal(f.ws, { proposalId: proposal.proposalId, + accepted: true, vendor: "codex", model: "gpt-5.6-sol", effort: "medium" }); + assert.strictEqual(accepted.ok, true); + await nextTurn(); + assert.strictEqual(f.additions.length, 1); + assert.strictEqual(f.delegations.length, 1); + assert.strictEqual(f.delegations[0].args.workerId, 3); + assert.strictEqual(f.sm.sessions.get(2), worker, "the running peer identity is unchanged"); + await assert.rejects(f.attached.respondToProposal(f.ws, { proposalId: proposal.proposalId, accepted: true }), /already been resolved/); + assert.strictEqual(f.additions.length, 1, "duplicate acceptance allocates nothing"); +}); + +test("a third Worker is rejected before allocation and stale add cards cannot mutate membership", async function () { + var feature = require("../lib/multi-worker-feature").fromServerConfig({ multiWorkerRuntimeEnabled: true }); + var workerA = { localId: 2, ownerId: null, sessionOriginId: "worker-a", _pairGeneration: 4, history: [] }; + var workerB = { localId: 3, ownerId: null, sessionOriginId: "worker-b", _pairGeneration: 5, history: [] }; + var fullGroup = { id: "full", members: [1, 2, 3], pair: { version: 2, driverId: 1, workerIds: [2, 3] } }; + var full = fixture({ group: fullGroup, workers: [workerA, workerB], multiWorkerFeature: feature }); + var fullTool = full.attached.getToolDefs(full.session, { persistent: true, controlsOnly: true }).find(function (item) { return item.name === "propose_worker"; }); + var denied = parseToolResult(await fullTool.handler({ summary: "No", plan: "No", message: "No", recommendationRationale: "No" })); + assert.match(denied.error, /at most two/); + assert.strictEqual(full.additions.length, 0); + assert.strictEqual(full.session.history.length, 0); + + var group = { id: "stale", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + var stale = fixture({ group: group, workers: [workerA], multiWorkerFeature: feature, + addWorkerForDriver: function () { throw new Error("must not allocate"); } }); + var tool = stale.attached.getToolDefs(stale.session, { persistent: true, controlsOnly: true }).find(function (item) { return item.name === "propose_worker"; }); + var posted = parseToolResult(await tool.handler({ summary: "Independent work", plan: "1. Verify", message: "Verify", + recommendedVendor: "codex", recommendedModel: "gpt-5.6-sol", recommendedEffort: "medium", recommendationRationale: "Exact runtime" })); + workerA._pairGeneration = 9; + await assert.rejects(stale.attached.respondToProposal(stale.ws, { proposalId: posted.proposalId, + accepted: true, vendor: "codex", model: "gpt-5.6-sol", effort: "medium" }), /membership changed/); + assert.strictEqual(stale.additions.length, 0); + assert.strictEqual(workerB._pairGeneration, 5); +}); + +test("a failed add transaction keeps the card pending and preserves the existing Worker", async function () { + var feature = require("../lib/multi-worker-feature").fromServerConfig({ multiWorkerRuntimeEnabled: true }); + var worker = { localId: 2, ownerId: null, sessionOriginId: "worker-a", _pairGeneration: 4, history: [] }; + var group = { id: "pair", members: [1, 2], pair: { driverId: 1, workerId: 2 } }; + var f = fixture({ group: group, workers: [worker], multiWorkerFeature: feature, + addWorkerForDriver: function () { throw new Error("Split Worker session persistence returned false"); } }); + var tool = f.attached.getToolDefs(f.session, { persistent: true, controlsOnly: true }).find(function (item) { return item.name === "propose_worker"; }); + var posted = parseToolResult(await tool.handler({ summary: "Independent work", plan: "1. Verify", message: "Verify", + recommendedVendor: "codex", recommendedModel: "gpt-5.6-sol", recommendedEffort: "medium", recommendationRationale: "Exact runtime" })); + var result = await f.attached.respondToProposal(f.ws, { proposalId: posted.proposalId, + accepted: true, vendor: "codex", model: "gpt-5.6-sol", effort: "medium" }); + assert.strictEqual(result.ok, false); + assert.match(result.error, /persistence returned false/); + assert.strictEqual(f.session.history[f.session.history.length - 1].status, "pending"); + assert.deepStrictEqual(group.members, [1, 2]); + assert.strictEqual(f.sm.sessions.get(2), worker); + assert.strictEqual(f.delegations.length, 0); +}); + test("pending proposals can be inspected and cancelled by exact id", async function () { var f = fixture(); var proposal = await postProposal(f);