From 9aca358befd31d13f61d0a738be479c2b7337a60 Mon Sep 17 00:00:00 2001 From: Jonathan Liebig Date: Mon, 12 Oct 2026 15:02:59 +0200 Subject: [PATCH 1/2] Support native ChatGPT clarification question handoffs --- AGENTS.md | 2 +- CHANGELOG.md | 7 + README.md | 21 +++ bin/ask-pro-cli.ts | 58 +++++- docs/manual-tests.md | 11 ++ docs/windows-work.md | 9 + skills/ask-pro/SKILL.md | 10 ++ src/ask-pro/browserRunner.ts | 59 +++++- src/ask-pro/questions.ts | 58 ++++++ src/ask-pro/session.ts | 1 + src/browser/actions/assistantResponse.ts | 11 +- src/browser/actions/questions.ts | 220 +++++++++++++++++++++++ src/browser/constants.ts | 4 +- src/browser/index.ts | 7 + src/browser/reattach.ts | 48 ++++- tests/ask-pro/browserRunner.test.ts | 68 ++++++- tests/ask-pro/cli.test.ts | 70 +++++++- tests/browser/assistantResponse.test.ts | 217 ++++++++++++++++++++++ tests/browser/reattach.test.ts | 58 ++++-- 19 files changed, 901 insertions(+), 38 deletions(-) create mode 100644 src/ask-pro/questions.ts create mode 100644 src/browser/actions/questions.ts diff --git a/AGENTS.md b/AGENTS.md index 529be3e0b8..5c0c9322af 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ ask_pro-specific notes: reduced to the V1 ask-pro surface. - Keep the CLI small: `ask-pro ""`, `--files`, `--prompt-file`, `--artifacts` / `--response-zip`, `--dry-run`, `--resume`, `--status`, - `--harvest`, `--copy`, `--temporary`, `--no-temporary`, and + `--harvest`, `--copy`, `--answers`, `--temporary`, `--no-temporary`, and `--verbose`, and `--github`. - Browser auth is human-controlled. Never ask for, type, read, or log passwords, MFA codes, recovery codes, session cookies, or raw auth tokens. diff --git a/CHANGELOG.md b/CHANGELOG.md index bcf22f512a..0b0cc2ca88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- Hand native ChatGPT clarification questions to the calling agent, including + numbered choices and free text. Resume with `--answers` to answer in the same + conversation, including recovery when the provider expires its question cards. + - Add `--github` with a connection check before upload, human setup in the managed browser, and a resumable handoff. Request read-only repository evidence and citations while retaining local-file context. The flag does not restrict @@ -14,6 +18,9 @@ ### Fixed +- Capture replies in ChatGPT's current search-unit message layout and close + the controller's DevTools connection when a resumed session hands off questions. + - Prefer ChatGPT's `Latest` model when available, otherwise select the highest numbered available GPT model and confirm `Pro` intelligence. - Ignore background Cloudflare scripts and unrelated notices on usable ChatGPT diff --git a/README.md b/README.md index cb590850f0..fdac828ca7 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,27 @@ Use `ask-pro --help` for the full CLI. For multiline prompts, use `--prompt-file`. Prefer `--no-temporary` when recovery matters; `--temporary` requires Temporary Chat. The [skill](skills/ask-pro/SKILL.md) has agent guidance. +When Pro asks native questions, ask-pro returns `awaiting_answers`, prints each +question and its numbered choices, and saves `QUESTIONS.json`. The calling agent +can answer directly, or ask its user when a decision needs their input. Write a +UTF-8 JSON file using the emitted `request_id`, with one answer per question: + +```json +{ "requestId": "", "answers": [2, 3, "A workshop on incident response"] } +``` + +Numbers select 1-based options; strings enter free text. Then continue the same chat: + +```powershell +ask-pro --resume --answers answers.json +``` + +`--answers -` accepts piped JSON. ask-pro requests a native pause; if ChatGPT expires +the cards anyway, it sends your answers as a reply in the same conversation. +Keep the session's original Temporary Chat mode when answering. +Further question rounds use the same +flow; `--status` repeats pending questions and `--harvest` waits for a final answer. + ## Optional GitHub context Use `--github` to check the GitHub connection in ask-pro's signed-in ChatGPT diff --git a/bin/ask-pro-cli.ts b/bin/ask-pro-cli.ts index dd4a0f89da..3ed2a5d7c9 100644 --- a/bin/ask-pro-cli.ts +++ b/bin/ask-pro-cli.ts @@ -1,6 +1,8 @@ #!/usr/bin/env node import "dotenv/config"; import { AssistantStoppedError } from "../src/browser/errors.js"; +import { AssistantQuestionsError } from "../src/browser/actions/questions.js"; +import { readAskProQuestions } from "../src/ask-pro/questions.js"; import { GITHUB_SETUP_URL, GitHubConnectionRequiredError } from "../src/browser/actions/github.js"; import fs from "node:fs/promises"; import path from "node:path"; @@ -32,6 +34,7 @@ interface AskProOptions { dryRun?: boolean; files?: string[]; promptFile?: string; + answers?: string; artifacts?: boolean; github?: boolean; responseZip?: boolean; @@ -65,6 +68,11 @@ program ]), ) .option("--resume [session-id]", "resume a prepared or waiting ask-pro session") + .addOption( + new Option("--answers ", "answer native questions from JSON; use - for stdin") + .implies({ resume: true }) + .conflicts(["status", "harvest", "copy", "dryRun", "promptFile"]), + ) .option("--status [session-id]", "show ask-pro session status") .option("--harvest [session-id]", "print harvested ANSWER.md for a session") .option("--copy [session-id]", "print the copy target for a session") @@ -90,6 +98,8 @@ await program.parseAsync(process.argv); async function runAskPro(question: string, options: AskProOptions): Promise { const cwd = resolveProjectCwd(options); + if (options.answers && question.trim()) + throw new Error("Use --answers with --resume, without a question argument."); await pruneExpiredAskProSessions({ cwd }); if (options.status !== undefined) { const { status } = await readAskProStatus({ cwd, sessionId: optionSessionId(options.status) }); @@ -150,6 +160,14 @@ async function runAskPro(question: string, options: AskProOptions): Promise { const { status } = await readAskProStatus({ cwd, sessionId }); + if (options.answers !== undefined && status.status !== "AWAITING_ANSWERS") + throw new Error( + "This session is not waiting for answers. Resume without --answers to inspect it.", + ); + if (options.answers === "-" && process.stdin.isTTY) + throw new Error("--answers - requires piped stdin."); if (status.status === "COMPLETED" || status.status === "HARVESTED") { printStatusRecord(status, { ...(await readBrowserPreflight(cwd, status)), @@ -251,6 +275,7 @@ async function submitOrResumeBrowserSession( } if ( status.status === "WAITING" || + status.status === "AWAITING_ANSWERS" || status.status === "WAIT_TIMED_OUT" || status.status === "INCOMPLETE_ANSWER" || status.status === "NEEDS_USER_AUTH" || @@ -262,6 +287,12 @@ async function submitOrResumeBrowserSession( sessionId, temporary: options.temporary, verbose: options.verbose, + answers: + options.answers === undefined + ? undefined + : options.answers === "-" + ? await readStdin() + : await fs.readFile(path.resolve(cwd, options.answers), "utf8"), }); } catch (error) { if (error instanceof GitHubConnectionRequiredError) { @@ -269,7 +300,7 @@ async function submitOrResumeBrowserSession( printStatusRecord(paused); return; } - if (error instanceof AssistantStoppedError) { + if (error instanceof AssistantStoppedError || error instanceof AssistantQuestionsError) { const { status: stopped } = await readAskProStatus({ cwd, sessionId }); printStatusRecord(stopped, await readBrowserPreflight(cwd, stopped)); return; @@ -305,7 +336,7 @@ async function submitOrResumeBrowserSession( printStatusRecord(paused); return; } - if (error instanceof AssistantStoppedError) { + if (error instanceof AssistantStoppedError || error instanceof AssistantQuestionsError) { const { status: stopped } = await readAskProStatus({ cwd, sessionId }); printStatusRecord(stopped, await readBrowserPreflight(cwd, stopped)); return; @@ -458,6 +489,8 @@ function actionForStatus(status: AskProStatusFile): string { switch (status.status) { case "NEEDS_GITHUB_CONNECTION": return "connect_github_then_resume"; + case "AWAITING_ANSWERS": + return "answer_questions"; case "DRY_RUN_COMPLETE": case "INCOMPLETE_ANSWER": case "READY_TO_SUBMIT": @@ -482,6 +515,7 @@ function shouldPrintResume(status: AskProStatusFile): boolean { "DRY_RUN_COMPLETE", "INCOMPLETE_ANSWER", "WAITING", + "AWAITING_ANSWERS", "READY_TO_SUBMIT", "NEEDS_USER_AUTH", "NEEDS_GITHUB_CONNECTION", @@ -510,7 +544,7 @@ async function readRecoverableCapturedAnswer( cwd: string, status: AskProStatusFile, ): Promise { - if (status.status === "INCOMPLETE_ANSWER") { + if (status.status === "INCOMPLETE_ANSWER" || status.status === "AWAITING_ANSWERS") { return null; } try { @@ -545,7 +579,25 @@ async function readBrowserPreflightForSession( if (!metadata) return {}; const profileDir = typeof metadata.profileDir === "string" ? metadata.profileDir : undefined; const profile = profileMode(profileDir); + const questions: AskProToonFields = {}; + if (metadata.status === "awaiting_answers") { + const request = await readAskProQuestions(getAskProSessionPaths(cwd, sessionId).dir); + questions.request_id = request.requestId; + questions.questions = request.questions.length; + questions.questions_file = `.ask-pro/sessions/${sessionId}/QUESTIONS.json`; + questions.reply = `${buildSessionCommand(cwd, ["--resume", sessionId, "--answers", ""])}`; + questions.answers_format = + '{"requestId":"","answers":[1,"free text"]} (one answer per question; choices are 1-based)'; + for (const [index, question] of request.questions.entries()) { + questions[`question_${index + 1}`] = question.question; + questions[`options_${index + 1}`] = question.options + .map((option, choice) => `${choice + 1}: ${option}`) + .join("; "); + questions[`free_text_${index + 1}`] = question.freeText; + } + } return compactFields({ + ...questions, profile, profile_path: profile && profileDir ? collapseHome(profileDir) : undefined, chrome: chromeMode(metadata), diff --git a/docs/manual-tests.md b/docs/manual-tests.md index ce4d0ceba5..381ebcb1e7 100644 --- a/docs/manual-tests.md +++ b/docs/manual-tests.md @@ -4,6 +4,17 @@ These checks exercise the real ChatGPT browser path. They are opt-in because they open Chrome, use the signed-in ChatGPT account, and can take several minutes or longer for Pro thinking. +## Native questions canary + +Ask Pro to use its `ask-user-input` skill for three native cards: two choices, +three choices, and a question whose Other field accepts free text, then wait and +summarize the answers. The October 12 tool rejects questions with zero options. +Verify `awaiting_answers` prints all cards, no answer is harvested, and the cards +can still be answered after their original countdown. Reply with `--resume +--answers answers.json` using `{"requestId":"","answers":[2,3,"custom theme"]}`. +Verify the final answer preserves all selections and custom text. Repeat with +one and two cards, and verify a stale request ID is rejected before submission. + ## Prerequisites - Node 24+. diff --git a/docs/windows-work.md b/docs/windows-work.md index bd5ecae043..9f7c5f033d 100644 --- a/docs/windows-work.md +++ b/docs/windows-work.md @@ -77,6 +77,15 @@ Read this when working on `ask_pro` from Windows and add new findings here. Future Windows gotchas belong here. +- October 12: native clarification cards use + `data-codex-composer-request-navigation` and show one question at a time. + Pointer interaction requests a pause, but the live server returned 409 + ("Request can no longer be snoozed") even for fresh cards. ask-pro + browses the cards without submitting defaults, retains Chrome, and returns + `AWAITING_ANSWERS`. Use a UTF-8 JSON file with `--resume ... --answers ...`; + this avoids PowerShell quoting problems with multiline free-text answers. + Expired cards are answered by replying in the same conversation. + - October 7: the live Select model view exposes `GPT-6`, `GPT-5.6 Sol`, and `GPT-5.5` instead of `Latest`. The default picker prefers `Latest` when present, otherwise compares visible enabled terminal GPT versions numerically. Pro diff --git a/skills/ask-pro/SKILL.md b/skills/ask-pro/SKILL.md index 5bd9234084..57dba1ae84 100644 --- a/skills/ask-pro/SKILL.md +++ b/skills/ask-pro/SKILL.md @@ -162,6 +162,16 @@ Normal `ask-pro` stdout is compact TOON-style telemetry. Use `state`, `action`, `resume`, and `harvest` to decide the next command. Browser progress may appear on stderr and can be ignored unless diagnosing a stuck run. +`AWAITING_ANSWERS` / `answer_questions` means Pro is asking native interactive +questions. Read the printed questions/options or `QUESTIONS.json`, answer from +the caller's context, and ask the human only for decisions you cannot infer. +Write UTF-8 JSON `{"requestId":"","answers":[2,3,"free text"]}` +with one answer per question in order, then run the emitted +`ask-pro --resume --answers ` command. Numbers select +1-based choices; strings enter the native free-text field. `--answers -` accepts +piped JSON. Do not submit defaults or treat the question preamble as the answer. +Repeat for further question rounds, then harvest the final answer normally. + When present, use `profile`, `profile_path`, `chrome`, and `language` only as diagnostic hints. They tell you whether the run used the shared profile, an isolated agent profile, saved DevTools state, and English browser steering. diff --git a/src/ask-pro/browserRunner.ts b/src/ask-pro/browserRunner.ts index aa97d69ba9..670650ab50 100644 --- a/src/ask-pro/browserRunner.ts +++ b/src/ask-pro/browserRunner.ts @@ -2,6 +2,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import { DEFAULT_CHATGPT_BROWSER_MODEL_LABEL } from "../browser/chatgptModelCatalog.js"; import { AssistantStoppedError, BrowserAutomationError } from "../browser/errors.js"; +import { AssistantQuestionsError } from "../browser/actions/questions.js"; +import { readAskProQuestions, parseQuestionAnswers, writeAskProQuestions } from "./questions.js"; import { GitHubConnectionRequiredError } from "../browser/actions/github.js"; import { askProAgentIdForLegacyBrowserProfileDir, @@ -43,6 +45,7 @@ export interface RunAskProBrowserSessionOptions { agentId?: string | null; allowStartMinimized?: boolean; verbose?: boolean; + answers?: string; } export async function runAskProBrowserSession( @@ -302,6 +305,12 @@ async function runAskProBrowserSessionWithLease({ throw new AskProNeedsAuthError(sessionId, browserProfile, classifyBrowserError(error)); } + if (error instanceof AssistantQuestionsError) { + await writeAskProQuestions(paths.dir, error.questions); + await writeTerminalBrowserMetadata(cwd, sessionId, "awaiting_answers", "questions"); + await updateAskProStatus({ cwd, sessionId, status: "AWAITING_ANSWERS", reason: "questions" }); + throw error; + } if (error instanceof AssistantStoppedError) { await writeTerminalBrowserMetadata( cwd, @@ -351,13 +360,36 @@ async function resumeAskProBrowserSessionWithLease({ sessionId, temporary, verbose, + answers, }: RunAskProBrowserSessionOptions): Promise { const paths = getAskProSessionPaths(cwd, sessionId); const prompt = await readAskProPrompt({ cwd, sessionId }); const { status: sessionStatus } = await readAskProStatus({ cwd, sessionId }); + const request = answers !== undefined ? await readAskProQuestions(paths.dir) : undefined; + if (answers !== undefined && sessionStatus.status !== "AWAITING_ANSWERS") + throw new Error( + "This session is not waiting for answers. Resume without --answers to inspect it.", + ); + const questionAnswers = + request && answers !== undefined + ? { + questions: request.questions, + answers: parseQuestionAnswers(answers, request), + submissionState: { started: false }, + } + : undefined; const artifactsRequested = sessionStatus.artifacts === true; const logger = buildAskProBrowserLogger(cwd, sessionId, verbose); const metadata = await readBrowserMetadata(paths.browser); + if ( + questionAnswers && + temporary !== undefined && + temporary !== (metadata.temporary ?? isTemporaryAskProUrl(metadata.url ?? "")) + ) { + throw new Error( + "Cannot change Temporary Chat mode while answering questions; resume with the saved mode.", + ); + } const effectiveTemporary = temporary ?? metadata.temporary; const chatgptUrl = effectiveTemporary === true @@ -468,6 +500,10 @@ async function resumeAskProBrowserSessionWithLease({ }, logger, { + questionAnswers, + afterQuestionsAnswered: async () => { + await fs.rm(path.join(paths.dir, "QUESTIONS.json"), { force: true }); + }, promptPreview: prompt, chromeModeCb: async (chromeMode) => { const current = await readBrowserMetadata(paths.browser).catch(() => metadata); @@ -565,6 +601,12 @@ async function resumeAskProBrowserSessionWithLease({ }); throw new AskProNeedsAuthError(sessionId, fallbackProfile, classifyBrowserError(error)); } + if (error instanceof AssistantQuestionsError) { + await writeAskProQuestions(paths.dir, error.questions); + await writeTerminalBrowserMetadata(cwd, sessionId, "awaiting_answers", "questions"); + await updateAskProStatus({ cwd, sessionId, status: "AWAITING_ANSWERS", reason: "questions" }); + throw error; + } if (error instanceof AssistantStoppedError) { await writeTerminalBrowserMetadata( cwd, @@ -591,11 +633,17 @@ async function resumeAskProBrowserSessionWithLease({ throw error; } const reason = error instanceof Error ? error.message : String(error); - await writeTerminalBrowserMetadata(cwd, sessionId, "failed", reason); + const answersPending = questionAnswers && !questionAnswers.submissionState.started; + await writeTerminalBrowserMetadata( + cwd, + sessionId, + answersPending ? "awaiting_answers" : questionAnswers ? "wait_timed_out" : "failed", + reason, + ); await updateAskProStatus({ cwd, sessionId, - status: "FAILED", + status: answersPending ? "AWAITING_ANSWERS" : questionAnswers ? "WAIT_TIMED_OUT" : "FAILED", reason, }); throw error; @@ -841,7 +889,12 @@ function authFailureChromeMode(chromeMode: AskProBrowserMetadata["chromeMode"]) async function writeTerminalBrowserMetadata( cwd: string, sessionId: string, - status: "failed" | "wait_timed_out" | "incomplete_answer" | "needs_github_connection", + status: + | "failed" + | "wait_timed_out" + | "incomplete_answer" + | "needs_github_connection" + | "awaiting_answers", reason: string, ): Promise { const paths = getAskProSessionPaths(cwd, sessionId); diff --git a/src/ask-pro/questions.ts b/src/ask-pro/questions.ts new file mode 100644 index 0000000000..e822272a91 --- /dev/null +++ b/src/ask-pro/questions.ts @@ -0,0 +1,58 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { atomicWriteFile } from "./atomicWrite.js"; +import type { AssistantQuestion } from "../browser/actions/questions.js"; + +export interface AskProQuestions { + requestId: string; + questions: AssistantQuestion[]; +} + +export async function readAskProQuestions(sessionDir: string): Promise { + return JSON.parse(await fs.readFile(path.join(sessionDir, "QUESTIONS.json"), "utf8")); +} + +export async function writeAskProQuestions(sessionDir: string, questions: AssistantQuestion[]) { + const previous = await readAskProQuestions(sessionDir).catch(() => null); + const request = { + requestId: + previous && JSON.stringify(previous.questions) === JSON.stringify(questions) + ? previous.requestId + : randomUUID(), + questions, + }; + await atomicWriteFile( + path.join(sessionDir, "QUESTIONS.json"), + `${JSON.stringify(request, null, 2)}\n`, + ); +} + +export function parseQuestionAnswers( + raw: string, + request: AskProQuestions, +): Array { + const value = JSON.parse(raw); + if (value?.requestId !== request.requestId) + throw new Error( + "Answers belong to a different question request; read the current QUESTIONS.json.", + ); + if (!Array.isArray(value.answers) || value.answers.length !== request.questions.length) { + throw new Error("Provide one answer per question in answers, in question order."); + } + for (const [index, answer] of value.answers.entries()) { + const question = request.questions[index]!; + if ( + typeof answer === "number" && + Number.isInteger(answer) && + answer >= 1 && + answer <= question.options.length + ) + continue; + if (typeof answer === "string" && answer.trim() && question.freeText) continue; + throw new Error( + `Question ${index + 1}: choose a numbered option or provide nonempty free text.`, + ); + } + return value.answers; +} diff --git a/src/ask-pro/session.ts b/src/ask-pro/session.ts index 5bfd13e58d..14be1402ee 100644 --- a/src/ask-pro/session.ts +++ b/src/ask-pro/session.ts @@ -12,6 +12,7 @@ export type AskProStatus = | "NEEDS_USER_AUTH" | "NEEDS_GITHUB_CONNECTION" | "WAITING" + | "AWAITING_ANSWERS" | "WAIT_TIMED_OUT" | "INCOMPLETE_ANSWER" | "HARVESTED" diff --git a/src/browser/actions/assistantResponse.ts b/src/browser/actions/assistantResponse.ts index 0e03c59eb4..9801f80e85 100644 --- a/src/browser/actions/assistantResponse.ts +++ b/src/browser/actions/assistantResponse.ts @@ -13,6 +13,7 @@ import { buildClickDispatcher } from "./domEvents.js"; import { AssistantStoppedError } from "../errors.js"; import { readComposerSnapshot, submitPrompt } from "./promptComposer.js"; import type { PostSubmitInputGuard } from "./inputGuard.js"; +import { AssistantQuestionsError, readAssistantQuestions } from "./questions.js"; const ASSISTANT_POLL_TIMEOUT_ERROR = "assistant-response-watchdog-timeout"; @@ -97,18 +98,21 @@ export async function waitForAssistantResponse( const deadline = Date.now() + timeoutMs; while (true) { try { - return await waitForAssistantResponseOnce( + const answer = await waitForAssistantResponseOnce( Runtime, Math.max(1, deadline - Date.now()), logger, minTurnIndex, expectedConversationId, ); + const questions = await readAssistantQuestions(Runtime); + if (questions) throw new AssistantQuestionsError(questions); + return answer; } catch (error) { - if (!(error instanceof AssistantStoppedError)) throw error; await Runtime.evaluate({ expression: "window.__askProCancelResponseObserver?.()", }).catch(() => undefined); + if (!(error instanceof AssistantStoppedError)) throw error; if (!continueResponse || Date.now() >= deadline) throw error; minTurnIndex = await continueResponse(error.turnIndex); } @@ -196,6 +200,7 @@ async function waitForAssistantResponseOnce( throw new Error("Timed out waiting for assistant response"); } else if (source === "poll") { pollerAbort.abort(); + evaluationPromise.catch(() => undefined); throw error; } else if (source === "evaluation") { const recovered = await recoverAssistantResponse( @@ -520,6 +525,8 @@ async function pollAssistantCompletion( if (abortSignal?.aborted) { return null; } + const questions = await readAssistantQuestions(Runtime); + if (questions) throw new AssistantQuestionsError(questions); const snapshot = await readAssistantSnapshot(Runtime, minTurnIndex, expectedConversationId); const normalized = normalizeAssistantSnapshot(snapshot); if (normalized) { diff --git a/src/browser/actions/questions.ts b/src/browser/actions/questions.ts new file mode 100644 index 0000000000..06cdef05a5 --- /dev/null +++ b/src/browser/actions/questions.ts @@ -0,0 +1,220 @@ +import type { ChromeClient } from "../types.js"; +import type { BrowserLogger } from "../types.js"; +import { readComposerSnapshot, submitPrompt } from "./promptComposer.js"; + +export interface AssistantQuestion { + question: string; + options: string[]; + freeText: boolean; +} + +export class AssistantQuestionsError extends Error { + constructor(readonly questions: AssistantQuestion[]) { + super("ChatGPT is waiting for answers to its questions."); + this.name = "AssistantQuestionsError"; + } +} + +// This function also runs in Chrome; keep it independent of module state. +async function questionCards( + answers?: Array, + expected?: AssistantQuestion[], + submission = { started: false }, +) { + const selector = '[data-codex-composer-request-navigation="true"]'; + const visible = (node: Element) => { + const rect = node.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0 && !node.closest('[inert], [aria-hidden="true"]'); + }; + const card = () => Array.from(document.querySelectorAll(selector)).find(visible); + let root = card(); + if (!root) return null; + // Native pointer interaction cancels ChatGPT's automatic question dismissal. + root.dispatchEvent(new PointerEvent("pointerdown", { bubbles: true, pointerType: "mouse" })); + const pause = () => new Promise((resolve) => setTimeout(resolve, 100)); + const read = (): AssistantQuestion => { + root = card(); + if (!root) throw new Error("ChatGPT question card disappeared."); + return { + question: root.querySelector(".text-size-chat")?.innerText.trim() ?? "", + options: Array.from(root.querySelectorAll('[role="radio"]')).map( + (option) => + option.getAttribute("aria-label") ?? option.innerText.replace(/^\d+\s*/, "").trim(), + ), + freeText: Boolean(root.querySelector('[role="textbox"], textarea, input:not([type="file"])')), + }; + }; + const position = () => { + const match = root?.innerText.match(/(?:^|\n)(\d+) of (\d+)(?:\n|$)/); + return match ? { index: Number(match[1]), count: Number(match[2]) } : { index: 1, count: 1 }; + }; + const move = async (direction: "Previous" | "Next") => { + const before = position().index; + const button = root?.querySelector( + `button[aria-label="${direction} question"]`, + ); + if (!button || button.disabled) throw new Error("Cannot browse ChatGPT's questions."); + button.click(); + const deadline = Date.now() + 5_000; + do { + await pause(); + read(); + if (position().index !== before) return; + } while (Date.now() < deadline); + throw new Error("ChatGPT did not change questions."); + }; + while (position().index > 1) await move("Previous"); + const questions: AssistantQuestion[] = []; + const count = position().count; + for (let index = 0; index < count; index++) { + const current = read(); + if (!current.question) throw new Error("Cannot read ChatGPT's question."); + questions.push(current); + if (index + 1 < count) await move("Next"); + } + while (position().index > 1) await move("Previous"); + if (!answers) return questions; + if (JSON.stringify(questions) !== JSON.stringify(expected)) { + throw new Error("ChatGPT's questions changed; read the current questions before answering."); + } + if (answers.length !== questions.length) throw new Error("Provide one answer per question."); + for (const [index, answer] of answers.entries()) { + const question = questions[index]!; + if (typeof answer === "number") { + if (!Number.isInteger(answer) || answer < 1 || answer > question.options.length) { + throw new Error( + `Question ${index + 1}: choose 1-${question.options.length} or provide free text.`, + ); + } + } else if (typeof answer !== "string" || !answer.trim() || !question.freeText) { + throw new Error(`Question ${index + 1}: a nonempty free-text answer is required.`); + } + } + for (const [index, answer] of answers.entries()) { + read(); + if (typeof answer === "number") { + const options = Array.from(root!.querySelectorAll('[role="radio"]')); + if (index + 1 === count) submission.started = true; + options[answer - 1]!.click(); + if (index + 1 < count) { + const deadline = Date.now() + 5_000; + do { + await pause(); + read(); + } while (position().index === index + 1 && Date.now() < deadline); + if (position().index !== index + 2) + throw new Error(`ChatGPT did not accept answer for question ${index + 1}.`); + } + } else { + const input = root!.querySelector( + '[role="textbox"], textarea, input:not([type="file"])', + ); + if (!input) throw new Error("ChatGPT's free-text input is missing."); + input.focus(); + if (input instanceof HTMLInputElement || input instanceof HTMLTextAreaElement) { + const prototype = + input instanceof HTMLInputElement + ? HTMLInputElement.prototype + : HTMLTextAreaElement.prototype; + Object.getOwnPropertyDescriptor(prototype, "value")!.set!.call(input, answer); + input.dispatchEvent(new Event("input", { bubbles: true })); + } else { + const selection = window.getSelection(); + const range = document.createRange(); + range.selectNodeContents(input); + selection?.removeAllRanges(); + selection?.addRange(range); + document.execCommand("insertText", false, answer); + } + await pause(); + if ( + (input instanceof HTMLInputElement || input instanceof HTMLTextAreaElement + ? input.value + : Array.from(input.children) + .map((child) => (child.textContent ? (child as HTMLElement).innerText : "")) + .join("\n")) !== answer + ) { + throw new Error(`Could not enter answer for question ${index + 1}.`); + } + } + if (typeof answer === "string" && index + 1 < count) await move("Next"); + } + // Choice clicks advance/submit natively; a custom answer uses the card's submit button. + if (typeof answers.at(-1) === "string") { + const submit = root!.querySelector("[data-request-input-skip]"); + if (!submit || submit.disabled || !/^(submit|send|continue)$/i.test(submit.innerText.trim())) + throw new Error("ChatGPT's answer submit button is not ready."); + submission.started = true; + submit.click(); + } + const deadline = Date.now() + 5_000; + while (card() && Date.now() < deadline) await pause(); + if (card()) + throw new Error( + "ChatGPT did not accept the answers; inspect the question card before retrying.", + ); + return questions; +} + +export function buildQuestionCardsExpression( + answers?: Array, + expected?: AssistantQuestion[], +) { + // tsx preserves function names with __name; Chrome only needs the function itself. + const call = `(${questionCards.toString()})(${JSON.stringify(answers) ?? "undefined"}, ${JSON.stringify(expected) ?? "undefined"}, submission)`; + return `(() => { const __name = (fn) => fn; const submission = { started: false }; return ${answers ? `${call}.then(questions => ({ questions, started: submission.started }), error => ({ error: String(error), started: submission.started }))` : call}; })()`; +} + +export async function readAssistantQuestions(Runtime: ChromeClient["Runtime"]) { + const evaluation = await Runtime.evaluate({ + expression: buildQuestionCardsExpression(), + awaitPromise: true, + returnByValue: true, + }); + if (evaluation.exceptionDetails) + throw new Error( + evaluation.exceptionDetails.exception?.description ?? "Cannot read ChatGPT's questions.", + ); + return Array.isArray(evaluation.result.value) && evaluation.result.value.length > 0 + ? (evaluation.result.value as AssistantQuestion[]) + : null; +} + +export async function answerAssistantQuestions( + Runtime: ChromeClient["Runtime"], + Input: ChromeClient["Input"], + logger: BrowserLogger, + answers: Array, + expected: AssistantQuestion[], + submissionState = { started: false }, +) { + // A lost transport may have submitted successfully; never replay that attempt automatically. + submissionState.started = true; + const evaluation = await Runtime.evaluate({ + expression: buildQuestionCardsExpression(answers, expected), + awaitPromise: true, + returnByValue: true, + }); + if (evaluation.exceptionDetails) + throw new Error( + evaluation.exceptionDetails.exception?.description ?? "Cannot answer ChatGPT's questions.", + ); + const value = evaluation.result.value as { + questions: AssistantQuestion[] | null; + started: boolean; + error?: string; + }; + submissionState.started = value.started; + if (value.error) throw new Error(value.error); + if (!value.questions) { + // Native snooze returned 409 in the canary; expired cards need a reply in the same chat. + const composer = await readComposerSnapshot(Runtime); + if (composer.activeValue.trim()) + throw new Error( + "The ChatGPT composer has a draft; clear it before answering expired questions.", + ); + const prompt = `Answers to your clarification questions:\n\n${expected.map((question, index) => `${index + 1}. ${question.question}\n${typeof answers[index] === "number" ? question.options[(answers[index] as number) - 1] : answers[index]}`).join("\n\n")}\n\nContinue the original request using these answers.`; + submissionState.started = true; + await submitPrompt({ runtime: Runtime, input: Input }, prompt, logger); + } +} diff --git a/src/browser/constants.ts b/src/browser/constants.ts index a25819c19e..c94625e2c2 100644 --- a/src/browser/constants.ts +++ b/src/browser/constants.ts @@ -38,9 +38,9 @@ export const ANSWER_SELECTORS = [ export const CONVERSATION_TURN_SELECTOR = 'article[data-testid^="conversation-turn"], div[data-testid^="conversation-turn"], section[data-testid^="conversation-turn"], ' + "article[data-message-author-role], div[data-message-author-role], section[data-message-author-role], " + - "article[data-turn], div[data-turn], section[data-turn]"; + 'article[data-turn], div[data-turn], section[data-turn], [data-content-search-unit-key$=":assistant"], [data-content-search-unit-key$=":user"]'; export const ASSISTANT_ROLE_SELECTOR = - '[data-message-author-role="assistant"], [data-turn="assistant"]'; + '[data-message-author-role="assistant"], [data-turn="assistant"], [data-markdown-text-style="assistant-message"]'; export const CLOUDFLARE_TITLE = "just a moment"; export const PROMPT_PRIMARY_SELECTOR = "#prompt-textarea"; export const PROMPT_FALLBACK_SELECTOR = 'textarea[name="prompt-textarea"]'; diff --git a/src/browser/index.ts b/src/browser/index.ts index 90c8f0e0ca..82f97fe04f 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -58,6 +58,7 @@ import { formatElapsed } from "./format.js"; import { CHATGPT_URL, CONVERSATION_TURN_SELECTOR, DEFAULT_MODEL_STRATEGY } from "./constants.js"; import type { LaunchedChrome } from "chrome-launcher"; import { AssistantStoppedError, BrowserAutomationError } from "./errors.js"; +import { AssistantQuestionsError } from "./actions/questions.js"; import { defaultAskProBrowserProfileDir } from "./profilePaths.js"; import { applyPageLanguageOverrides, seedChromeProfileLanguage } from "./language.js"; import { @@ -1626,6 +1627,12 @@ export async function runBrowserMode(options: BrowserRunOptions): Promise; + questions: AssistantQuestion[]; + submissionState: { started: boolean }; + }; + afterQuestionsAnswered?: () => Promise; continuationState?: { used: boolean }; listTargets?: () => Promise; connect?: (options?: unknown) => Promise; @@ -220,6 +231,7 @@ export async function resumeBrowserSession( ? await acquireReattachRunLease(directUserDataDir, resolvedConfig.profileLockTimeoutMs, logger) : null; let closeConnection: (() => Promise) | null = null; + let answeringQuestions = false; try { const liveRuntime = await refreshAttachRuntime(runtime, logger).catch(() => runtime); @@ -266,7 +278,8 @@ export async function resumeBrowserSession( )) as unknown as ChromeClient, close: async () => undefined, } as const); - closeConnection = connection.close; + closeConnection = + browserWSEndpoint && !deps.connect ? connection.close : () => connection.client.close(); const connectedTargetId = "targetId" in connection ? connection.targetId : target?.targetId; const ownsConnectedTarget = Boolean( !resolvedConfig.browserTabRef && @@ -341,6 +354,18 @@ export async function resumeBrowserSession( ); await ensureConversationOpen(); const minTurnIndex = await readConversationTurnIndex(Runtime, logger); + if (deps.questionAnswers) { + answeringQuestions = true; + await answerAssistantQuestions( + Runtime, + Input, + logger, + deps.questionAnswers.answers, + deps.questionAnswers.questions, + deps.questionAnswers.submissionState, + ); + await deps.afterQuestionsAnswered?.(); + } const promptEcho = buildPromptEchoMatcher(deps.promptPreview); const answer = await withTimeout( waitForResponse( @@ -379,7 +404,7 @@ export async function resumeBrowserSession( }, }); - await connection.close().catch(() => undefined); + await closeConnection().catch(() => undefined); closeConnection = null; const keepBrowserOpen = Boolean( resolvedConfig.keepBrowser || @@ -419,7 +444,13 @@ export async function resumeBrowserSession( await closeConnection?.().catch(() => undefined); await releaseReattachRunLease(reattachRunLease, logger); reattachRunLease = null; - if (error instanceof AssistantStoppedError) throw error; + if ( + error instanceof AssistantStoppedError || + error instanceof AssistantQuestionsError || + answeringQuestions || + deps.questionAnswers?.submissionState.started + ) + throw error; const message = error instanceof Error ? error.message : String(error); logger( `Existing Chrome reattach failed (${message}); reopening browser to locate the session.`, @@ -666,6 +697,17 @@ async function resumeBrowserSessionViaNewChrome( const captureMarkdown = deps.captureAssistantMarkdown ?? captureAssistantMarkdown; const timeoutMs = resolved.timeoutMs ?? 120_000; const minTurnIndex = await readConversationTurnIndex(Runtime, logger); + if (deps.questionAnswers) { + await answerAssistantQuestions( + Runtime, + client.Input, + logger, + deps.questionAnswers.answers, + deps.questionAnswers.questions, + deps.questionAnswers.submissionState, + ); + await deps.afterQuestionsAnswered?.(); + } const promptEcho = buildPromptEchoMatcher(deps.promptPreview); const answer = await waitForResponse( Runtime, diff --git a/tests/ask-pro/browserRunner.test.ts b/tests/ask-pro/browserRunner.test.ts index 667e9832ba..fbfe5152e1 100644 --- a/tests/ask-pro/browserRunner.test.ts +++ b/tests/ask-pro/browserRunner.test.ts @@ -1,4 +1,6 @@ import { AssistantStoppedError } from "../../src/browser/errors.js"; +import { AssistantQuestionsError } from "../../src/browser/actions/questions.js"; +import { readAskProQuestions, parseQuestionAnswers } from "../../src/ask-pro/questions.js"; import { BrowserAutomationError } from "../../src/browser/errors.js"; import { GitHubConnectionRequiredError } from "../../src/browser/actions/github.js"; import fs from "node:fs/promises"; @@ -937,6 +939,13 @@ describe("ask-pro browser runner", () => { }); test.each([ + { + error: new AssistantQuestionsError([ + { question: "Which format?", options: ["Online", "In person"], freeText: true }, + ]), + status: "awaiting_answers", + reason: "questions", + }, { error: new Error("assistant response timed out"), status: "wait_timed_out", @@ -976,10 +985,29 @@ describe("ask-pro browser runner", () => { const { status } = await readAskProStatus({ cwd, sessionId: session.id }); expect(status).toMatchObject({ status: failure.status.toUpperCase(), reason: failure.reason }); expect(metadata).toMatchObject({ - status: failure.status, + status: failure.status.toLowerCase(), chromeMode: "launched", reason: failure.reason, }); + if (failure.error instanceof AssistantQuestionsError) { + const request = await readAskProQuestions(session.dir); + const answers = JSON.stringify({ requestId: request.requestId, answers: [2] }); + await expect( + resumeAskProBrowserSession({ cwd, sessionId: session.id, answers, temporary: false }), + ).rejects.toThrow(/Cannot change Temporary Chat mode/); + resumeBrowserSessionMock.mockRejectedValueOnce(new Error("The composer has a draft")); + await expect( + resumeAskProBrowserSession({ cwd, sessionId: session.id, answers }), + ).rejects.toThrow(/composer has a draft/); + expect((await readAskProStatus({ cwd, sessionId: session.id })).status.status).toBe( + "AWAITING_ANSWERS", + ); + expect(await readAskProQuestions(session.dir)).toEqual(request); + await resumeAskProBrowserSession({ cwd, sessionId: session.id, answers }); + expect((await readAskProStatus({ cwd, sessionId: session.id })).status.status).toBe( + "COMPLETED", + ); + } }); test("preserves launched Chrome mode for post-launch failures", async () => { @@ -1577,6 +1605,13 @@ describe("ask-pro browser runner", () => { status: "INCOMPLETE_ANSWER", reason: "stopped_without_answer", }, + { + error: new AssistantQuestionsError([ + { question: "Which format?", options: ["Online", "In person"], freeText: true }, + ]), + status: "AWAITING_ANSWERS", + reason: "questions", + }, ])("reattach failure remains resumable: $reason", async (failure) => { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "ask-pro-reattach-timeout-")); tempDirs.push(cwd); @@ -1610,6 +1645,37 @@ describe("ask-pro browser runner", () => { const { status } = await readAskProStatus({ cwd, sessionId: session.id }); expect(status.status).toBe(failure.status); + if (failure.error instanceof AssistantQuestionsError) { + const request = await readAskProQuestions(session.dir); + expect(request.questions).toEqual(failure.error.questions); + expect( + parseQuestionAnswers( + JSON.stringify({ requestId: request.requestId, answers: [2], extra: true }), + request, + ), + ).toEqual([2]); + expect( + parseQuestionAnswers( + JSON.stringify({ requestId: request.requestId, answers: ["A custom format"] }), + request, + ), + ).toEqual(["A custom format"]); + expect(() => + parseQuestionAnswers(JSON.stringify({ requestId: "stale", answers: [1] }), request), + ).toThrow(/different question request/); + expect(() => + parseQuestionAnswers( + JSON.stringify({ requestId: request.requestId, answers: [3] }), + request, + ), + ).toThrow(/Question 1/); + expect(() => + parseQuestionAnswers( + JSON.stringify({ requestId: request.requestId, answers: [] }), + request, + ), + ).toThrow(/one answer per question/); + } const metadata = JSON.parse( await fs.readFile(path.join(session.dir, "browser.json"), "utf8"), ) as { status?: string; chromeMode?: string; reason?: string }; diff --git a/tests/ask-pro/cli.test.ts b/tests/ask-pro/cli.test.ts index c72f365e69..a7abf2a224 100644 --- a/tests/ask-pro/cli.test.ts +++ b/tests/ask-pro/cli.test.ts @@ -356,7 +356,7 @@ describe("ask-pro cli", () => { ); }, 30000); - test.each(["preamble_without_artifacts", "stopped_without_answer"])( + test.each(["preamble_without_artifacts", "stopped_without_answer", "questions"])( "harvest preserves caller recovery choice: %s", async (reason) => { const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "ask-pro-cli-harvest-incomplete-")); @@ -376,9 +376,10 @@ describe("ask-pro cli", () => { const sessionDir = path.join(cwd, ".ask-pro", "sessions", sessions[0]!); const statusPath = path.join(sessionDir, "status.json"); const status = JSON.parse(await fs.readFile(statusPath, "utf8")); + const pendingStatus = reason === "questions" ? "AWAITING_ANSWERS" : "INCOMPLETE_ANSWER"; await fs.writeFile( statusPath, - `${JSON.stringify({ ...status, status: "INCOMPLETE_ANSWER", reason }, null, 2)}\n`, + `${JSON.stringify({ ...status, status: pendingStatus, reason }, null, 2)}\n`, "utf8", ); await fs.writeFile( @@ -386,6 +387,25 @@ describe("ask-pro cli", () => { "I'll inspect the bundle and create the files.\n", "utf8", ); + if (reason === "questions") { + await fs.writeFile( + path.join(sessionDir, "browser.json"), + JSON.stringify({ + status: "awaiting_answers", + temporary: true, + url: "https://chatgpt.com/?temporary-chat=true", + }), + ); + await fs.writeFile( + path.join(sessionDir, "QUESTIONS.json"), + JSON.stringify({ + requestId: "request-1", + questions: [ + { question: "Which format?", options: ["Online", "In person"], freeText: true }, + ], + }), + ); + } const { stdout } = await execFileAsync( process.execPath, @@ -393,7 +413,7 @@ describe("ask-pro cli", () => { { cwd }, ); - expect(stdout).toContain(" state: incomplete_answer\n"); + expect(stdout).toContain(` state: ${pendingStatus.toLowerCase()}\n`); expect(stdout).toContain(` reason: ${reason}\n`); if (reason === "stopped_without_answer") { expect(stdout).toContain(" action: choose_resume_or_retry\n"); @@ -402,8 +422,50 @@ describe("ask-pro cli", () => { "Run the original request again without --resume to start a new chat.", ); } + if (reason === "questions") { + expect(stdout).toContain(" action: answer_questions\n"); + expect(stdout).toContain(' question_1: "Which format?"'); + expect(stdout).toContain(' options_1: "1: Online; 2: In person"'); + expect(stdout).toContain("--answers "); + expect(stdout).not.toContain("I'll inspect the bundle"); + await fs.writeFile( + path.join(cwd, "answers.json"), + JSON.stringify({ requestId: "stale", answers: [1] }), + ); + await expect( + execFileAsync( + process.execPath, + ["--import", tsxLoader, cli, "--resume", sessions[0]!, "--answers", "answers.json"], + { cwd }, + ), + ).rejects.toMatchObject({ stdout: expect.stringContaining("different question request") }); + await fs.writeFile( + path.join(cwd, "answers.json"), + JSON.stringify({ requestId: "request-1", answers: [1] }), + ); + const beforeModeChange = await snapshotFiles(cwd); + await expect( + execFileAsync( + process.execPath, + [ + "--import", + tsxLoader, + cli, + "--resume", + sessions[0]!, + "--answers", + "answers.json", + "--no-temporary", + ], + { cwd }, + ), + ).rejects.toMatchObject({ + stdout: expect.stringContaining("Cannot change Temporary Chat mode"), + }); + expect(await snapshotFiles(cwd)).toEqual(beforeModeChange); + } const updated = JSON.parse(await fs.readFile(statusPath, "utf8")); - expect(updated.status).toBe("INCOMPLETE_ANSWER"); + expect(updated.status).toBe(pendingStatus); }, 30000, ); diff --git a/tests/browser/assistantResponse.test.ts b/tests/browser/assistantResponse.test.ts index 51f1c42e93..27ce50631b 100644 --- a/tests/browser/assistantResponse.test.ts +++ b/tests/browser/assistantResponse.test.ts @@ -7,6 +7,10 @@ import { waitForAssistantResponse, } from "../../src/browser/actions/assistantResponse.js"; import { AssistantStoppedError } from "../../src/browser/errors.js"; +import { + answerAssistantQuestions, + buildQuestionCardsExpression, +} from "../../src/browser/actions/questions.js"; import { submitPrompt } from "../../src/browser/actions/promptComposer.js"; import type { BrowserLogger, ChromeClient } from "../../src/browser/types.js"; @@ -21,6 +25,218 @@ afterEach(() => { }); describe("assistant response actions", () => { + test("captures native search-unit replies outside the legacy main thread", () => { + class Element { + content: Element | null = null; + constructor(readonly innerText: string) {} + get textContent() { + return this.innerText; + } + get innerHTML() { + return `

${this.innerText}

`; + } + getAttribute() { + return null; + } + closest() { + return null; + } + querySelector(selector: string) { + return selector.includes('data-markdown-text-style="assistant-message"') + ? this.content + : null; + } + querySelectorAll() { + return []; + } + } + const turn = new Element("ChatGPT said: Blue\nCANARY_ONE_OK"); + turn.content = new Element("Blue\nCANARY_ONE_OK"); + const document = { querySelectorAll: () => [turn], querySelector: () => null }; + const snapshot = new Function( + "document", + "HTMLElement", + `return ${buildAssistantSnapshotExpressionForTest()}`, + )(document, Element); + expect(snapshot).toMatchObject({ text: "Blue\nCANARY_ONE_OK", turnIndex: 0 }); + }); + test("answers expired cards in the same chat, while preserving an existing composer draft", async () => { + const evaluate = vi + .fn() + .mockResolvedValue({ result: { value: { questions: null, started: false } } }); + const runtime = { evaluate } as unknown as ChromeClient["Runtime"]; + const input = {} as ChromeClient["Input"]; + const logger = vi.fn() as unknown as BrowserLogger; + const questions = [ + { question: "Which format?", options: ["Online", "In person"], freeText: true }, + { question: "Theme?", options: ["Engineering"], freeText: true }, + ]; + await answerAssistantQuestions( + runtime, + input, + logger, + [2, "Incident response\nPostmortems"], + questions, + ); + expect(vi.mocked(submitPrompt).mock.calls[0]?.[1]).toContain( + "1. Which format?\nIn person\n\n2. Theme?\nIncident response\nPostmortems", + ); + evaluate + .mockResolvedValueOnce({ result: { value: { questions: null, started: false } } }) + .mockResolvedValueOnce({ result: { value: { activeValue: "An unsent draft" } } }); + const submissionState = { started: false }; + await expect( + answerAssistantQuestions(runtime, input, logger, [1, "Theme"], questions, submissionState), + ).rejects.toThrow(/composer has a draft/); + expect(submissionState.started).toBe(false); + expect(submitPrompt).toHaveBeenCalledOnce(); + vi.mocked(submitPrompt).mockRejectedValueOnce(new Error("Disconnected during send")); + await expect( + answerAssistantQuestions(runtime, input, logger, [1, "Theme"], questions, submissionState), + ).rejects.toThrow(/Disconnected during send/); + expect(submissionState.started).toBe(true); + submissionState.started = false; + evaluate.mockRejectedValueOnce(new Error("DevTools disconnected")); + await expect( + answerAssistantQuestions(runtime, input, logger, [1, "Theme"], questions, submissionState), + ).rejects.toThrow(/DevTools disconnected/); + expect(submissionState.started).toBe(true); + }); + test.each([1, 2, 3])( + "hands off %i native questions without defaults, then submits choices and free text", + async (count) => { + const questions = Array.from({ length: count }, (_, index) => ({ + question: `Question ${index + 1}?`, + options: index === 0 ? ["Online", "In person"] : ["One hour", "Half day", "Full day"], + freeText: true, + })); + let index = 0; + let active = true; + let canceledCountdown = false; + let submitted: Array | undefined; + const selections = questions.map(() => 0); + const texts = questions.map(() => ""); + class DomEvent { + key?: string; + constructor( + readonly type: string, + fields: object, + ) { + Object.assign(this, fields); + } + } + class InputElement {} + class TextAreaElement {} + const input = { + focus() {}, + get innerText() { + return texts[index]; + }, + get children() { + return texts[index]!.split("\n").map((text) => ({ innerText: text, textContent: text })); + }, + }; + const submit = () => { + submitted = questions.map((_, i) => texts[i] || selections[i]! + 1); + active = false; + }; + const root = { + getBoundingClientRect: () => ({ width: 400, height: 300 }), + closest: () => null, + focus() {}, + get innerText() { + return `${questions[index]!.question}\n${index + 1} of ${count}\n`; + }, + querySelector(selector: string) { + if (selector === ".text-size-chat") return { innerText: questions[index]!.question }; + if (selector.includes("textbox")) return input; + if (selector.includes("data-request-input-dismiss")) return null; + if (selector.includes("data-request-input-skip")) + return { innerText: "Submit", disabled: false, click: submit }; + const previous = selector.includes("Previous"); + return { + disabled: previous ? index === 0 : index + 1 === count, + click() { + index += previous ? -1 : 1; + }, + }; + }, + querySelectorAll() { + return questions[index]!.options.map((option, choice) => ({ + getAttribute(name: string) { + return name === "aria-label" ? option : String(selections[index] === choice); + }, + click() { + selections[index] = choice; + if (index + 1 === count) submit(); + else index++; + }, + })); + }, + dispatchEvent(event: DomEvent) { + if (event.type === "pointerdown") canceledCountdown = true; + if (event.key === "ArrowDown") + selections[index] = Math.min( + selections[index]! + 1, + questions[index]!.options.length - 1, + ); + if (event.key === "ArrowUp") selections[index] = Math.max(selections[index]! - 1, 0); + if (event.key === "Enter") { + submitted = questions.map((_, i) => texts[i] || selections[i]! + 1); + active = false; + } + }, + }; + const document = { + querySelectorAll: () => (active ? [root] : []), + createRange: () => ({ selectNodeContents() {} }), + execCommand(_command: string, _show: boolean, text: string) { + texts[index] = text; + }, + }; + const run = async (answers?: Array, expected = questions) => { + const value = await new Function( + "document", + "window", + "PointerEvent", + "KeyboardEvent", + "HTMLInputElement", + "HTMLTextAreaElement", + `return ${buildQuestionCardsExpression(answers, expected)}`, + )( + document, + { getSelection: () => null }, + DomEvent, + DomEvent, + InputElement, + TextAreaElement, + ); + if (answers && value.error) throw new Error(value.error); + return answers ? value.questions : value; + }; + expect(await run()).toEqual(questions); + expect(canceledCountdown).toBe(true); + expect(submitted).toBeUndefined(); + expect(selections).toEqual(questions.map(() => 0)); + await expect( + run( + questions.map(() => 1), + [{ ...questions[0]!, question: "Old question?" }], + ), + ).rejects.toThrow(/questions changed/); + expect(submitted).toBeUndefined(); + const answers = + count === 1 + ? [2] + : count === 2 + ? [2, "Custom theme\nwith details"] + : [2, 3, "Custom theme\nwith details"]; + expect(await run(answers)).toEqual(questions); + expect(submitted).toEqual(answers); + expect(await run()).toBeNull(); + }, + ); + test("captures the full current ChatGPT answer instead of its last inline citation", () => { const answer = { innerText: "Complete answer with several paragraphs.", @@ -390,6 +606,7 @@ describe("assistant response actions", () => { const evaluate = vi .fn() .mockResolvedValueOnce({ result: { value: null } }) + .mockResolvedValueOnce({ result: { value: null } }) .mockResolvedValueOnce({ result: { value: { matchedControls: 0, truncated: false, controls: [] } }, }); diff --git a/tests/browser/reattach.test.ts b/tests/browser/reattach.test.ts index 42b2eb22c8..692d99b3fa 100644 --- a/tests/browser/reattach.test.ts +++ b/tests/browser/reattach.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterEach, describe, expect, test, vi } from "vitest"; import { resumeBrowserSession } from "../../src/browser/reattach.js"; import { AssistantStoppedError } from "../../src/browser/errors.js"; +import { AssistantQuestionsError } from "../../src/browser/actions/questions.js"; import { buildConversationUrl, extractConversationIdFromUrl, @@ -46,9 +47,16 @@ describe("reattach browser lease", () => { ).toBeNull(); expect(extractConversationIdFromUrl(saved)).toBe("795d079e-6c4d-4334-b0ae-4534c70305bd"); }); - test("returns an exhausted continuation to the caller instead of relaunching", async () => { + test.each([ + new AssistantStoppedError(3), + new AssistantQuestionsError([{ question: "Which color?", options: ["Blue"], freeText: true }]), + ])("returns $name to the caller and closes its connection without relaunching", async (error) => { const recoverSession = vi.fn(); + let connected = true; const client = { + close: async () => { + connected = false; + }, Runtime: { evaluate: async ({ expression }: { expression: string }) => ({ result: { value: expression === "location.href" ? "https://chatgpt.com/c/test" : 2 }, @@ -67,27 +75,29 @@ describe("reattach browser lease", () => { ], connect: async () => client, waitForAssistantResponse: async () => { - throw new AssistantStoppedError(3); + throw error; }, recoverSession, }, ), - ).rejects.toBeInstanceOf(AssistantStoppedError); + ).rejects.toBe(error); + expect(connected).toBe(false); expect(recoverSession).not.toHaveBeenCalled(); }); - test("releases its lease before falling back after an attach failure", async () => { - const profileDir = await fs.mkdtemp(path.join(os.tmpdir(), "ask-pro-reattach-lease-")); - tempDirs.push(profileDir); - const recovered = { - answerText: "recovered", - answerMarkdown: "recovered", - chromeMode: "relaunched" as const, - }; - const recoverSession = vi.fn().mockResolvedValue(recovered); + test.each(["ordinary", "pending answers", "uncertain submission"])( + "releases its lease after an attach failure: %s", + async (mode) => { + const profileDir = await fs.mkdtemp(path.join(os.tmpdir(), "ask-pro-reattach-lease-")); + tempDirs.push(profileDir); + const recovered = { + answerText: "recovered", + answerMarkdown: "recovered", + chromeMode: "relaunched" as const, + }; + const recoverSession = vi.fn().mockResolvedValue(recovered); - await expect( - resumeBrowserSession( + const result = resumeBrowserSession( { chromePort: 9222, userDataDir: profileDir }, { manualLogin: true, @@ -100,11 +110,21 @@ describe("reattach browser lease", () => { throw new Error("attach failed"); }, recoverSession, + questionAnswers: + mode === "ordinary" + ? undefined + : { + answers: [1], + questions: [{ question: "Color?", options: ["Blue"], freeText: true }], + submissionState: { started: mode === "uncertain submission" }, + }, }, - ), - ).resolves.toEqual(recovered); + ); + if (mode === "uncertain submission") await expect(result).rejects.toThrow("attach failed"); + else await expect(result).resolves.toEqual(recovered); - expect(recoverSession).toHaveBeenCalledOnce(); - await expect(fs.readdir(path.join(profileDir, "ask-pro-browser-runs"))).resolves.toEqual([]); - }); + expect(recoverSession).toHaveBeenCalledTimes(mode === "uncertain submission" ? 0 : 1); + await expect(fs.readdir(path.join(profileDir, "ask-pro-browser-runs"))).resolves.toEqual([]); + }, + ); }); From eae972a2389f21881e693ec45e69fb4a9b326cf8 Mon Sep 17 00:00:00 2001 From: Jonathan Liebig Date: Mon, 12 Oct 2026 16:29:10 +0200 Subject: [PATCH 2/2] Fix expired question answer capture and pre-send retries --- CHANGELOG.md | 3 ++ docs/windows-work.md | 4 +- src/ask-pro/questions.ts | 3 ++ src/browser/actions/promptComposer.ts | 6 +++ src/browser/actions/questions.ts | 14 +++++- src/browser/reattach.ts | 10 ++-- tests/ask-pro/browserRunner.test.ts | 3 ++ tests/browser/assistantResponse.test.ts | 11 +++-- tests/browser/promptComposer.test.ts | 60 +++++++++++++++++++++- tests/browser/reattach.test.ts | 66 +++++++++++++++++++++++++ 10 files changed, 168 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b0cc2ca88..63fc8123ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ ### Fixed +- Wait for the new reply when answering expired question cards, and keep + answers retryable when the composer rejects input before sending. + - Capture replies in ChatGPT's current search-unit message layout and close the controller's DevTools connection when a resumed session hands off questions. diff --git a/docs/windows-work.md b/docs/windows-work.md index 9f7c5f033d..ca65c91d8a 100644 --- a/docs/windows-work.md +++ b/docs/windows-work.md @@ -84,7 +84,9 @@ Future Windows gotchas belong here. browses the cards without submitting defaults, retains Chrome, and returns `AWAITING_ANSWERS`. Use a UTF-8 JSON file with `--resume ... --answers ...`; this avoids PowerShell quoting problems with multiline free-text answers. - Expired cards are answered by replying in the same conversation. + Expired cards are answered by replying in the same conversation; capture waits + for that new reply. A composer that rejects input before sending leaves answers + pending for retry. - October 7: the live Select model view exposes `GPT-6`, `GPT-5.6 Sol`, and `GPT-5.5` instead of `Latest`. The default picker prefers `Latest` when present, diff --git a/src/ask-pro/questions.ts b/src/ask-pro/questions.ts index e822272a91..45606a4d0d 100644 --- a/src/ask-pro/questions.ts +++ b/src/ask-pro/questions.ts @@ -33,6 +33,9 @@ export function parseQuestionAnswers( request: AskProQuestions, ): Array { const value = JSON.parse(raw); + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Answers must be a JSON object with requestId and answers."); + } if (value?.requestId !== request.requestId) throw new Error( "Answers belong to a different question request; read the current QUESTIONS.json.", diff --git a/src/browser/actions/promptComposer.ts b/src/browser/actions/promptComposer.ts index 6375985461..774e34d7c0 100644 --- a/src/browser/actions/promptComposer.ts +++ b/src/browser/actions/promptComposer.ts @@ -277,6 +277,7 @@ export async function submitPrompt( baselineTurns?: number | null; inputTimeoutMs?: number | null; afterSubmit?: () => Promise; + submissionState?: { started: boolean }; }, prompt: string, logger: BrowserLogger, @@ -417,6 +418,7 @@ export async function submitPrompt( logger, deps?.attachmentNames, deps.inputTimeoutMs ?? undefined, + deps.submissionState, ); if (!clicked) { const canSubmitViaEnter = await canSubmitPromptViaEnter(runtime); @@ -430,6 +432,7 @@ export async function submitPrompt( }, ); } + if (deps.submissionState) deps.submissionState.started = true; await input.dispatchKeyEvent({ type: "keyDown", ...ENTER_KEY_EVENT, @@ -668,6 +671,7 @@ async function attemptSendButton( logger?: BrowserLogger, attachmentNames?: string[], inputTimeoutMs?: number, + submissionState?: { started: boolean }, ): Promise { const needAttachment = Array.isArray(attachmentNames) && attachmentNames.length > 0; const deadline = @@ -707,6 +711,7 @@ async function attemptSendButton( continue; } } + if (submissionState) submissionState.started = true; const { result } = await Runtime.evaluate({ expression: buildComposerSendClickExpression(), returnByValue: true, @@ -714,6 +719,7 @@ async function attemptSendButton( if (result.value === "clicked") { return true; } + if (submissionState) submissionState.started = false; if (result.value === "stop-button") { throw new BrowserAutomationError("Refusing to click ChatGPT stop control as a send button.", { stage: "submit-prompt", diff --git a/src/browser/actions/questions.ts b/src/browser/actions/questions.ts index 06cdef05a5..9ed3665ff4 100644 --- a/src/browser/actions/questions.ts +++ b/src/browser/actions/questions.ts @@ -1,6 +1,7 @@ import type { ChromeClient } from "../types.js"; import type { BrowserLogger } from "../types.js"; import { readComposerSnapshot, submitPrompt } from "./promptComposer.js"; +import { CONVERSATION_TURN_SELECTOR } from "../constants.js"; export interface AssistantQuestion { question: string; @@ -214,7 +215,16 @@ export async function answerAssistantQuestions( "The ChatGPT composer has a draft; clear it before answering expired questions.", ); const prompt = `Answers to your clarification questions:\n\n${expected.map((question, index) => `${index + 1}. ${question.question}\n${typeof answers[index] === "number" ? question.options[(answers[index] as number) - 1] : answers[index]}`).join("\n\n")}\n\nContinue the original request using these answers.`; - submissionState.started = true; - await submitPrompt({ runtime: Runtime, input: Input }, prompt, logger); + const { result } = await Runtime.evaluate({ + expression: `document.querySelectorAll(${JSON.stringify(CONVERSATION_TURN_SELECTOR)}).length`, + returnByValue: true, + }); + const baselineTurns = result.value as number; + await submitPrompt( + { runtime: Runtime, input: Input, baselineTurns, submissionState }, + prompt, + logger, + ); + return baselineTurns; } } diff --git a/src/browser/reattach.ts b/src/browser/reattach.ts index 3263235ffb..3285faaa7c 100644 --- a/src/browser/reattach.ts +++ b/src/browser/reattach.ts @@ -353,10 +353,10 @@ export async function resumeBrowserSession( "Reattach target did not respond", ); await ensureConversationOpen(); - const minTurnIndex = await readConversationTurnIndex(Runtime, logger); + let minTurnIndex = await readConversationTurnIndex(Runtime, logger); if (deps.questionAnswers) { answeringQuestions = true; - await answerAssistantQuestions( + const replyBoundary = await answerAssistantQuestions( Runtime, Input, logger, @@ -364,6 +364,7 @@ export async function resumeBrowserSession( deps.questionAnswers.questions, deps.questionAnswers.submissionState, ); + if (replyBoundary !== undefined) minTurnIndex = replyBoundary; await deps.afterQuestionsAnswered?.(); } const promptEcho = buildPromptEchoMatcher(deps.promptPreview); @@ -696,9 +697,9 @@ async function resumeBrowserSessionViaNewChrome( const waitForResponse = deps.waitForAssistantResponse ?? waitForAssistantResponse; const captureMarkdown = deps.captureAssistantMarkdown ?? captureAssistantMarkdown; const timeoutMs = resolved.timeoutMs ?? 120_000; - const minTurnIndex = await readConversationTurnIndex(Runtime, logger); + let minTurnIndex = await readConversationTurnIndex(Runtime, logger); if (deps.questionAnswers) { - await answerAssistantQuestions( + const replyBoundary = await answerAssistantQuestions( Runtime, client.Input, logger, @@ -706,6 +707,7 @@ async function resumeBrowserSessionViaNewChrome( deps.questionAnswers.questions, deps.questionAnswers.submissionState, ); + if (replyBoundary !== undefined) minTurnIndex = replyBoundary; await deps.afterQuestionsAnswered?.(); } const promptEcho = buildPromptEchoMatcher(deps.promptPreview); diff --git a/tests/ask-pro/browserRunner.test.ts b/tests/ask-pro/browserRunner.test.ts index fbfe5152e1..22beca8098 100644 --- a/tests/ask-pro/browserRunner.test.ts +++ b/tests/ask-pro/browserRunner.test.ts @@ -1648,6 +1648,9 @@ describe("ask-pro browser runner", () => { if (failure.error instanceof AssistantQuestionsError) { const request = await readAskProQuestions(session.dir); expect(request.questions).toEqual(failure.error.questions); + for (const value of [null, [], 1, "answer"]) { + expect(() => parseQuestionAnswers(JSON.stringify(value), request)).toThrow(/JSON object/); + } expect( parseQuestionAnswers( JSON.stringify({ requestId: request.requestId, answers: [2], extra: true }), diff --git a/tests/browser/assistantResponse.test.ts b/tests/browser/assistantResponse.test.ts index 27ce50631b..d40fadadd7 100644 --- a/tests/browser/assistantResponse.test.ts +++ b/tests/browser/assistantResponse.test.ts @@ -61,9 +61,9 @@ describe("assistant response actions", () => { expect(snapshot).toMatchObject({ text: "Blue\nCANARY_ONE_OK", turnIndex: 0 }); }); test("answers expired cards in the same chat, while preserving an existing composer draft", async () => { - const evaluate = vi - .fn() - .mockResolvedValue({ result: { value: { questions: null, started: false } } }); + const evaluate = vi.fn().mockImplementation(async ({ expression }: { expression: string }) => ({ + result: { value: expression.endsWith(".length") ? 2 : { questions: null, started: false } }, + })); const runtime = { evaluate } as unknown as ChromeClient["Runtime"]; const input = {} as ChromeClient["Input"]; const logger = vi.fn() as unknown as BrowserLogger; @@ -90,7 +90,10 @@ describe("assistant response actions", () => { ).rejects.toThrow(/composer has a draft/); expect(submissionState.started).toBe(false); expect(submitPrompt).toHaveBeenCalledOnce(); - vi.mocked(submitPrompt).mockRejectedValueOnce(new Error("Disconnected during send")); + vi.mocked(submitPrompt).mockImplementationOnce(async (deps) => { + deps.submissionState!.started = true; + throw new Error("Disconnected during send"); + }); await expect( answerAssistantQuestions(runtime, input, logger, [1, "Theme"], questions, submissionState), ).rejects.toThrow(/Disconnected during send/); diff --git a/tests/browser/promptComposer.test.ts b/tests/browser/promptComposer.test.ts index d1e296309d..dccb007383 100644 --- a/tests/browser/promptComposer.test.ts +++ b/tests/browser/promptComposer.test.ts @@ -1,8 +1,66 @@ import { runInNewContext } from "node:vm"; import { describe, expect, test, vi } from "vitest"; -import { __test__ } from "../../src/browser/actions/promptComposer.js"; +import { __test__, submitPrompt } from "../../src/browser/actions/promptComposer.js"; describe("prompt composer actions", () => { + test("keeps answers retryable when the composer rejects input before sending", async () => { + const evaluate = vi.fn(async ({ expression }: { expression: string }) => ({ + result: { + value: expression.includes("document.readyState") + ? { ready: true, composer: true } + : { healthy: false, reason: "disabled", activeDisabled: true }, + }, + })); + const insertText = vi.fn(); + const submissionState = { started: false }; + await expect( + submitPrompt( + { runtime: { evaluate } as never, input: { insertText } as never, submissionState }, + "Answers", + vi.fn<(message: string) => void>(), + ), + ).rejects.toThrow("not accepting input"); + expect(submissionState.started).toBe(false); + expect(insertText).not.toHaveBeenCalled(); + }); + + test("tracks whether a send click happened or may have happened", async () => { + const submissionState = { started: false }; + const evaluate = vi.fn().mockResolvedValue({ result: { value: "missing" } }); + await expect( + __test__.attemptSendButton( + { evaluate } as never, + undefined, + undefined, + undefined, + submissionState, + ), + ).resolves.toBe(false); + expect(submissionState.started).toBe(false); + evaluate.mockResolvedValueOnce({ result: { value: "clicked" } }); + await expect( + __test__.attemptSendButton( + { evaluate } as never, + undefined, + undefined, + undefined, + submissionState, + ), + ).resolves.toBe(true); + expect(submissionState.started).toBe(true); + submissionState.started = false; + evaluate.mockRejectedValueOnce(new Error("Disconnected during click")); + await expect( + __test__.attemptSendButton( + { evaluate } as never, + undefined, + undefined, + undefined, + submissionState, + ), + ).rejects.toThrow("Disconnected"); + expect(submissionState.started).toBe(true); + }); test("accepts an active answer when turn selectors miss the submitted prompt", async () => { const evaluate = vi.fn(async (_: { expression: string }) => ({ result: { diff --git a/tests/browser/reattach.test.ts b/tests/browser/reattach.test.ts index 692d99b3fa..87f44e7c44 100644 --- a/tests/browser/reattach.test.ts +++ b/tests/browser/reattach.test.ts @@ -5,6 +5,8 @@ import { afterEach, describe, expect, test, vi } from "vitest"; import { resumeBrowserSession } from "../../src/browser/reattach.js"; import { AssistantStoppedError } from "../../src/browser/errors.js"; import { AssistantQuestionsError } from "../../src/browser/actions/questions.js"; +import { readAssistantSnapshot } from "../../src/browser/actions/assistantResponse.js"; +import { submitPrompt } from "../../src/browser/actions/promptComposer.js"; import { buildConversationUrl, extractConversationIdFromUrl, @@ -12,12 +14,76 @@ import { import type { ChromeClient } from "../../src/browser/types.js"; const tempDirs: string[] = []; +vi.mock("../../src/browser/actions/promptComposer.js", async (importOriginal) => ({ + ...(await importOriginal()), + submitPrompt: vi.fn(), +})); afterEach(async () => { + vi.clearAllMocks(); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); describe("reattach browser lease", () => { + test.each(["native", "expired"])( + "captures the answer after %s questions are submitted", + async (mode) => { + const questions = [{ question: "Color?", options: ["Blue"], freeText: true }]; + let turns = 2; + let snapshot = { text: "Previous reply", turnIndex: 1 }; + const client = { + close: vi.fn().mockResolvedValue(undefined), + Input: {}, + Runtime: { + evaluate: async ({ expression }: { expression: string }) => ({ + result: { + value: + expression === "location.href" + ? "https://chatgpt.com/c/test" + : expression.endsWith(".length") + ? turns + : expression.includes("const MIN_TURN_INDEX") + ? snapshot + : expression.includes("const submission =") + ? { + questions: mode === "native" ? questions : null, + started: mode === "native", + } + : 2, + }, + }), + }, + } as unknown as ChromeClient; + // Sending commits a new user turn before the next assistant turn appears. + vi.mocked(submitPrompt).mockImplementationOnce(async () => { + turns = 3; + return turns; + }); + const result = await resumeBrowserSession( + { chromePort: 9222, conversationId: "test" }, + {}, + vi.fn<(message: string) => void>(), + { + listTargets: async () => [ + { targetId: "test", type: "page", url: "https://chatgpt.com/c/test" }, + ], + connect: async () => client, + questionAnswers: { answers: [1], questions, submissionState: { started: false } }, + waitForAssistantResponse: async (runtime, _timeout, _logger, minTurnIndex) => { + if (mode === "expired") + expect(await readAssistantSnapshot(runtime, minTurnIndex)).toBeNull(); + snapshot = { text: "Fresh answer: Blue", turnIndex: mode === "native" ? 1 : 3 }; + const answer = await readAssistantSnapshot(runtime, minTurnIndex); + expect(answer?.text).toBe("Fresh answer: Blue"); + return { text: answer!.text!, meta: {} }; + }, + captureAssistantMarkdown: async () => null, + }, + ); + expect(result.answerText).toBe("Fresh answer: Blue"); + expect(client.close).toHaveBeenCalledOnce(); + }, + ); test("discards a stored provisional route before recovering an interrupted session", async () => { const recoverSession = vi.fn().mockResolvedValue({ answerText: "Recovered answer",