From 3639526a5713a1adaa5c74dba03bec54e86d08c7 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:54:35 -0400 Subject: [PATCH 1/8] ci: capture hosted WebKit VoiceOver journeys Co-Authored-By: Claude Opus 5.5 --- .github/workflows/voiceover.yml | 55 ++++++++++++ package-lock.json | 73 ++++++++++++++- package.json | 6 +- playwright.voiceover.config.js | 14 +++ tests/voiceover/README.md | 20 +++++ tests/voiceover/control.spec.js | 20 +++++ tests/voiceover/journeys.spec.js | 149 +++++++++++++++++++++++++++++++ tests/voiceover/session.js | 102 +++++++++++++++++++++ 8 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/voiceover.yml create mode 100644 playwright.voiceover.config.js create mode 100644 tests/voiceover/README.md create mode 100644 tests/voiceover/control.spec.js create mode 100644 tests/voiceover/journeys.spec.js create mode 100644 tests/voiceover/session.js diff --git a/.github/workflows/voiceover.yml b/.github/workflows/voiceover.yml new file mode 100644 index 0000000..fdedb94 --- /dev/null +++ b/.github/workflows/voiceover.yml @@ -0,0 +1,55 @@ +name: VoiceOver evidence +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/voiceover.yml' + - 'playwright.voiceover.config.js' + - 'tests/voiceover/**' +permissions: + contents: read +concurrency: + group: voiceover-${{ github.ref }} + cancel-in-progress: true +jobs: + voiceover: + runs-on: macos-26 + timeout-minutes: 100 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '22.23.3' + cache: npm + # Matches Guidepup's macOS 26 WebKit CI setup. + - run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 setup --ci --macos-record + - run: npm ci --no-audit --no-fund --no-progress + - run: npx playwright install webkit + - run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 install + - name: Record environment + run: | + mkdir -p test-results/voiceover-environment + { + sw_vers + echo "Runner image: $ImageOS $ImageVersion" + node --version + npm --version + npx playwright --version + npm ls @guidepup/guidepup + /usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' /System/Library/CoreServices/VoiceOver.app/Contents/Info.plist + /usr/libexec/PlistBuddy -c 'Print :CFBundleVersion' /System/Library/CoreServices/VoiceOver.app/Contents/Info.plist + } > test-results/voiceover-environment/versions.txt + say -v '?' > test-results/voiceover-environment/voices.txt + - run: npm run demo + - name: Positive control, required before journeys + run: npm run test:voiceover -- --grep 'positive control' + env: + VO_PHASE: control + - name: Record the nine patterns in English and French + run: npm run test:voiceover -- --grep-invert 'positive control' + - uses: actions/upload-artifact@v4 + if: always() + with: + name: voiceover-${{ github.run_id }}-${{ github.run_attempt }} + path: test-results/ + retention-days: 14 diff --git a/package-lock.json b/package-lock.json index 0d836f2..7b9bdda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "@axe-core/playwright": "^4.10.2", "@fontsource/source-sans-3": "^5.3.0", "@fontsource/source-serif-4": "^5.3.0", - "@playwright/test": "^1.56.0", + "@guidepup/guidepup": "0.35.0", + "@playwright/test": "1.63.0", "@tabler/icons": "^3.49.0", "typescript": "^5.9.3" }, @@ -51,6 +52,17 @@ "url": "https://github.com/sponsors/ayuhito" } }, + "node_modules/@guidepup/guidepup": { + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@guidepup/guidepup/-/guidepup-0.35.0.tgz", + "integrity": "sha512-US64a++RucCtLfoq/DqnjzaHYpKzERVYmhtsWViz38MyklICICDpb50AKLn9sbxUwTceK2rS0fjDy7fYpWK/QA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "plist": "^4.0.0" + } + }, "node_modules/@playwright/test": { "version": "1.63.0", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", @@ -78,6 +90,16 @@ "url": "https://github.com/sponsors/codecalm" } }, + "node_modules/@xmldom/xmldom": { + "version": "0.9.12", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.9.12.tgz", + "integrity": "sha512-5AXjrcMClTryPe9LgZrygpB1lj7s0S9E0+W+AHaVKAVyHanafK86iPSvG5xHVSp/jC+VH1UXu0TAEmY279xH7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.6" + } + }, "node_modules/axe-core": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", @@ -88,6 +110,31 @@ "node": ">=4" } }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, "node_modules/playwright": { "version": "1.63.0", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", @@ -117,6 +164,20 @@ "node": ">=20" } }, + "node_modules/plist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/plist/-/plist-4.0.0.tgz", + "integrity": "sha512-4dOqNo0Y2NpfSf9q4+zr4bh7pzNWeckIam34Z0KYJhg8qtNNfh59VbD+Yna5SjwcxawVvLKx5w5FtuCijpEF4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@xmldom/xmldom": "^0.9.10", + "xmlbuilder": "^15.1.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -130,6 +191,16 @@ "engines": { "node": ">=14.17" } + }, + "node_modules/xmlbuilder": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0" + } } } } diff --git a/package.json b/package.json index 05dba26..b75e20b 100644 --- a/package.json +++ b/package.json @@ -15,13 +15,15 @@ "demo": "node scripts/build-demo.mjs", "test:browser": "playwright test --workers=2", "preview:deploy": "npm run demo && npx wrangler@4 deploy", - "icons": "node scripts/build-icons.mjs" + "icons": "node scripts/build-icons.mjs", + "test:voiceover": "playwright test --config=playwright.voiceover.config.js" }, "devDependencies": { "@axe-core/playwright": "^4.10.2", "@fontsource/source-sans-3": "^5.3.0", "@fontsource/source-serif-4": "^5.3.0", - "@playwright/test": "^1.56.0", + "@guidepup/guidepup": "0.35.0", + "@playwright/test": "1.63.0", "@tabler/icons": "^3.49.0", "typescript": "^5.9.3" } diff --git a/playwright.voiceover.config.js b/playwright.voiceover.config.js new file mode 100644 index 0000000..4dd9be9 --- /dev/null +++ b/playwright.voiceover.config.js @@ -0,0 +1,14 @@ +import { defineConfig } from '@playwright/test'; + +const port = Number(process.env.LP_PORT ?? 4187); +export default defineConfig({ + testDir: './tests/voiceover', + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 20 * 60 * 1000, + outputDir: `test-results/voiceover-${process.env.VO_PHASE ?? 'journeys'}`, + reporter: [['list']], + use: { browserName: 'webkit', headless: false, baseURL: `http://127.0.0.1:${port}`, viewport: { width: 1280, height: 900 } }, + webServer: { command: 'node scripts/serve-demo.mjs', url: `http://127.0.0.1:${port}/index.html`, reuseExistingServer: false, env: { LP_PORT: String(port) } } +}); diff --git a/tests/voiceover/README.md b/tests/voiceover/README.md new file mode 100644 index 0000000..6ded58f --- /dev/null +++ b/tests/voiceover/README.md @@ -0,0 +1,20 @@ +# Hosted VoiceOver evidence + +The separate VoiceOver workflow runs on GitHub-hosted macOS 26 with headed Playwright WebKit and one worker. It runs only when its workflow, config or tests change in a pull request, or when dispatched manually. It builds the standard demo with `npm run demo`. The existing browser checks remain separate. + +Guidepup 0.35.0, its setup tool 0.29.1 and Playwright 1.63.0 are pinned. Web-content entry follows [Guidepup's MIT fixture](https://github.com/guidepup/guidepup/blob/d5c9d8059954f214b82689dffcb5866a9b55fe6c/examples/playwright-voiceover/voiceover-test.ts). This tests Playwright WebKit, not installed Safari. + +The positive control must capture the heading, button name and role, and one live-message occurrence before any pattern journey runs. Each English and French journey records native VoiceOver keyboard actions, spoken phrases, cursor text, DOM focus and status mutations. The artifacts include per-step JSON, phrase logs, HTML, accessibility snapshots and screenshots. Journey errors are recorded without failing CI, so a green job means the control passed and evidence was collected, not that the pages passed an accessibility review. + +Navigation uses Guidepup's initial capture; result actions use full capture. Initial capture can miss later phrases in a long utterance. Guidepup polls VoiceOver's last phrase and combines changed phrases; consecutive identical announcements may not appear separately. Interpret announcement counts with those limits. Installed voices and portable preferences are recorded, but phrase text cannot prove French pronunciation. + +Run on a dedicated macOS desktop configured for Guidepup: + +```sh +npm ci +npm run demo +VO_PHASE=control npm run test:voiceover -- --grep 'positive control' +npm run test:voiceover -- --grep-invert 'positive control' +``` + +Read the artifacts against each pattern's README before assigning a page verdict. A navigation error needs investigation rather than an automatic page failure. diff --git a/tests/voiceover/control.spec.js b/tests/voiceover/control.spec.js new file mode 100644 index 0000000..feeeb89 --- /dev/null +++ b/tests/voiceover/control.spec.js @@ -0,0 +1,20 @@ +import { test, expect } from '@playwright/test'; +import { startSession } from './session.js'; + +test('positive control captures heading, button name and role, and one live message', async ({ page }, info) => { + await page.goto('/index.html'); + await page.setContent('VoiceOver positive control

Control heading

'); + const session = await startSession(page, info); + try { + await session.observe(); await session.enter(); + await session.next('Control button name and role', true); + const arrival = session.rows.flatMap(row => row.speech).join('\n'); + expect(arrival).toContain('Control heading'); + expect(arrival).toMatch(/Control action.*button|button.*Control action/i); + await session.activate('button', 'Activate control button', 'Control+Alt+Space'); + await session.snapshot('control-result'); + const activated = session.rows.filter(row => row.step === 'Activate control button').flatMap(row => row.speech).join('\n'); + expect(activated.split('Control live message').length - 1).toBe(1); + expect(await page.locator('#status').textContent()).toBe('Control live message'); + } finally { await session.stop(); } +}); diff --git a/tests/voiceover/journeys.spec.js b/tests/voiceover/journeys.spec.js new file mode 100644 index 0000000..708a551 --- /dev/null +++ b/tests/voiceover/journeys.spec.js @@ -0,0 +1,149 @@ +import { test } from '@playwright/test'; +import { readFile, writeFile } from 'node:fs/promises'; +import { startSession } from './session.js'; + +const patterns = ['dont-know', 'first-answer', 'formats', 'highlight', 'retrieval-sheet', 'review-prompts', 'self-check', 'test-out', 'write-distractors']; +for (const pattern of patterns) for (const lang of ['en', 'fr']) { + test(`${pattern} ${lang}`, async ({ page, browser }, info) => { + await page.goto(`/${pattern}/${lang}.html`); + await page.waitForFunction(() => window.lpReady); + await writeFile(info.outputPath('browser-version.txt'), browser.version()); + const content = JSON.parse(await readFile(new URL(`../../patterns/${pattern}/examples/${lang}.json`, import.meta.url), 'utf8')); + const s = await startSession(page, info); + try { + await s.observe(); await s.enter(); + for (let i = 0; i < 8; i++) await s.next(`Arrival instructions and labels ${i + 1}`); + await s.snapshot('arrival'); + await journey(s, page, pattern, lang, content); + await s.snapshot('result'); + await s.read('Local feedback and closing summary'); + await writeFile(info.outputPath('journey-result.json'), JSON.stringify({ complete: true, pattern, lang })); + } catch (error) { + // A page or navigation finding is evidence, not a CI assertion failure. + await writeFile(info.outputPath('journey-result.json'), JSON.stringify({ complete: false, pattern, lang, error: error.stack })); + await s.snapshot('blocked'); + console.error(`${pattern} ${lang}: ${error.stack}`); + } finally { await s.stop(); } + }); +} + +async function journey(s, page, pattern, lang, content) { + async function choose(inputs, index, label, inspect = false) { + await s.seek(`[id="${await inputs.first().getAttribute('id')}"]`, label); + await s.key('Space', `${label}: select A`); + if (inspect) for (let i = 1; i < await inputs.count(); i++) await s.key('ArrowDown', `${label}: option ${i + 1}`); + const active = await inputs.evaluateAll(elements => elements.indexOf(document.activeElement)); + if (active < 0) throw new Error(`${label}: radio focus lost`); + const distance = (index - active + await inputs.count()) % await inputs.count(); + for (let i = 0; i < distance; i++) await s.key('ArrowDown', `${label}: choose answer ${i + 1}`); + await s.key('Space', `${label}: confirm`); + } + if (pattern === 'dont-know') { + await s.activate('[data-lp-check]', 'Incomplete submit'); await s.snapshot('incomplete'); + for (let q = 0; q < content.questions.length; q++) { + const inputs = page.locator('fieldset').nth(q).locator('input'); + const index = q === 1 ? 0 : q === 2 ? await inputs.count() - 1 : content.questions[q].options.findIndex(option => option.id === content.questions[q].correct); + await choose(inputs, index, `Question ${q + 1}`, true); + } + await s.activate('[data-lp-check]', 'Submit mixed right wrong and uncertain answers'); + await s.snapshot('mixed'); await s.read('Mixed answer local feedback'); + await s.activate('[data-lp-restart]', 'Start over'); + for (let q = 0; q < content.questions.length; q++) await choose(page.locator('fieldset').nth(q).locator('input'), content.questions[q].options.findIndex(option => option.id === content.questions[q].correct), `Correct question ${q + 1}`); + await s.activate('[data-lp-check]', 'Submit all correct answers'); + } else if (pattern === 'self-check') { + await s.activate('[data-lp-check]', 'Empty answer submit'); await s.snapshot('incomplete'); + await s.seek('textarea', 'Answer field'); + await s.type(lang === 'fr' ? 'Bonjour Sam, les données sont arrivées tard. Puis-je remettre le rapport demain?' : 'Hi Sam, the data arrived late. Could I submit the report tomorrow?', 'Write answer'); + await s.activate('[data-lp-check]', 'Check answer'); + const boxes = page.locator('input[type="checkbox"]'); + for (let i = 0; i < await boxes.count(); i++) { + await s.seek(`[id="${await boxes.nth(i).getAttribute('id')}"]`, `Checklist ${i + 1}`); + if (i === 0) await s.key('Space', 'Include first part'); + } + await s.activate('[data-lp-show]', 'Submit partial checklist'); await s.snapshot('partial'); + await s.read('Partial checklist local hints and model'); + for (let i = 0; i < await boxes.count(); i++) { + await s.seek(`[id="${await boxes.nth(i).getAttribute('id')}"]`, `Checklist ${i + 1} revisit`); + if (!await boxes.nth(i).isChecked()) await s.key('Space', `Include part ${i + 1}`); + } + await s.activate('[data-lp-show]', 'Submit complete checklist'); + } else if (pattern === 'first-answer') { + await s.activate('[data-lp-save-first]', 'Blank first answer submit'); await s.snapshot('incomplete'); + await s.seek('[data-lp-first-input]', 'First answer'); + await s.type(lang === 'fr' ? "Arrêtez de m'interrompre." : 'Stop interrupting me.', 'Write first answer'); + await s.activate('[data-lp-save-first]', 'Save first answer'); + await s.activate('[data-lp-skip]', 'Skip to end'); + await s.activate('[data-lp-compare]', 'Blank current answer submit'); + await s.seek('[data-lp-now-input]', 'Current answer'); + await s.type(lang === 'fr' ? 'Puis-je finir mon idée avant de vous écouter?' : 'Can I finish my thought, then hear your view?', 'Write current answer'); + await s.activate('[data-lp-compare]', 'Compare answers'); + for (const box of await page.locator('input[type="checkbox"]').all()) { + await s.seek(`[id="${await box.getAttribute('id')}"]`, 'Improvement checkbox'); await s.key('Space', 'Tick improvement'); + } + } else if (pattern === 'highlight') { + await s.seek('[data-lp-chunk]', 'First passage chunk'); + await s.key('Space', 'Mark wrong passage'); await s.key('Tab', 'Next passage at limit'); + await s.key('Space', 'Attempt second mark at limit'); await s.snapshot('limit'); + await s.activate('[data-lp-check]', 'Check wrong mark'); await s.snapshot('partial'); + await s.read('Wrong mark local feedback'); await s.activate('[data-lp-restart]', 'Start over'); + for (const chunk of content.paragraphs.flat()) { + await s.seek(`[data-lp-chunk="${chunk.id}"]`, `Passage ${chunk.id}`); + if (chunk.key) await s.key('Space', 'Mark correct evidence'); + } + await s.activate('[data-lp-check]', 'Check correct mark'); + } else if (pattern === 'review-prompts') { + await s.activate('[data-lp-commit]', 'Reveal recalled answer'); + await s.activate('[data-lp-result="forgot"]', 'Rate forgot'); await s.snapshot('forgot'); + await s.activate('[data-lp-result="remembered"]', 'Rate remembered'); + } else if (pattern === 'retrieval-sheet') { + await s.seek('[data-lp-date]', 'Date field'); await s.key('ArrowUp', 'Change date segment'); + await s.key('Tab', 'Leave date'); await s.seek('[data-lp-tab="front"]', 'Front tab'); + await s.key('ArrowRight', 'Back tab'); await s.key('ArrowLeft', 'Front tab'); + await s.key('End', 'Back tab End'); await s.key('Home', 'Front tab Home'); + await s.key('ArrowRight', 'Back answers'); await s.read('Back answers and print instructions'); + await s.seek('[data-lp-date]', 'Date revisit'); await s.key('Command+A', 'Select date'); + await s.key('Backspace', 'Clear date'); await s.key('Tab', 'Leave blank date'); + await s.snapshot('incomplete'); + await s.seek('[data-lp-date]', 'Invalid date error'); + await s.type('10152026', 'Recover date with keyboard'); await s.key('Tab', 'Leave recovered date'); + } else if (pattern === 'test-out') { + await s.activate('[data-lp-start]', 'Start check'); + await s.activate('[data-lp-next]:visible', 'Missing first answer Next'); await s.snapshot('incomplete'); + for (let q = 0; q < content.questions.length; q++) { + const index = q === 0 ? 0 : content.questions[q].options.findIndex(option => option.id === content.questions[q].correct); + await choose(page.locator('fieldset').nth(q).locator('input'), index, `Question ${q + 1}`, true); + if (q + 1 < content.questions.length) await s.activate('[data-lp-next]:visible', `Next to question ${q + 2}`); + } + await s.activate('[data-lp-check]', 'Submit placement check'); + await s.activate('[data-lp-review] > summary', 'Open Review answers', 'Space'); + } else if (pattern === 'write-distractors') { + await s.activate('[data-lp-check]', 'Blank recall submit'); + await s.seek('[data-lp-answer]', 'Your answer'); await s.type(content.rightAnswer, 'Write recall'); + await s.activate('[data-lp-check]', 'Check recall'); + await s.activate('[data-lp-had-it]', 'Self-report Yes'); + await s.activate('[data-lp-compare]', 'Incomplete wrong options submit'); await s.snapshot('incomplete'); + for (let i = 0; i < content.count; i++) { + await s.seek(`[id="example-option-${i}-text"]`, `Wrong option ${i + 2}`); + await s.type(content.authorOptions[i].text, `Write wrong option ${i + 2}`); + await s.seek(`[id="example-option-${i}-misconception"]`, `Misconception ${i + 2}`); + await s.key('Space', 'Open misconception menu'); + await s.key('Home', 'First misconception'); + const index = content.misconceptions.findIndex(item => item.id === content.authorOptions[i].misconception); + for (let j = 0; j <= index; j++) await s.key('ArrowDown', `Misconception option ${j + 1}`); + await s.key('Enter', 'Commit misconception'); + } + await s.activate('[data-lp-compare]', 'Compare valid options'); + } else if (pattern === 'formats') { + for (const format of ['text', 'slides', 'audio', 'quiz']) await s.activate(`[data-lp-format="${format}"]`, `Choose ${format}`); + await s.activate('[data-lp-check]:visible', 'Missing quiz answer submit'); await s.snapshot('incomplete'); + for (let q = 0; q < content.quiz.length; q++) { + if (q === 1) { await s.activate('[data-lp-next]', 'Next section'); await s.activate('[data-lp-next]', 'Last section'); } + const inputs = page.locator('[data-lp-question]:visible input'); + await choose(inputs, 0, `Quiz ${q + 1} wrong answer`); + await s.activate('[data-lp-check]:visible', 'Check wrong quiz option'); await s.snapshot(`wrong-${q}`); + await s.previous('Local wrong quiz feedback VO+Left'); + await choose(inputs, content.quiz[q].options.findIndex(option => option.correct), `Quiz ${q + 1} correct answer`, true); + await s.activate('[data-lp-check]:visible', 'Check correct quiz option'); + } + } +} diff --git a/tests/voiceover/session.js b/tests/voiceover/session.js new file mode 100644 index 0000000..c2f4ef2 --- /dev/null +++ b/tests/voiceover/session.js @@ -0,0 +1,102 @@ +// Web-content entry follows Guidepup's MIT Playwright fixture at d5c9d805. +import { voiceOver, macOSActivate, MacOSKeyCodes, voiceOverKeyCodeCommands } from '@guidepup/guidepup'; +import { appendFile, writeFile } from 'node:fs/promises'; + +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +export async function startSession(page, info) { + const rows = []; + await voiceOver.start({ capture: true, retries: 3 }); + await writeFile(info.outputPath('settings.json'), JSON.stringify(await voiceOver.getSettings(), null, 2)); + + async function state() { + return page.evaluate(() => { + const el = document.activeElement; + return { + hasFocus: document.hasFocus(), lang: document.documentElement.lang, + active: { tag: el.tagName, id: el.id, text: el.innerText ?? '', name: el.getAttribute('aria-label'), value: el.value, checked: el.checked, pressed: el.getAttribute('aria-pressed'), selected: el.getAttribute('aria-selected') }, + statuses: [...document.querySelectorAll('[role="status"]')].map(el => el.textContent), + mutations: window.voStatusChanges ?? [] + }; + }); + } + async function step(label, action, capture = true) { + const before = await state(), begin = new Date().toISOString(); + await voiceOver.clearSpokenPhraseLog(); + await voiceOver.clearItemTextLog(); + let error; + try { await action(capture); } catch (e) { error = e; } + const row = { step: label, begin, end: new Date().toISOString(), before, after: await state(), speech: await voiceOver.spokenPhraseLog(), cursor: await voiceOver.itemTextLog(), error: error?.stack }; + rows.push(row); + await appendFile(info.outputPath('steps.jsonl'), JSON.stringify(row) + '\n'); + await appendFile(info.outputPath('spoken-phrases.txt'), `${label}\n${JSON.stringify(row.speech)}\n`); + console.log(`${info.title}: ${label}: ${JSON.stringify(row.speech)}`); + if (error) throw error; + return row; + } + const key = (key, label = key, capture = true) => step(label, c => voiceOver.press(key, { capture: c }), capture); + const next = (label, capture = 'initial') => step(label, c => voiceOver.next({ capture: c }), capture); + const previous = label => step(label, c => voiceOver.previous({ capture: c }), 'initial'); + + async function enter() { + await macOSActivate('Playwright'); + await voiceOver.press('Control', { capture: false }); + await page.bringToFront(); + await page.evaluate(() => { + const marker = document.createElement('input'); + marker.id = '__guidepup_marker__'; marker.value = 'Guidepup Marker'; marker.readOnly = true; marker.tabIndex = -1; + marker.setAttribute('aria-label', 'Guidepup Marker'); + marker.style.cssText = 'position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0 0 0 0);white-space:nowrap'; + document.body.prepend(marker); + }); + try { + await voiceOver.perform(voiceOverKeyCodeCommands.openItemChooser, { capture: false }); + await pause(500); + await voiceOver.type('web content', { capture: false }); + await voiceOver.press('Enter', { capture: false }); + await pause(100); + await voiceOver.interact({ capture: false }); + await voiceOver.perform(voiceOverKeyCodeCommands.moveToBeginningOfText, { capture: false }); + await voiceOver.perform({ keyCode: MacOSKeyCodes.Control }, { capture: false }); + await pause(100); + await voiceOver.perform(voiceOverKeyCodeCommands.moveToBeginningOfText, { capture: false }); + await next('Arrival heading', true); + } finally { + await page.evaluate(() => document.querySelector('#__guidepup_marker__')?.remove()); + } + } + async function observe() { + await page.evaluate(() => { + window.voStatusChanges = []; + for (const el of document.querySelectorAll('[role="status"]')) new MutationObserver(() => window.voStatusChanges.push({ at: new Date().toISOString(), text: el.textContent })).observe(el, { childList: true, characterData: true, subtree: true }); + }); + } + async function seek(selector, label) { + const target = page.locator(selector).first(); + if (!await target.isVisible()) throw new Error(`Journey target not visible: ${selector}`); + for (let i = 0; i < 35; i++) { + if (await target.evaluate(el => el === document.activeElement)) return; + const backwards = await target.evaluate(el => document.activeElement !== document.body && Boolean(el.compareDocumentPosition(document.activeElement) & Node.DOCUMENT_POSITION_FOLLOWING)); + await key(backwards ? 'Shift+Tab' : 'Tab', `${label}: ${backwards ? 'Shift+Tab' : 'Tab'} ${i + 1}`, 'initial'); + } + throw new Error(`Keyboard could not reach ${selector}`); + } + const activate = async (selector, label, press = 'Enter') => { await seek(selector, label); return key(press, label); }; + const type = (text, label) => step(label, c => voiceOver.perform({ characters: text }, { capture: c })); + async function snapshot(label) { + await writeFile(info.outputPath(`${label}.html`), await page.content()); + await writeFile(info.outputPath(`${label}-aria.txt`), await page.locator('body').ariaSnapshot()); + await page.screenshot({ path: info.outputPath(`${label}.png`), fullPage: true }); + } + // Explicit rereads are separate steps; they are not spontaneous duplicate announcements. + async function read(label, limit = 70) { + await enter(); + for (let i = 0; i < limit; i++) { + const row = await next(`${label}: VO+Right ${i + 1}`); + if (row.cursor.some(text => /end of web content|bottom of web content/i.test(text)) || row.speech.some(text => /end of web content|bottom of web content/i.test(text))) break; + if (i > 1 && JSON.stringify(row.cursor) === JSON.stringify(rows.at(-2)?.cursor)) break; + } + } + return { rows, step, key, next, previous, enter, observe, seek, activate, type, snapshot, read, state, + stop: async () => { await writeFile(info.outputPath('final-state.json'), JSON.stringify(await state(), null, 2)); await voiceOver.stop(); } + }; +} From 5f9864cbb4d3e9bae60316a070e64caeca667235 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:03:08 -0400 Subject: [PATCH 2/8] ci: bound Guidepup setup and preserve early diagnostics Omit optional setup recording after ffmpeg prevented the first hosted attempt from completing. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/voiceover.yml | 18 +++++++++++++++--- playwright.voiceover.config.js | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/voiceover.yml b/.github/workflows/voiceover.yml index fdedb94..d0e82c2 100644 --- a/.github/workflows/voiceover.yml +++ b/.github/workflows/voiceover.yml @@ -21,11 +21,23 @@ jobs: with: node-version: '22.23.3' cache: npm - # Matches Guidepup's macOS 26 WebKit CI setup. - - run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 setup --ci --macos-record + - name: Record runner before setup + run: | + mkdir -p test-results/voiceover-environment + sw_vers > test-results/voiceover-environment/runner.txt + echo "Runner image: $ImageOS $ImageVersion" >> test-results/voiceover-environment/runner.txt + say -v '?' > test-results/voiceover-environment/voices.txt + # Setup recording stalled on this image; speech capture does not need ffmpeg. + - name: Configure Guidepup + timeout-minutes: 5 + shell: bash + run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 setup --ci 2>&1 | tee test-results/voiceover-environment/setup.log - run: npm ci --no-audit --no-fund --no-progress - run: npx playwright install webkit - - run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 install + - name: Install Guidepup assets + timeout-minutes: 5 + shell: bash + run: npx -y --min-release-age=0 --ignore-scripts=false @guidepup/setup@0.29.1 install 2>&1 | tee test-results/voiceover-environment/install.log - name: Record environment run: | mkdir -p test-results/voiceover-environment diff --git a/playwright.voiceover.config.js b/playwright.voiceover.config.js index 4dd9be9..b2dc78b 100644 --- a/playwright.voiceover.config.js +++ b/playwright.voiceover.config.js @@ -9,6 +9,6 @@ export default defineConfig({ timeout: 20 * 60 * 1000, outputDir: `test-results/voiceover-${process.env.VO_PHASE ?? 'journeys'}`, reporter: [['list']], - use: { browserName: 'webkit', headless: false, baseURL: `http://127.0.0.1:${port}`, viewport: { width: 1280, height: 900 } }, + use: { browserName: 'webkit', headless: false, baseURL: `http://127.0.0.1:${port}`, viewport: { width: 1280, height: 900 }, screenshot: 'only-on-failure' }, webServer: { command: 'node scripts/serve-demo.mjs', url: `http://127.0.0.1:${port}/index.html`, reuseExistingServer: false, env: { LP_PORT: String(port) } } }); From 188e2922b64f08d9ef5de52ca6cc3f7cc69ea209 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:23:08 -0400 Subject: [PATCH 3/8] docs: retain Guidepup fixture MIT notice Co-Authored-By: Claude Opus 5.5 --- THIRD_PARTY_NOTICES.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 THIRD_PARTY_NOTICES.md diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..0dd367e --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,25 @@ +# Third-party notices + +The web-content entry routine in `tests/voiceover/session.js` adapts the [Guidepup Playwright fixture](https://github.com/guidepup/guidepup/blob/d5c9d8059954f214b82689dffcb5866a9b55fe6c/examples/playwright-voiceover/voiceover-test.ts). Its licence follows. + +MIT License + +Copyright (c) 2023 Craig Morten + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 48974911f36fb98a81ebe956b53a79777154d917 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:28:34 -0400 Subject: [PATCH 4/8] test: calibrate native text input and VoiceOver control searches Co-Authored-By: Claude Opus 5.5 --- tests/voiceover/README.md | 2 +- tests/voiceover/control.spec.js | 6 +++++- tests/voiceover/session.js | 23 ++++++++++++++++------- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/voiceover/README.md b/tests/voiceover/README.md index 6ded58f..c200c1b 100644 --- a/tests/voiceover/README.md +++ b/tests/voiceover/README.md @@ -4,7 +4,7 @@ The separate VoiceOver workflow runs on GitHub-hosted macOS 26 with headed Playw Guidepup 0.35.0, its setup tool 0.29.1 and Playwright 1.63.0 are pinned. Web-content entry follows [Guidepup's MIT fixture](https://github.com/guidepup/guidepup/blob/d5c9d8059954f214b82689dffcb5866a9b55fe6c/examples/playwright-voiceover/voiceover-test.ts). This tests Playwright WebKit, not installed Safari. -The positive control must capture the heading, button name and role, and one live-message occurrence before any pattern journey runs. Each English and French journey records native VoiceOver keyboard actions, spoken phrases, cursor text, DOM focus and status mutations. The artifacts include per-step JSON, phrase logs, HTML, accessibility snapshots and screenshots. Journey errors are recorded without failing CI, so a green job means the control passed and evidence was collected, not that the pages passed an accessibility review. +The positive control must capture the heading, button name and role, and one live-message occurrence before any pattern journey runs. It also checks native text entry. Each English and French journey records native VoiceOver keyboard actions, spoken phrases, cursor text, DOM focus and status mutations. Tab searches fall back to VO navigation where WebKit skips buttons. The artifacts include per-step JSON, phrase logs, HTML, accessibility snapshots and screenshots. Journey errors are recorded without failing CI, so a green job means the control passed and evidence was collected, not that the pages passed an accessibility review. Navigation uses Guidepup's initial capture; result actions use full capture. Initial capture can miss later phrases in a long utterance. Guidepup polls VoiceOver's last phrase and combines changed phrases; consecutive identical announcements may not appear separately. Interpret announcement counts with those limits. Installed voices and portable preferences are recorded, but phrase text cannot prove French pronunciation. diff --git a/tests/voiceover/control.spec.js b/tests/voiceover/control.spec.js index feeeb89..c8f942d 100644 --- a/tests/voiceover/control.spec.js +++ b/tests/voiceover/control.spec.js @@ -3,7 +3,7 @@ import { startSession } from './session.js'; test('positive control captures heading, button name and role, and one live message', async ({ page }, info) => { await page.goto('/index.html'); - await page.setContent('VoiceOver positive control

Control heading

'); + await page.setContent('VoiceOver positive control

Control heading

'); const session = await startSession(page, info); try { await session.observe(); await session.enter(); @@ -16,5 +16,9 @@ test('positive control captures heading, button name and role, and one live mess const activated = session.rows.filter(row => row.step === 'Activate control button').flatMap(row => row.speech).join('\n'); expect(activated.split('Control live message').length - 1).toBe(1); expect(await page.locator('#status').textContent()).toBe('Control live message'); + await session.seek('input', 'Control entry'); + await session.type('Control input', 'Type control text'); + expect(await page.locator('input').inputValue()).toBe('Control input'); + await session.snapshot('control-input'); } finally { await session.stop(); } }); diff --git a/tests/voiceover/session.js b/tests/voiceover/session.js index c2f4ef2..7eef2ce 100644 --- a/tests/voiceover/session.js +++ b/tests/voiceover/session.js @@ -15,7 +15,7 @@ export async function startSession(page, info) { hasFocus: document.hasFocus(), lang: document.documentElement.lang, active: { tag: el.tagName, id: el.id, text: el.innerText ?? '', name: el.getAttribute('aria-label'), value: el.value, checked: el.checked, pressed: el.getAttribute('aria-pressed'), selected: el.getAttribute('aria-selected') }, statuses: [...document.querySelectorAll('[role="status"]')].map(el => el.textContent), - mutations: window.voStatusChanges ?? [] + mutations: window.voStatusChanges ?? [], keys: (window.voKeys ?? []).slice(-20) }; }); } @@ -33,11 +33,11 @@ export async function startSession(page, info) { if (error) throw error; return row; } - const key = (key, label = key, capture = true) => step(label, c => voiceOver.press(key, { capture: c }), capture); + const key = (key, label = key, capture = true) => step(label, c => voiceOver.press(key, { capture: c, application: 'Playwright' }), capture); const next = (label, capture = 'initial') => step(label, c => voiceOver.next({ capture: c }), capture); const previous = label => step(label, c => voiceOver.previous({ capture: c }), 'initial'); - async function enter() { + async function enter(label = 'Arrival heading', capture = true) { await macOSActivate('Playwright'); await voiceOver.press('Control', { capture: false }); await page.bringToFront(); @@ -59,7 +59,7 @@ export async function startSession(page, info) { await voiceOver.perform({ keyCode: MacOSKeyCodes.Control }, { capture: false }); await pause(100); await voiceOver.perform(voiceOverKeyCodeCommands.moveToBeginningOfText, { capture: false }); - await next('Arrival heading', true); + await next(label, capture); } finally { await page.evaluate(() => document.querySelector('#__guidepup_marker__')?.remove()); } @@ -67,21 +67,30 @@ export async function startSession(page, info) { async function observe() { await page.evaluate(() => { window.voStatusChanges = []; + window.voKeys = []; + document.addEventListener('keydown', event => window.voKeys.push({ key: event.key, code: event.code, ctrl: event.ctrlKey, alt: event.altKey, shift: event.shiftKey, meta: event.metaKey, trusted: event.isTrusted }), true); for (const el of document.querySelectorAll('[role="status"]')) new MutationObserver(() => window.voStatusChanges.push({ at: new Date().toISOString(), text: el.textContent })).observe(el, { childList: true, characterData: true, subtree: true }); }); } async function seek(selector, label) { const target = page.locator(selector).first(); if (!await target.isVisible()) throw new Error(`Journey target not visible: ${selector}`); - for (let i = 0; i < 35; i++) { + for (let i = 0; i < 4; i++) { if (await target.evaluate(el => el === document.activeElement)) return; const backwards = await target.evaluate(el => document.activeElement !== document.body && Boolean(el.compareDocumentPosition(document.activeElement) & Node.DOCUMENT_POSITION_FOLLOWING)); await key(backwards ? 'Shift+Tab' : 'Tab', `${label}: ${backwards ? 'Shift+Tab' : 'Tab'} ${i + 1}`, 'initial'); } + if (await target.evaluate(el => el === document.activeElement)) return; + // WebKit's default Tab policy skips some buttons. VO navigation reaches them. + await enter(`${label}: VO beginning`, 'initial'); + for (let i = 0; i < 100; i++) { + await next(`${label}: VO+Right ${i + 1}`); + if (await target.evaluate(el => el === document.activeElement)) return; + } throw new Error(`Keyboard could not reach ${selector}`); } const activate = async (selector, label, press = 'Enter') => { await seek(selector, label); return key(press, label); }; - const type = (text, label) => step(label, c => voiceOver.perform({ characters: text }, { capture: c })); + const type = (text, label) => step(label, () => voiceOver.type(text, { capture: 'initial', application: 'Playwright' })); async function snapshot(label) { await writeFile(info.outputPath(`${label}.html`), await page.content()); await writeFile(info.outputPath(`${label}-aria.txt`), await page.locator('body').ariaSnapshot()); @@ -89,7 +98,7 @@ export async function startSession(page, info) { } // Explicit rereads are separate steps; they are not spontaneous duplicate announcements. async function read(label, limit = 70) { - await enter(); + await enter(`${label}: beginning`, 'initial'); for (let i = 0; i < limit; i++) { const row = await next(`${label}: VO+Right ${i + 1}`); if (row.cursor.some(text => /end of web content|bottom of web content/i.test(text)) || row.speech.some(text => /end of web content|bottom of web content/i.test(text))) break; From 1409606695bdc81e7ed8c9d72f781464700772d5 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:31:11 -0400 Subject: [PATCH 5/8] test: drop removed formats VoiceOver journey Co-Authored-By: Claude Opus 5.5 --- .github/workflows/voiceover.yml | 2 +- tests/voiceover/journeys.spec.js | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/voiceover.yml b/.github/workflows/voiceover.yml index d0e82c2..d003388 100644 --- a/.github/workflows/voiceover.yml +++ b/.github/workflows/voiceover.yml @@ -57,7 +57,7 @@ jobs: run: npm run test:voiceover -- --grep 'positive control' env: VO_PHASE: control - - name: Record the nine patterns in English and French + - name: Record the eight patterns in English and French run: npm run test:voiceover -- --grep-invert 'positive control' - uses: actions/upload-artifact@v4 if: always() diff --git a/tests/voiceover/journeys.spec.js b/tests/voiceover/journeys.spec.js index 708a551..45dfd06 100644 --- a/tests/voiceover/journeys.spec.js +++ b/tests/voiceover/journeys.spec.js @@ -2,7 +2,7 @@ import { test } from '@playwright/test'; import { readFile, writeFile } from 'node:fs/promises'; import { startSession } from './session.js'; -const patterns = ['dont-know', 'first-answer', 'formats', 'highlight', 'retrieval-sheet', 'review-prompts', 'self-check', 'test-out', 'write-distractors']; +const patterns = ['dont-know', 'first-answer', 'highlight', 'retrieval-sheet', 'review-prompts', 'self-check', 'test-out', 'write-distractors']; for (const pattern of patterns) for (const lang of ['en', 'fr']) { test(`${pattern} ${lang}`, async ({ page, browser }, info) => { await page.goto(`/${pattern}/${lang}.html`); @@ -133,17 +133,5 @@ async function journey(s, page, pattern, lang, content) { await s.key('Enter', 'Commit misconception'); } await s.activate('[data-lp-compare]', 'Compare valid options'); - } else if (pattern === 'formats') { - for (const format of ['text', 'slides', 'audio', 'quiz']) await s.activate(`[data-lp-format="${format}"]`, `Choose ${format}`); - await s.activate('[data-lp-check]:visible', 'Missing quiz answer submit'); await s.snapshot('incomplete'); - for (let q = 0; q < content.quiz.length; q++) { - if (q === 1) { await s.activate('[data-lp-next]', 'Next section'); await s.activate('[data-lp-next]', 'Last section'); } - const inputs = page.locator('[data-lp-question]:visible input'); - await choose(inputs, 0, `Quiz ${q + 1} wrong answer`); - await s.activate('[data-lp-check]:visible', 'Check wrong quiz option'); await s.snapshot(`wrong-${q}`); - await s.previous('Local wrong quiz feedback VO+Left'); - await choose(inputs, content.quiz[q].options.findIndex(option => option.correct), `Quiz ${q + 1} correct answer`, true); - await s.activate('[data-lp-check]:visible', 'Check correct quiz option'); - } } } From ecde28a7c064672c255c3ca9eae0338ade5c0c78 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:03:21 -0400 Subject: [PATCH 6/8] Target the WebKit process before native VoiceOver input Co-Authored-By: Claude Opus 5.5 --- tests/voiceover/README.md | 2 ++ tests/voiceover/session.js | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/voiceover/README.md b/tests/voiceover/README.md index c200c1b..50b1bc6 100644 --- a/tests/voiceover/README.md +++ b/tests/voiceover/README.md @@ -8,6 +8,8 @@ The positive control must capture the heading, button name and role, and one liv Navigation uses Guidepup's initial capture; result actions use full capture. Initial capture can miss later phrases in a long utterance. Guidepup polls VoiceOver's last phrase and combines changed phrases; consecutive identical announcements may not appear separately. Interpret announcement counts with those limits. Installed voices and portable preferences are recorded, but phrase text cannot prove French pronunciation. +Before native keys or text entry, System Events brings the Playwright process to the foreground and checks its name. Keys then go directly to the foreground application. This avoids repeated application activation through AppleScript, which timed out during native text entry on the hosted runner. Highlight uses the current inline span buttons through Tab and Space. + Run on a dedicated macOS desktop configured for Guidepup: ```sh diff --git a/tests/voiceover/session.js b/tests/voiceover/session.js index 7eef2ce..26f03c1 100644 --- a/tests/voiceover/session.js +++ b/tests/voiceover/session.js @@ -1,8 +1,11 @@ // Web-content entry follows Guidepup's MIT Playwright fixture at d5c9d805. -import { voiceOver, macOSActivate, MacOSKeyCodes, voiceOverKeyCodeCommands } from '@guidepup/guidepup'; +import { voiceOver, MacOSKeyCodes, voiceOverKeyCodeCommands } from '@guidepup/guidepup'; import { appendFile, writeFile } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +const run = promisify(execFile); export async function startSession(page, info) { const rows = []; await voiceOver.start({ capture: true, retries: 3 }); @@ -33,12 +36,20 @@ export async function startSession(page, info) { if (error) throw error; return row; } - const key = (key, label = key, capture = true) => step(label, c => voiceOver.press(key, { capture: c, application: 'Playwright' }), capture); + async function focusBrowser() { + // Repeated AppleScript application activation stalled WebKit during typing. + const { stdout } = await run('/usr/bin/osascript', ['-e', 'with timeout of 5 seconds\ntell application "System Events"\nset frontmost of process "Playwright" to true\nreturn name of first application process whose frontmost is true\nend tell\nend timeout'], { timeout: 6000 }); + if (stdout.trim() !== 'Playwright') throw new Error(`Native keyboard target is ${stdout.trim()}`); + } + const key = (key, label = key, capture = true) => step(label, async c => { + await focusBrowser(); + await voiceOver.press(key, { capture: c }); + }, capture); const next = (label, capture = 'initial') => step(label, c => voiceOver.next({ capture: c }), capture); const previous = label => step(label, c => voiceOver.previous({ capture: c }), 'initial'); async function enter(label = 'Arrival heading', capture = true) { - await macOSActivate('Playwright'); + await focusBrowser(); await voiceOver.press('Control', { capture: false }); await page.bringToFront(); await page.evaluate(() => { @@ -90,7 +101,10 @@ export async function startSession(page, info) { throw new Error(`Keyboard could not reach ${selector}`); } const activate = async (selector, label, press = 'Enter') => { await seek(selector, label); return key(press, label); }; - const type = (text, label) => step(label, () => voiceOver.type(text, { capture: 'initial', application: 'Playwright' })); + const type = (text, label) => step(label, async () => { + await focusBrowser(); + await voiceOver.type(text, { capture: 'initial' }); + }); async function snapshot(label) { await writeFile(info.outputPath(`${label}.html`), await page.content()); await writeFile(info.outputPath(`${label}-aria.txt`), await page.locator('body').ariaSnapshot()); From cc4fd8a87706e18f78909af9be87c0bf81a10713 Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:00:11 -0400 Subject: [PATCH 7/8] test: verify native radio selections and recover date year Use signed arrow navigation because hosted WebKit radio groups stop at their bounds. Restore only the cleared year segment and verify intended native input values. Co-Authored-By: Claude Opus 5.5 --- tests/voiceover/journeys.spec.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/voiceover/journeys.spec.js b/tests/voiceover/journeys.spec.js index 45dfd06..01639ec 100644 --- a/tests/voiceover/journeys.spec.js +++ b/tests/voiceover/journeys.spec.js @@ -1,4 +1,5 @@ import { test } from '@playwright/test'; +import assert from 'node:assert/strict'; import { readFile, writeFile } from 'node:fs/promises'; import { startSession } from './session.js'; @@ -34,9 +35,11 @@ async function journey(s, page, pattern, lang, content) { if (inspect) for (let i = 1; i < await inputs.count(); i++) await s.key('ArrowDown', `${label}: option ${i + 1}`); const active = await inputs.evaluateAll(elements => elements.indexOf(document.activeElement)); if (active < 0) throw new Error(`${label}: radio focus lost`); - const distance = (index - active + await inputs.count()) % await inputs.count(); - for (let i = 0; i < distance; i++) await s.key('ArrowDown', `${label}: choose answer ${i + 1}`); + // Native WebKit radio groups stop at their ends rather than wrapping. + const distance = index - active; + for (let i = 0; i < Math.abs(distance); i++) await s.key(distance < 0 ? 'ArrowUp' : 'ArrowDown', `${label}: choose answer ${i + 1}`); await s.key('Space', `${label}: confirm`); + assert.equal(await inputs.nth(index).isChecked(), true, `${label}: intended radio answer not selected`); } if (pattern === 'dont-know') { await s.activate('[data-lp-check]', 'Incomplete submit'); await s.snapshot('incomplete'); @@ -105,7 +108,10 @@ async function journey(s, page, pattern, lang, content) { await s.key('Backspace', 'Clear date'); await s.key('Tab', 'Leave blank date'); await s.snapshot('incomplete'); await s.seek('[data-lp-date]', 'Invalid date error'); - await s.type('10152026', 'Recover date with keyboard'); await s.key('Tab', 'Leave recovered date'); + // Clear date removed only the active year segment; month/day remain 10/15. + await s.type('2026', 'Recover date year with keyboard'); + assert.equal(await page.locator('[data-lp-date]').inputValue(), '2026-10-15', 'Intended date not recovered'); + await s.key('Tab', 'Leave recovered date'); } else if (pattern === 'test-out') { await s.activate('[data-lp-start]', 'Start check'); await s.activate('[data-lp-next]:visible', 'Missing first answer Next'); await s.snapshot('incomplete'); From f13051f4e73f6473fcfaecd70e3bcffc57c4629b Mon Sep 17 00:00:00 2001 From: Ariel Harlap <3870784+aharlap@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:54:28 -0400 Subject: [PATCH 8/8] test: VoiceOver journey for the retrieval-sheet radio group; reread past tabs; disable hot corners The closing reread stopped at the tabs because VoiceOver reports them as empty cursor strings. A hot corner opened Quick Note during run 37843643548. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/voiceover.yml | 5 ++++ tests/voiceover/journeys.spec.js | 39 ++++++++++++++++++++++---------- tests/voiceover/session.js | 4 +++- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/.github/workflows/voiceover.yml b/.github/workflows/voiceover.yml index d003388..40b350c 100644 --- a/.github/workflows/voiceover.yml +++ b/.github/workflows/voiceover.yml @@ -27,6 +27,11 @@ jobs: sw_vers > test-results/voiceover-environment/runner.txt echo "Runner image: $ImageOS $ImageVersion" >> test-results/voiceover-environment/runner.txt say -v '?' > test-results/voiceover-environment/voices.txt + # VoiceOver moves the pointer; a hot corner opened Quick Note mid-journey in run 37843643548. + - name: Disable hot corners + run: | + for corner in tl tr bl br; do defaults write com.apple.dock "wvous-$corner-corner" -int 0; done + killall Dock # Setup recording stalled on this image; speech capture does not need ffmpeg. - name: Configure Guidepup timeout-minutes: 5 diff --git a/tests/voiceover/journeys.spec.js b/tests/voiceover/journeys.spec.js index 01639ec..68b4faa 100644 --- a/tests/voiceover/journeys.spec.js +++ b/tests/voiceover/journeys.spec.js @@ -13,7 +13,7 @@ for (const pattern of patterns) for (const lang of ['en', 'fr']) { const s = await startSession(page, info); try { await s.observe(); await s.enter(); - for (let i = 0; i < 8; i++) await s.next(`Arrival instructions and labels ${i + 1}`); + for (let i = 0; i < (pattern === 'retrieval-sheet' ? 14 : 8); i++) await s.next(`Arrival instructions and labels ${i + 1}`); await s.snapshot('arrival'); await journey(s, page, pattern, lang, content); await s.snapshot('result'); @@ -99,19 +99,34 @@ async function journey(s, page, pattern, lang, content) { await s.activate('[data-lp-result="forgot"]', 'Rate forgot'); await s.snapshot('forgot'); await s.activate('[data-lp-result="remembered"]', 'Rate remembered'); } else if (pattern === 'retrieval-sheet') { - await s.seek('[data-lp-date]', 'Date field'); await s.key('ArrowUp', 'Change date segment'); - await s.key('Tab', 'Leave date'); await s.seek('[data-lp-tab="front"]', 'Front tab'); + const radios = page.locator('[data-lp-spacing] input'), date = page.locator('[data-lp-date]'); + await s.seek('[data-lp-spacing] input:checked', 'Spacing group'); + await s.key('ArrowUp', 'Choose In 2 days'); + for (const label of ['In 1 week', 'In 2 weeks', 'In 1 month', 'Another date']) await s.key('ArrowDown', `Choose ${label}`); + assert.equal(await radios.last().isChecked(), true, 'Another date not selected'); + assert.equal(await date.isVisible(), true, 'Date field not revealed'); + await s.key('Tab', 'Date field revealed'); + // Shift+Tab into WebKit's date control lands on the year segment, which run 7 proved recoverable by typing. + const year = Number((await date.inputValue()).slice(0, 4)), rest = (await date.inputValue()).slice(4); + for (const [cycle, typed] of [['Different date', year + 1], ['Same date', year + 1]]) { + await s.seek('[data-lp-tab][aria-selected="true"]', `${cycle}: leave Date field`); + await s.key('Shift+Tab', `${cycle}: return to year segment`); + await s.key('Command+A', `${cycle}: select year`); await s.key('Backspace', `${cycle}: clear year`); + await s.key('Tab', `${cycle}: leave invalid date`); + assert.equal(await date.inputValue(), '', `${cycle}: date not cleared`); + await s.snapshot(`incomplete-${cycle === 'Same date' ? 'same' : 'different'}`); + await s.key('Shift+Tab', `${cycle}: return to invalid date`); + await s.type(String(typed), `${cycle}: recover year with digits`); + assert.equal(await date.inputValue(), `${typed}${rest}`, `${cycle}: intended date not recovered`); + await s.key('Tab', `${cycle}: leave recovered date`); + } + await s.seek('[data-lp-tab="front"]', 'Front tab'); await s.key('ArrowRight', 'Back tab'); await s.key('ArrowLeft', 'Front tab'); await s.key('End', 'Back tab End'); await s.key('Home', 'Front tab Home'); - await s.key('ArrowRight', 'Back answers'); await s.read('Back answers and print instructions'); - await s.seek('[data-lp-date]', 'Date revisit'); await s.key('Command+A', 'Select date'); - await s.key('Backspace', 'Clear date'); await s.key('Tab', 'Leave blank date'); - await s.snapshot('incomplete'); - await s.seek('[data-lp-date]', 'Invalid date error'); - // Clear date removed only the active year segment; month/day remain 10/15. - await s.type('2026', 'Recover date year with keyboard'); - assert.equal(await page.locator('[data-lp-date]').inputValue(), '2026-10-15', 'Intended date not recovered'); - await s.key('Tab', 'Leave recovered date'); + await s.key('ArrowRight', 'Back answers'); + // Harness-owned: the native print sheet would block WebKit; the page's status write still runs. + await page.evaluate(() => { window.print = () => {}; }); + await s.activate('[data-lp-print]', 'Print'); } else if (pattern === 'test-out') { await s.activate('[data-lp-start]', 'Start check'); await s.activate('[data-lp-next]:visible', 'Missing first answer Next'); await s.snapshot('incomplete'); diff --git a/tests/voiceover/session.js b/tests/voiceover/session.js index 26f03c1..819fa33 100644 --- a/tests/voiceover/session.js +++ b/tests/voiceover/session.js @@ -116,7 +116,9 @@ export async function startSession(page, info) { for (let i = 0; i < limit; i++) { const row = await next(`${label}: VO+Right ${i + 1}`); if (row.cursor.some(text => /end of web content|bottom of web content/i.test(text)) || row.speech.some(text => /end of web content|bottom of web content/i.test(text))) break; - if (i > 1 && JSON.stringify(row.cursor) === JSON.stringify(rows.at(-2)?.cursor)) break; + // VoiceOver reports tab items as empty cursor strings; only a repeated non-empty cursor means the end. + const cursor = row.cursor.filter(Boolean); + if (i > 1 && cursor.length && JSON.stringify(cursor) === JSON.stringify(rows.at(-2)?.cursor.filter(Boolean))) break; } } return { rows, step, key, next, previous, enter, observe, seek, activate, type, snapshot, read, state,