From ffab16c95a9b74fe439811155da305f5221e6faa Mon Sep 17 00:00:00 2001 From: Eyalm321 Date: Fri, 18 Sep 2026 01:41:29 -0400 Subject: [PATCH] a11y: expose pane content over AT-SPI (Linux) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The terminal grid is drawn to a texture, so assistive tech saw each pane as an empty rectangle. TerminalPane now carries the visible screen as accessible text. - Slint's `accessibility` feature is requested per-target for Linux only. It is dropped workspace-wide because accesskit pins `windows 0.58` against wgpu-hal 29's `0.62` — a conflict that only exists when building for Windows, so the Windows resolution is untouched. - PaneItem.a11y-text carries the text; TerminalPane exposes it via accessible-role/label/value (read-only). - paneview fills it from the existing TerminalPane::screen_text(), gated by a11y_wanted() so it costs nothing when no assistive client is connected (HYPERPANES_A11Y=1 forces on, NO_AT_BRIDGE=1 forces off, else the GNOME toolkit-accessibility setting). Co-Authored-By: Claude Opus 5 (1M context) --- rs/crates/app/Cargo.lock | 227 ++++++++++++++++++++++ rs/crates/app/Cargo.toml | 8 + rs/crates/app/src/paneview.rs | 28 +++ rs/crates/app/ui/paneview.slint | 2 + rs/crates/app/ui/types.slint | 3 + rs/crates/terminal-widget/Cargo.toml | 8 + rs/crates/terminal-widget/ui/widget.slint | 8 + 7 files changed, 284 insertions(+) diff --git a/rs/crates/app/Cargo.lock b/rs/crates/app/Cargo.lock index 6bae29b4..7efde4b0 100644 --- a/rs/crates/app/Cargo.lock +++ b/rs/crates/app/Cargo.lock @@ -18,6 +18,114 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "accesskit" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b7f7f85a7e5f68090000ed7622545829afd484d210358702ae4cb97dd0c320" +dependencies = [ + "uuid", +] + +[[package]] +name = "accesskit_atspi_common" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023da0e5097f46df7092d5280b02efb9bbf8d93298daeced42652463e357d636" +dependencies = [ + "accesskit", + "accesskit_consumer", + "atspi-common", + "phf", + "serde", + "zvariant", +] + +[[package]] +name = "accesskit_consumer" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d10a236f96f87d70732e44520046785431ef01d5bcd6b041317bfadd2f88245" +dependencies = [ + "accesskit", + "hashbrown 0.16.1", +] + +[[package]] +name = "accesskit_ios" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "750c4e9f6ce888dfe8a10c0f1b5ceb646a7854fd46579d74919219d1bb314083" +dependencies = [ + "accesskit", + "accesskit_consumer", + "hashbrown 0.16.1", + "objc2 0.5.2", + "objc2-foundation 0.2.2", + "objc2-ui-kit 0.2.2", +] + +[[package]] +name = "accesskit_macos" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce02dc63b43f0c9296af9ac946312a2dc8814427d7a64d2d600971dac55b6076" +dependencies = [ + "accesskit", + "accesskit_consumer", + "hashbrown 0.16.1", + "objc2 0.5.2", + "objc2-app-kit 0.2.2", + "objc2-foundation 0.2.2", +] + +[[package]] +name = "accesskit_unix" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e156ed3802e35eefe894ef2671bc6c889303d8a7e110b5e1b48f504b91362f" +dependencies = [ + "accesskit", + "accesskit_atspi_common", + "async-channel", + "async-executor", + "async-task", + "atspi", + "futures-lite", + "futures-util", + "serde", + "zbus", +] + +[[package]] +name = "accesskit_windows" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106c2b961215864d1c2e703ee63269c25c4e80a577ffb2c1017b9c17dcdf83a1" +dependencies = [ + "accesskit", + "accesskit_consumer", + "hashbrown 0.16.1", + "static_assertions", + "windows", + "windows-core", +] + +[[package]] +name = "accesskit_winit" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b41e63a69f36d9f1f41e70464c7e5f72eee485ef26aab19f0b4f86e6c0a84c" +dependencies = [ + "accesskit", + "accesskit_ios", + "accesskit_macos", + "accesskit_unix", + "accesskit_windows", + "raw-window-handle", + "winit", +] + [[package]] name = "adler2" version = "2.0.1" @@ -392,6 +500,43 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atspi" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77886257be21c9cd89a4ae7e64860c6f0eefca799bb79127913052bd0eefb3d" +dependencies = [ + "atspi-common", + "atspi-proxies", +] + +[[package]] +name = "atspi-common" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20c5617155740c98003016429ad13fe43ce7a77b007479350a9f8bf95a29f63d" +dependencies = [ + "enumflags2", + "serde", + "static_assertions", + "zbus", + "zbus-lockstep", + "zbus-lockstep-macros", + "zbus_names", + "zvariant", +] + +[[package]] +name = "atspi-proxies" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2230e48787ed3eb4088996eab66a32ca20c0b67bbd4fd6cdfe79f04f1f04c9fc" +dependencies = [ + "atspi-common", + "serde", + "zbus", +] + [[package]] name = "auto_enums" version = "0.8.8" @@ -2267,6 +2412,8 @@ name = "i-slint-backend-winit" version = "1.17.0" source = "git+https://github.com/slint-ui/slint?rev=9463a107aeb5f5021c162e82c55977c5333229ae#9463a107aeb5f5021c162e82c55977c5333229ae" dependencies = [ + "accesskit", + "accesskit_winit", "block2 0.6.2", "bytemuck", "cfg-if", @@ -4024,6 +4171,49 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pico-args" version = "0.5.0" @@ -4298,6 +4488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", + "serde", ] [[package]] @@ -7411,6 +7602,30 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zbus-lockstep" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6998de05217a084b7578728a9443d04ea4cd80f2a0839b8d78770b76ccd45863" +dependencies = [ + "zbus_xml", + "zvariant", +] + +[[package]] +name = "zbus-lockstep-macros" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10da05367f3a7b7553c8cdf8fa91aee6b64afebe32b51c95177957efc47ca3a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "zbus-lockstep", + "zbus_xml", + "zvariant", +] + [[package]] name = "zbus_macros" version = "5.16.0" @@ -7437,6 +7652,18 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zbus_xml" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8067892e940ed1727dea64690378601603b31d62dfde019a5335fbb7c0e0ed9" +dependencies = [ + "quick-xml", + "serde", + "zbus_names", + "zvariant", +] + [[package]] name = "zeno" version = "0.3.3" diff --git a/rs/crates/app/Cargo.toml b/rs/crates/app/Cargo.toml index 3e40332e..062340ec 100644 --- a/rs/crates/app/Cargo.toml +++ b/rs/crates/app/Cargo.toml @@ -70,6 +70,14 @@ qrcode = { version = "0.14", default-features = false } # — lifted from spike-tearoff. Bumped 0.58 -> 0.62 to unify with wgpu-hal 29 / gpu-allocator and # avoid two `windows` crate versions in the graph (which breaks wgpu-hal's d3d12 backend). # Target-gated so Linux/macOS builds never pull the Win32 bindings. +# AccessKit → AT-SPI, Linux only. The workspace drops Slint's `accessibility` feature because +# accesskit pins `windows 0.58` against wgpu-hal 29's `0.62` — a conflict that only exists +# when building for Windows. Requesting the feature per-target keeps Windows resolution +# untouched while letting the terminal grid (drawn to a texture, invisible to AT-SPI +# otherwise) expose its text to screen readers and assistive automation on Linux. +[target.'cfg(target_os = "linux")'.dependencies] +slint = { git = "https://github.com/slint-ui/slint", rev = "9463a107aeb5f5021c162e82c55977c5333229ae", default-features = false, features = ["accessibility"] } + [target.'cfg(windows)'.dependencies] # Native Open/Save dialogs via the COM IFileDialog (`windows-sys`, separate from the # `windows` crate below so it doesn't perturb the pinned 0.62 graph); no backend feature diff --git a/rs/crates/app/src/paneview.rs b/rs/crates/app/src/paneview.rs index 84fe726e..a09d5134 100644 --- a/rs/crates/app/src/paneview.rs +++ b/rs/crates/app/src/paneview.rs @@ -196,6 +196,28 @@ fn replay_cursor_pos(_app: &AppWindow, _link_active: bool) {} /// Build a model row for pane `i`. `editing` flags the pane whose label is being renamed /// inline; `show_frame`/`show_dot` are the GLOBAL Appearance prefs, folded here over each /// pane's per-pane override (a clean new pane resolves OFF, a git-project pane ON). +/// Whether an assistive client is listening. The terminal grid is drawn to a texture, so AT-SPI +/// sees an empty rectangle unless we hand it the screen as text — but extracting that text on +/// every model rebuild is wasted work when nobody is reading it. Checking once per process is +/// enough to decide: AccessKit only builds a tree after a client connects anyway. +fn a11y_wanted() -> bool { + use std::sync::OnceLock; + static WANTED: OnceLock = OnceLock::new(); + *WANTED.get_or_init(|| { + if std::env::var("HYPERPANES_A11Y").is_ok_and(|v| v == "1") { + return true; + } + if std::env::var("NO_AT_BRIDGE").is_ok_and(|v| v == "1") { + return false; + } + std::process::Command::new("gsettings") + .args(["get", "org.gnome.desktop.interface", "toolkit-accessibility"]) + .output() + .map(|o| String::from_utf8_lossy(&o.stdout).trim() == "true") + .unwrap_or(false) + }) +} + fn pane_item( ps: &PaneState, focused: bool, @@ -265,6 +287,12 @@ fn pane_item( PaneItem { surface: ps.surface.clone(), title: ps.title.clone(), + // The pane's visible text for AT-SPI; empty (and free) when nothing is listening. + a11y_text: if a11y_wanted() { + ps.pane.screen_text().into() + } else { + SharedString::new() + }, subtitle: ps.subtitle.clone().unwrap_or_default(), ai_subtitle, // The cached shell-type badge (computed once at pane creation; "" → not shown). diff --git a/rs/crates/app/ui/paneview.slint b/rs/crates/app/ui/paneview.slint index a8d6c446..8c99d23f 100644 --- a/rs/crates/app/ui/paneview.slint +++ b/rs/crates/app/ui/paneview.slint @@ -543,6 +543,8 @@ export component PaneView inherits Rectangle { width: parent.width - 4px; height: parent.height - 4px; surface: pane.surface; + title: pane.title; + a11y-text: pane.a11y-text; accent: pane.accent; chrome: false; // clickable-path hover overlay (controller-driven after hit-testing) diff --git a/rs/crates/app/ui/types.slint b/rs/crates/app/ui/types.slint index 8d4c9043..db4c8438 100644 --- a/rs/crates/app/ui/types.slint +++ b/rs/crates/app/ui/types.slint @@ -7,6 +7,9 @@ import { HiRect } from "@widgets"; export struct PaneItem { surface: image, title: string, + // Visible screen text, for assistive tech (AT-SPI). Empty unless an + // accessibility client is connected — see paneview::a11y_wanted(). + a11y-text: string, // optional secondary line under the label (user subtitle / ambient-AI line). subtitle: string, // ambient-AI summary line, already truncated to the typewriter-revealed prefix by the diff --git a/rs/crates/terminal-widget/Cargo.toml b/rs/crates/terminal-widget/Cargo.toml index 42538f16..bdb965c0 100644 --- a/rs/crates/terminal-widget/Cargo.toml +++ b/rs/crates/terminal-widget/Cargo.toml @@ -48,6 +48,14 @@ arboard = "3" # core's feature set so the runtime has time/sync/macros available. tokio = { version = "1", features = ["rt-multi-thread", "sync", "time", "macros"] } +# AccessKit → AT-SPI, Linux only. The workspace drops Slint's `accessibility` feature because +# accesskit pins `windows 0.58` against wgpu-hal 29's `0.62` — a conflict that only exists +# when building for Windows. Requesting the feature per-target keeps Windows resolution +# untouched while letting the terminal grid (drawn to a texture, invisible to AT-SPI +# otherwise) expose its text to screen readers and assistive automation on Linux. +[target.'cfg(target_os = "linux")'.dependencies] +slint = { git = "https://github.com/slint-ui/slint", rev = "9463a107aeb5f5021c162e82c55977c5333229ae", default-features = false, features = ["accessibility"] } + [dev-dependencies] # PNG encode for the headless font-fallback before/after verification bin only. png = "0.17" diff --git a/rs/crates/terminal-widget/ui/widget.slint b/rs/crates/terminal-widget/ui/widget.slint index 64e3b917..f51c5cb4 100644 --- a/rs/crates/terminal-widget/ui/widget.slint +++ b/rs/crates/terminal-widget/ui/widget.slint @@ -82,9 +82,17 @@ component SearchBtn inherits Rectangle { } export component TerminalPane inherits Rectangle { + accessible-role: text; + accessible-label: root.title; + accessible-value: root.a11y-text; + accessible-read-only: true; // --- inputs (the controller / parent layout drive these) --- in property surface; in property title: ""; + // What assistive tech reads instead of the rendered pixels: the terminal + // screen as text. The grid is drawn to a texture, so without this the pane + // is a blank rectangle to AT-SPI. + in property a11y-text: ""; in property accent: #7aa2f7; in property chrome: true;