From 6c9cb08da412eaf86e627ef54c82fdbfef242e86 Mon Sep 17 00:00:00 2001 From: Praket7 <149203433+Praket7@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:05:23 -0500 Subject: [PATCH 1/2] fix: verify Windows UIA presses and plugin launch --- crates/comptrol-platform-windows/src/lib.rs | 2 +- crates/comptrol-platform-windows/src/uia.rs | 359 ++++---------- docs/adapters.md | 2 +- docs/client-conformance.md | 2 + docs/client-integration.md | 2 +- docs/platform-brokers.md | 4 +- docs/plugin.md | 4 +- fixtures/windows/UIAutomationDecoyFixture.ps1 | 28 ++ fixtures/windows/UIAutomationFixture.ps1 | 66 +++ plugins/comptrol/.codex-plugin/plugin.json | 19 +- plugins/comptrol/.mcp.json | 14 +- plugins/comptrol/CHATGPT_SETUP.md | 11 +- plugins/comptrol/mcp.json | 15 +- .../comptrol/scripts/launch-comptrol-mcp.cjs | 55 +++ scripts/npm_mcp_conformance.py | 123 +++++ scripts/validate_plugin_package.py | 42 +- scripts/windows_uia_a02_batch.py | 273 +++++++++++ scripts/windows_uia_conformance.py | 26 +- scripts/windows_uia_edge_batch.py | 445 ++++++++++++++++++ 19 files changed, 1173 insertions(+), 319 deletions(-) create mode 100644 fixtures/windows/UIAutomationDecoyFixture.ps1 create mode 100644 plugins/comptrol/scripts/launch-comptrol-mcp.cjs create mode 100644 scripts/npm_mcp_conformance.py create mode 100644 scripts/windows_uia_a02_batch.py create mode 100644 scripts/windows_uia_edge_batch.py diff --git a/crates/comptrol-platform-windows/src/lib.rs b/crates/comptrol-platform-windows/src/lib.rs index 5d9bcbba..cb260e9d 100644 --- a/crates/comptrol-platform-windows/src/lib.rs +++ b/crates/comptrol-platform-windows/src/lib.rs @@ -7,7 +7,7 @@ mod uia; #[cfg(windows)] -pub use uia::{Action, Request, SemanticCacheEntry, UiaScopedCache, execute}; +pub use uia::{Action, Request, execute}; #[cfg(not(windows))] pub mod unsupported { diff --git a/crates/comptrol-platform-windows/src/uia.rs b/crates/comptrol-platform-windows/src/uia.rs index daa90ba1..b02a2ed1 100644 --- a/crates/comptrol-platform-windows/src/uia.rs +++ b/crates/comptrol-platform-windows/src/uia.rs @@ -1,20 +1,15 @@ #![allow(non_upper_case_globals, non_camel_case_types, clippy::collapsible_if)] use serde_json::{Value, json}; -use std::collections::HashMap; use std::sync::OnceLock; -use std::time::Instant; -use windows::Win32::Foundation::RECT; +use std::time::{Duration, Instant}; use windows::Win32::System::Com::{ CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, }; use windows::Win32::UI::Accessibility::{ - CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationElement, - IUIAutomationInvokePattern, IUIAutomationValuePattern, TreeScope_Children, - TreeScope_Descendants, UIA_AutomationIdPropertyId, UIA_BoundingRectanglePropertyId, - UIA_ButtonControlTypeId, UIA_CheckBoxControlTypeId, UIA_ComboBoxControlTypeId, - UIA_ControlTypePropertyId, UIA_EditControlTypeId, UIA_InvokePatternId, UIA_IsEnabledPropertyId, - UIA_IsOffscreenPropertyId, UIA_NamePropertyId, UIA_ProcessIdPropertyId, UIA_ValuePatternId, - UIA_WindowControlTypeId, UIA_WindowPatternId, + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern, + IUIAutomationValuePattern, TreeScope_Children, TreeScope_Descendants, UIA_ButtonControlTypeId, + UIA_CheckBoxControlTypeId, UIA_ComboBoxControlTypeId, UIA_EditControlTypeId, + UIA_InvokePatternId, UIA_ValuePatternId, }; #[allow(non_upper_case_globals)] @@ -78,157 +73,6 @@ impl OwnedRequest { } } -#[derive(Clone, Debug)] -pub struct SemanticCacheEntry { - pub process_id: u32, - pub automation_id: Option, - pub name: Option, - pub role: Option, - pub bounding_rect: Option<(f64, f64, f64, f64)>, - pub is_enabled: bool, - pub is_offscreen: bool, - pub window_handle: u64, -} - -#[derive(Debug)] -pub struct UiaScopedCache { - cache_request: IUIAutomationCacheRequest, - per_window_index: HashMap>, -} - -impl UiaScopedCache { - pub fn new(automation: &IUIAutomation) -> Result { - let cache_request = unsafe { automation.CreateCacheRequest() } - .map_err(|error| format!("cache request creation failed: {error}"))?; - unsafe { - cache_request - .AddProperty(UIA_ProcessIdPropertyId) - .map_err(|error| format!("failed to add ProcessId to cache: {error}"))?; - cache_request - .AddProperty(UIA_NamePropertyId) - .map_err(|error| format!("failed to add Name to cache: {error}"))?; - cache_request - .AddProperty(UIA_AutomationIdPropertyId) - .map_err(|error| format!("failed to add AutomationId to cache: {error}"))?; - cache_request - .AddProperty(UIA_ControlTypePropertyId) - .map_err(|error| format!("failed to add ControlType to cache: {error}"))?; - cache_request - .AddProperty(UIA_BoundingRectanglePropertyId) - .map_err(|error| format!("failed to add BoundingRectangle to cache: {error}"))?; - cache_request - .AddProperty(UIA_IsEnabledPropertyId) - .map_err(|error| format!("failed to add IsEnabled to cache: {error}"))?; - cache_request - .AddProperty(UIA_IsOffscreenPropertyId) - .map_err(|error| format!("failed to add IsOffscreen to cache: {error}"))?; - cache_request - .AddPattern(UIA_InvokePatternId) - .map_err(|error| format!("failed to add InvokePattern to cache: {error}"))?; - cache_request - .AddPattern(UIA_ValuePatternId) - .map_err(|error| format!("failed to add ValuePattern to cache: {error}"))?; - cache_request - .AddPattern(UIA_WindowPatternId) - .map_err(|error| format!("failed to add WindowPattern to cache: {error}"))?; - } - Ok(Self { - cache_request, - per_window_index: HashMap::new(), - }) - } - - pub fn index_process_windows( - &mut self, - automation: &IUIAutomation, - process_id: u32, - ) -> Result, String> { - let root = unsafe { automation.GetRootElement() } - .map_err(|error| format!("UI Automation root unavailable: {error}"))?; - let condition = unsafe { automation.CreateTrueCondition() } - .map_err(|error| format!("UI Automation condition unavailable: {error}"))?; - let cached = unsafe { - root.FindAllBuildCache(TreeScope_Descendants, &condition, &self.cache_request) - } - .map_err(|error| format!("scoped tree cache query failed: {error}"))?; - let count = unsafe { cached.Length() } - .map_err(|error| format!("cache result count failed: {error}"))?; - let mut entries = Vec::new(); - for index in 0..count.min(4096) { - let element = unsafe { cached.GetElement(index) } - .map_err(|error| format!("cache element read failed: {error}"))?; - let entry_process_id = unsafe { element.CachedProcessId() } - .map_err(|error| format!("cached process id failed: {error}"))? - as u32; - if entry_process_id != process_id { - continue; - } - let rect: RECT = unsafe { element.CachedBoundingRectangle() } - .map_err(|error| format!("cached bounding rect failed: {error}"))?; - let automation_id = unsafe { element.CachedAutomationId() } - .ok() - .and_then(|bstr| { - let s = bstr.to_string(); - if s.is_empty() { None } else { Some(s) } - }); - let name = unsafe { element.CachedName() }.ok().and_then(|bstr| { - let s = bstr.to_string(); - if s.is_empty() { None } else { Some(s) } - }); - let control_type = unsafe { element.CachedControlType() } - .map_err(|error| format!("cached control type failed: {error}"))?; - let is_enabled = unsafe { element.CachedIsEnabled() } - .map_err(|error| format!("cached enabled failed: {error}"))? - .as_bool(); - let is_offscreen = unsafe { element.CachedIsOffscreen() } - .map_err(|error| format!("cached offscreen failed: {error}"))? - .as_bool(); - let window_handle = unsafe { element.CurrentNativeWindowHandle() } - .map_err(|error| format!("window handle failed: {error}"))? - .0 as u64; - let role = match control_type { - UIA_ButtonControlTypeId => Some("button"), - UIA_CheckBoxControlTypeId => Some("checkbox"), - UIA_ComboBoxControlTypeId => Some("combobox"), - UIA_EditControlTypeId => Some("edit"), - UIA_WindowControlTypeId => Some("window"), - _ => None, - } - .map(str::to_owned); - let _ = control_type; // suppress unused warning for match arm constants - entries.push(SemanticCacheEntry { - process_id: entry_process_id, - automation_id, - name, - role, - bounding_rect: Some(( - rect.left as f64, - rect.top as f64, - (rect.right - rect.left) as f64, - (rect.bottom - rect.top) as f64, - )), - is_enabled, - is_offscreen, - window_handle, - }); - } - self.per_window_index - .insert(process_id as u64, entries.clone()); - Ok(entries) - } - - pub fn get_indexed(&self, process_id: u32) -> Vec<&SemanticCacheEntry> { - self.per_window_index - .get(&(process_id as u64)) - .map(|entries| entries.iter().collect()) - .unwrap_or_default() - } - - pub fn invalidate_process(&mut self, process_id: u32) { - self.per_window_index.remove(&(process_id as u64)); - } -} - struct UiaWorker { requests: WorkerSender, } @@ -254,17 +98,11 @@ fn worker() -> &'static UiaWorker { } Err(_) => None, }; - let mut cache = automation - .as_ref() - .and_then(|a| UiaScopedCache::new(a).ok()); while let Ok((request, response)) = _receiver.recv() { - let result = match (&com, &automation, &mut cache) { - (Ok(_), Some(automation), Some(cache)) => { - execute_once(automation, cache, request.as_request()) - } - (Ok(_), Some(_), None) => Err("UIA cache not initialized".to_owned()), - (_, None, _) => Err("UI Automation activation unavailable".to_owned()), - (Err(error), _, _) => Err(error.clone()), + let result = match (&com, &automation) { + (Ok(_), Some(automation)) => execute_once(automation, request.as_request()), + (Err(error), _) => Err(error.clone()), + (Ok(_), None) => Err("UI Automation activation unavailable".to_owned()), }; let _ = response.send(result); } @@ -285,62 +123,14 @@ pub fn execute(request: Request<'_>) -> Result { .map_err(|_| "Windows UIA worker stopped before responding".to_owned())? } -fn execute_once( - automation: &IUIAutomation, - cache: &mut UiaScopedCache, - request: Request<'_>, -) -> Result { +fn execute_once(automation: &IUIAutomation, request: Request<'_>) -> Result { let started = Instant::now(); - let entries = cache.get_indexed(request.process_id); - if !entries.is_empty() { - for entry in entries { - if matches_cached_entry(entry, &request) { - let window_handle = entry.window_handle; - let element = find_element_by_window_handle(automation, window_handle)?; - let enabled = unsafe { element.CurrentIsEnabled() } - .map_err(|error| format!("UI Automation enabled state failed: {error}"))?; - if !enabled.as_bool() { - return Err("target_disabled".to_owned()); - } - match request.action { - Action::Press => { - let pattern: IUIAutomationInvokePattern = - unsafe { element.GetCurrentPatternAs(UIA_InvokePatternId) } - .map_err(|error| format!("Invoke pattern unavailable: {error}"))?; - unsafe { pattern.Invoke() } - .map_err(|error| format!("Invoke failed: {error}"))?; - } - Action::SetValue => { - let value = request.value.ok_or("value_required")?; - let pattern: IUIAutomationValuePattern = - unsafe { element.GetCurrentPatternAs(UIA_ValuePatternId) } - .map_err(|error| format!("Value pattern unavailable: {error}"))?; - let value = windows::core::BSTR::from(value); - unsafe { pattern.SetValue(&value) } - .map_err(|error| format!("SetValue failed: {error}"))?; - } - } - let verified = verify(&element, &request)?; - return Ok(json!({ - "verified": verified, - "route": "windows_uia_cached", - "process_id": request.process_id, - "cache_hit": true, - "window_handle": window_handle, - "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0, - "mouse": "untouched", - "clipboard": "untouched" - })); - } - } - } let root = unsafe { automation.GetRootElement() } .map_err(|error| format!("UI Automation root unavailable: {error}"))?; let condition = unsafe { automation.CreateTrueCondition() } .map_err(|error| format!("UI Automation condition unavailable: {error}"))?; - let windows = - unsafe { root.FindAllBuildCache(TreeScope_Children, &condition, &cache.cache_request) } - .map_err(|error| format!("desktop window query failed: {error}"))?; + let windows = unsafe { root.FindAll(TreeScope_Children, &condition) } + .map_err(|error| format!("desktop window query failed: {error}"))?; let window_count = unsafe { windows.Length() } .map_err(|error| format!("desktop window count failed: {error}"))?; let mut matches = Vec::new(); @@ -399,59 +189,35 @@ fn execute_once( .map_err(|error| format!("SetValue failed: {error}"))?; } } - let verified = verify(&element, &request)?; - let _ = cache.index_process_windows(automation, request.process_id); + let verified = verify_after_action(automation, &element, &request)?; Ok(json!({ "verified": verified, - "route": "windows_uia_scoped_cache", + "route": "windows_uia_direct", "process_id": request.process_id, "candidate_count": 1, "bounded_nodes": bounded_nodes, "elapsed_ms": started.elapsed().as_secs_f64() * 1000.0, "mouse": "untouched", "clipboard": "untouched", - "cache_hit": false, "window_handle": window_handle, - "indexed_entries": cache.get_indexed(request.process_id).len() })) } -fn matches_cached_entry(entry: &SemanticCacheEntry, request: &Request<'_>) -> bool { - if let Some(name) = request.name { - if entry.name.as_deref() != Some(name) { - return false; - } - } - if let Some(automation_id) = request.automation_id { - if entry.automation_id.as_deref() != Some(automation_id) { - return false; - } - } - if let Some(role) = request.role { - if entry.role.as_deref() != Some(role) { - return false; - } - } - true -} - -#[allow(dead_code)] -fn verify_cached_entry(entry: &SemanticCacheEntry, request: &Request<'_>) -> Result { - match request.expected_attribute { - None => Ok(false), - Some("name") => Ok(entry.name.as_deref() == request.expected_value), - Some("enabled") => Ok(entry.is_enabled == (request.expected_value == Some("true"))), - Some(_) => Err("unsupported_verification_attribute".to_owned()), - } +fn matches_element(element: &IUIAutomationElement, request: &Request<'_>) -> Result { + matches_element_identity(element, request, None) } -fn matches_element(element: &IUIAutomationElement, request: &Request<'_>) -> Result { +fn matches_element_identity( + element: &IUIAutomationElement, + request: &Request<'_>, + expected_name: Option<&str>, +) -> Result { let process_id = unsafe { element.CurrentProcessId() } .map_err(|error| format!("UI Automation process identity failed: {error}"))?; if process_id != request.process_id as i32 { return Ok(false); } - if let Some(name) = request.name { + if let Some(name) = expected_name.or(request.name) { let current = unsafe { element.CurrentName() } .map_err(|error| format!("UI Automation name read failed: {error}"))?; if current != name { @@ -482,28 +248,79 @@ fn matches_element(element: &IUIAutomationElement, request: &Request<'_>) -> Res Ok(true) } -fn find_element_by_window_handle( +fn find_postcondition_element( automation: &IUIAutomation, - window_handle: u64, -) -> Result { + request: &Request<'_>, +) -> Result, String> { + let expected_name = request.expected_value; let root = unsafe { automation.GetRootElement() } .map_err(|error| format!("UI Automation root unavailable: {error}"))?; let condition = unsafe { automation.CreateTrueCondition() } .map_err(|error| format!("UI Automation condition unavailable: {error}"))?; - let candidates = unsafe { root.FindAll(TreeScope_Descendants, &condition) } - .map_err(|error| format!("UI Automation tree query failed: {error}"))?; - let count = unsafe { candidates.Length() } - .map_err(|error| format!("UI Automation result count failed: {error}"))?; - for index in 0..count.min(4096) { - let element = unsafe { candidates.GetElement(index) } - .map_err(|error| format!("UI Automation element read failed: {error}"))?; - let handle = unsafe { element.CurrentNativeWindowHandle() } - .map_err(|error| format!("window handle read failed: {error}"))?; - if handle.0 as u64 == window_handle { - return Ok(element); + let windows = unsafe { root.FindAll(TreeScope_Children, &condition) } + .map_err(|error| format!("desktop window query failed: {error}"))?; + let window_count = unsafe { windows.Length() } + .map_err(|error| format!("desktop window count failed: {error}"))?; + let mut matched = None; + for window_index in 0..window_count.min(256) { + let window = unsafe { windows.GetElement(window_index) } + .map_err(|error| format!("desktop window read failed: {error}"))?; + if unsafe { window.CurrentProcessId() } + .map_err(|error| format!("desktop window process identity failed: {error}"))? + != request.process_id as i32 + { + continue; + } + let candidates = unsafe { window.FindAll(TreeScope_Descendants, &condition) } + .map_err(|error| format!("UI Automation tree query failed: {error}"))?; + let count = unsafe { candidates.Length() } + .map_err(|error| format!("UI Automation result count failed: {error}"))?; + for index in 0..count.min(2048) { + let element = unsafe { candidates.GetElement(index) } + .map_err(|error| format!("UI Automation element read failed: {error}"))?; + if matches_element_identity(&element, request, expected_name)? { + if matched.is_some() { + return Ok(None); + } + matched = Some(element); + } + } + } + Ok(matched) +} + +fn verify_after_action( + automation: &IUIAutomation, + element: &IUIAutomationElement, + request: &Request<'_>, +) -> Result { + const POST_INVOKE_VERIFY_TIMEOUT: Duration = Duration::from_millis(750); + const POST_INVOKE_VERIFY_INTERVAL: Duration = Duration::from_millis(25); + + if request.action != Action::Press || request.expected_attribute != Some("name") { + return verify(element, request); + } + + let deadline = Instant::now() + POST_INVOKE_VERIFY_TIMEOUT; + loop { + // Reacquire by process, AutomationId, and role. The accessible name is + // the postcondition and may legitimately change as a result of Invoke. + let current = if request.automation_id.is_some() { + find_postcondition_element(automation, request) + .ok() + .flatten() + .unwrap_or_else(|| element.clone()) + } else { + element.clone() + }; + if verify(¤t, request)? { + return Ok(true); + } + if Instant::now() >= deadline { + return Ok(false); } + std::thread::sleep(POST_INVOKE_VERIFY_INTERVAL); } - Err("window handle not found in automation tree".to_owned()) } fn verify(element: &IUIAutomationElement, request: &Request<'_>) -> Result { diff --git a/docs/adapters.md b/docs/adapters.md index ee1b7499..94e1c12b 100644 --- a/docs/adapters.md +++ b/docs/adapters.md @@ -29,7 +29,7 @@ All adapters run out of process with loopback-only network and declared filesyst ### Creative application coverage -The Codex plugin launches a local stdio MCP process and opts into routine first-party adapter intents with `COMPTROL_ALLOW_ADAPTERS=1`, Gmail send with `COMPTROL_ALLOW_MAIL_SEND=1`, Blender/Resolve/Canva/presentation intents with `COMPTROL_ALLOW_CREATIVE_ADAPTERS=1`, browser CDP with `COMPTROL_ALLOW_BROWSER_CDP=1`, app launching, macOS Accessibility, and typed settings. This enables only `mail.send` among the R3 adapter actions; Discord message deletion and recording remain gated by `COMPTROL_ALLOW_HIGH_CONSEQUENCE_ADAPTERS`. macOS TCC permissions still apply, and browser actions require an active Comptrol browser session or configured CDP endpoint. Gmail API and Discord Bot API credentials are never extracted from signed-in apps or stored by the plugin. The Gmail API route needs `COMPTROL_GMAIL_ACCESS_TOKEN`; the Discord API route needs a bot token in `COMPTROL_DISCORD_BOT_TOKEN`, and intentionally refuses user tokens and self-bot behavior. For signed-in personal accounts, use the local browser/app UI route. +The checked-in Codex plugin launches the local stdio MCP process with browser CDP and Windows UI Automation enabled, while automatic Chrome startup is disabled. Other adapter families, Gmail send, creative editing, app launching, macOS Accessibility, and typed settings remain opt-in through their respective policy flags. Discord message deletion and recording remain gated by `COMPTROL_ALLOW_HIGH_CONSEQUENCE_ADAPTERS`. Browser actions require an active Comptrol browser session or configured CDP endpoint. Gmail API and Discord Bot API credentials are never extracted from signed-in apps or stored by the plugin. The Gmail API route needs `COMPTROL_GMAIL_ACCESS_TOKEN`; the Discord API route needs a bot token in `COMPTROL_DISCORD_BOT_TOKEN`, and intentionally refuses user tokens and self-bot behavior. For signed-in personal accounts, use the local browser/app UI route. | App | Current typed editing surface | Required live/runtime dependencies | Known ceiling | |---|---|---|---| diff --git a/docs/client-conformance.md b/docs/client-conformance.md index 27b6684a..a770dadc 100644 --- a/docs/client-conformance.md +++ b/docs/client-conformance.md @@ -4,3 +4,5 @@ The repository owned harness checks the same MCP stdio contract using profiles n This does not claim that proprietary client applications were automated. It proves the server side contract that each client must consume. Run `python3 scripts/client_conformance.py` after building the binary. +With Node.js and the global npm package installed, run `python scripts/npm_mcp_conformance.py` to exercise the packaged plugin launcher, resolve the actual `comptrolling` entry point, enumerate its registered tools, and verify `system.ping` through the launched server. + diff --git a/docs/client-integration.md b/docs/client-integration.md index 87ab9480..31518db5 100644 --- a/docs/client-integration.md +++ b/docs/client-integration.md @@ -1,6 +1,6 @@ # Client integration -Run the compiled binary with the command `comptrol mcp` as the MCP server command. +For the npm installation, configure the MCP command as `comptrolling` with no arguments. The launcher starts the bundled native runtime and keeps the stdio connection alive. If you build from source, configure the full path to the compiled `comptrol` binary with the argument `mcp` instead. The server speaks newline delimited JSON on standard input and standard output. Logs go to standard error. The same binary can be used by Codex, Claude Code, Cursor, and any other client that supports MCP stdio. diff --git a/docs/platform-brokers.md b/docs/platform-brokers.md index bdc8a3c7..8c591f1d 100644 --- a/docs/platform-brokers.md +++ b/docs/platform-brokers.md @@ -6,7 +6,7 @@ On macOS it checks the public System Events accessibility surface. On Windows it Detection is not actuation. A detected display or bus does not make semantic mutation available. `platform.broker.observe` returns the broker descriptor and the explicit read only boundary. The action capability stays unavailable until its adapter and verification suite pass. -Windows semantic routes use the direct Rust COM UI Automation adapter in `comptrol-platform-windows`. `windows.uia.press` requires an exact process id plus a name or automation id and uses InvokePattern. `windows.uia.set_value` uses ValuePattern and reads the value back before reporting verified. The route uses one bounded COM worker call and does not spawn PowerShell in the hot path, synthesize mouse or keyboard input, or silently foreground the target. `COMPTROL_WINDOWS_UIA_LEGACY=1` is an explicit compatibility escape hatch for older hosts only. +Windows semantic routes use the direct Rust COM UI Automation adapter in `comptrol-platform-windows`. `windows.uia.press` requires an exact process id plus a name or automation id and uses InvokePattern. When a press postcondition checks a changing accessible name, Comptrol reacquires the control by process, AutomationId, and role and polls for up to 750 ms before returning `unverified`. It does not resolve a cached window handle back to a possibly different control. `windows.uia.set_value` uses ValuePattern and reads the value back before reporting verified. The route uses a bounded COM worker call and does not spawn PowerShell in the hot path, synthesize mouse or keyboard input, or silently foreground the target. `COMPTROL_WINDOWS_UIA_LEGACY=1` is an explicit compatibility escape hatch for older hosts only. Linux semantic routes use the direct Rust `comptrol-platform-linux` AT-SPI/zbus adapter when the session exposes the bus. `linux.atspi.press` resolves one accessible by exact process id, name, and optional role, then invokes its action interface. `linux.atspi.set_value` uses the editable text interface when available and verifies the resulting text. The old Python fixture provider is retained only for non-Linux compatibility diagnostics; it is not used by the Linux hot path. Wayland input injection remains separate and unsupported until a portal and libei consent path is validated. @@ -14,7 +14,7 @@ Doctor reports broker state explicitly. A configured but read only broker is deg macOS now includes a target-gated direct Rust Application Services adapter in `comptrol-platform-macos`. When an exact `process_id` is supplied, `macos.ax.press` and `macos.ax.set_value` use AXUIElement APIs directly, with bounded tree traversal and readback verification; the AppleScript route remains a compatibility fallback when no process identity is supplied. The macOS fixture source is `fixtures/macos/AccessibilityFixture.swift`. `scripts/macos_ax_conformance.py` compiles it and exercises an AX press with a name postcondition. The Windows fixture is `fixtures/windows/UIAutomationFixture.ps1` and the Linux fixture is `fixtures/linux/atspi_fixture.py`. Their conformance harnesses skip on other operating systems and fail when explicitly required but unavailable. -The live macOS AX and Windows UI Automation fixtures are opt-in in GitHub Actions because they require an interactive desktop session; macOS also requires Accessibility permission. Set `COMPTROL_RUN_LIVE_MACOS_AX` or `COMPTROL_RUN_LIVE_WINDOWS_UIA` to `true` only on a runner with the required desktop session. Run `python scripts/windows_uia_conformance.py` locally on an interactive Windows desktop to exercise the UIA fixture. +The live macOS AX and Windows UI Automation fixtures are opt-in in GitHub Actions because they require an interactive desktop session; macOS also requires Accessibility permission. Set `COMPTROL_RUN_LIVE_MACOS_AX` or `COMPTROL_RUN_LIVE_WINDOWS_UIA` to `true` only on a runner with the required desktop session. On Windows, run `python scripts/npm_mcp_conformance.py` to check the installed npm launcher, MCP tool registration, and `system.ping`; run `python scripts/windows_uia_conformance.py` on an interactive desktop to press the synthetic fixture control and compare Comptrol's result to the fixture-owned state file. Application launch has a separate opt in harness in `scripts/macos_app_conformance.py`. It launches the exact installed application name through LaunchServices and verifies that the process is present. The normal check does not activate an application. Set `COMPTROL_RUN_LIVE_APP_CONFORMANCE=1` to run the live check. diff --git a/docs/plugin.md b/docs/plugin.md index 7b081f13..a1c25a9f 100644 --- a/docs/plugin.md +++ b/docs/plugin.md @@ -1,8 +1,8 @@ # ChatGPT and Codex plugin -The repository contains a portable Agent Plugins package at `plugins/comptrol`. Its root `plugin.json` uses the Agent Plugins schema, both MCP manifests launch the local `comptrol mcp` stdio runtime, its compatibility manifest remains under `.codex-plugin/plugin.json`, and its skill explains the verified-control contract. +The repository contains a portable Agent Plugins package at `plugins/comptrol`. Its root `plugin.json` uses the Agent Plugins schema, its portable MCP entry declares a `stdio` server using a Node launcher that resolves the installed global `comptrolling` package across operating systems, its compatibility manifest remains under `.codex-plugin/plugin.json`, and its skill explains the verified-control contract. -The repo marketplace is `.agents/plugins/marketplace.json`. Validate the package with `python3 scripts/validate_plugin_package.py` and the Codex compatibility manifest with the plugin creator validator. +The repo marketplace is `.agents/plugins/marketplace.json` and retains the existing `personal` marketplace identity. From the repository root, register and install it with `codex plugin marketplace add .` and `codex plugin add comptrol@personal`. Verify `codex plugin list` shows Comptrol as installed and enabled, then restart the Codex desktop app to load its MCP tools in new tasks. Validate the package with `python scripts/validate_plugin_package.py`. Follow [`plugins/comptrol/CHATGPT_SETUP.md`](../plugins/comptrol/CHATGPT_SETUP.md) to install the local Codex plugin and grant macOS Accessibility access. This package does not connect to ChatGPT web: ChatGPT custom apps require a remote MCP connection, and a local stdio command cannot be launched by a browser chat. Secure MCP Tunnel is a separate connection that this repository does not configure. Plan availability for write-capable custom MCP apps must be checked in ChatGPT; do not assume Plus grants it. diff --git a/fixtures/windows/UIAutomationDecoyFixture.ps1 b/fixtures/windows/UIAutomationDecoyFixture.ps1 new file mode 100644 index 00000000..2b299f05 --- /dev/null +++ b/fixtures/windows/UIAutomationDecoyFixture.ps1 @@ -0,0 +1,28 @@ +param( + [string]$StatePath +) + +Add-Type -AssemblyName PresentationFramework +Add-Type -AssemblyName PresentationCore + +$state = [ordered]@{ status = 'Ready'; decoy_count = 0 } +if ($StatePath) { + $directory = Split-Path -Parent $StatePath + if ($directory) { New-Item -ItemType Directory -Force -Path $directory | Out-Null } + [System.IO.File]::WriteAllText($StatePath, ($state | ConvertTo-Json -Compress), [System.Text.UTF8Encoding]::new($false)) +} + +$window = New-Object Windows.Window +$window.Title = 'Comptrol UIA Decoy Fixture' +$button = New-Object Windows.Controls.Button +$button.Content = 'Unrelated' +[Windows.Automation.AutomationProperties]::SetName($button, 'Unrelated') +[Windows.Automation.AutomationProperties]::SetAutomationId($button, 'UnrelatedButton') +$button.Add_Click({ + $state.decoy_count++ + if ($StatePath) { + [System.IO.File]::WriteAllText($StatePath, ($state | ConvertTo-Json -Compress), [System.Text.UTF8Encoding]::new($false)) + } +}) +$window.Content = $button +[void]$window.ShowDialog() diff --git a/fixtures/windows/UIAutomationFixture.ps1 b/fixtures/windows/UIAutomationFixture.ps1 index 753f7207..df818c39 100644 --- a/fixtures/windows/UIAutomationFixture.ps1 +++ b/fixtures/windows/UIAutomationFixture.ps1 @@ -1,17 +1,83 @@ +param( + [string]$StatePath = $env:COMPTROL_FIXTURE_STATE +) + Add-Type -AssemblyName PresentationFramework Add-Type -AssemblyName PresentationCore +$script:fixtureState = [ordered]@{ + status = 'Ready' + submission_count = 0 + field_value = '' + duplicate_one_count = 0 + duplicate_two_count = 0 + disabled_count = 0 +} + +function Write-FixtureState { + if ($StatePath) { + $stateDirectory = Split-Path -Parent $StatePath + if ($stateDirectory) { New-Item -ItemType Directory -Force -Path $stateDirectory | Out-Null } + [System.IO.File]::WriteAllText($StatePath, ($script:fixtureState | ConvertTo-Json -Compress), [System.Text.UTF8Encoding]::new($false)) + } +} +Write-FixtureState + $window = New-Object Windows.Window $window.Title = 'Comptrol UIA Fixture' $panel = New-Object Windows.Controls.StackPanel +$textBox = New-Object Windows.Controls.TextBox +$textBox.Text = '' +[Windows.Automation.AutomationProperties]::SetName($textBox, 'Synthetic value') +[Windows.Automation.AutomationProperties]::SetAutomationId($textBox, 'SyntheticValue') +$textBox.Add_TextChanged({ + $script:fixtureState.field_value = $textBox.Text + Write-FixtureState +}) +[void]$panel.Children.Add($textBox) $button = New-Object Windows.Controls.Button $button.Content = 'Submit' [Windows.Automation.AutomationProperties]::SetName($button, 'Submit') +[Windows.Automation.AutomationProperties]::SetAutomationId($button, 'SubmitButton') $button.Add_Click({ $button.Content = 'Submitted' [Windows.Automation.AutomationProperties]::SetName($button, 'Submitted') + $script:fixtureState.status = 'Submitted' + $script:fixtureState.submission_count++ + Write-FixtureState }) [void]$panel.Children.Add($button) + +$duplicateOne = New-Object Windows.Controls.Button +$duplicateOne.Content = 'First duplicate' +[Windows.Automation.AutomationProperties]::SetName($duplicateOne, 'Duplicate') +[Windows.Automation.AutomationProperties]::SetAutomationId($duplicateOne, 'DuplicateOne') +$duplicateOne.Add_Click({ + $script:fixtureState.duplicate_one_count++ + Write-FixtureState +}) +[void]$panel.Children.Add($duplicateOne) + +$duplicateTwo = New-Object Windows.Controls.Button +$duplicateTwo.Content = 'Second duplicate' +[Windows.Automation.AutomationProperties]::SetName($duplicateTwo, 'Duplicate') +[Windows.Automation.AutomationProperties]::SetAutomationId($duplicateTwo, 'DuplicateTwo') +$duplicateTwo.Add_Click({ + $script:fixtureState.duplicate_two_count++ + Write-FixtureState +}) +[void]$panel.Children.Add($duplicateTwo) + +$disabled = New-Object Windows.Controls.Button +$disabled.Content = 'Disabled' +[Windows.Automation.AutomationProperties]::SetName($disabled, 'Disabled') +[Windows.Automation.AutomationProperties]::SetAutomationId($disabled, 'DisabledButton') +$disabled.IsEnabled = $false +$disabled.Add_Click({ + $script:fixtureState.disabled_count++ + Write-FixtureState +}) +[void]$panel.Children.Add($disabled) $window.Content = $panel $window.Width = 320 $window.Height = 160 diff --git a/plugins/comptrol/.codex-plugin/plugin.json b/plugins/comptrol/.codex-plugin/plugin.json index 9c01f80b..ec368486 100644 --- a/plugins/comptrol/.codex-plugin/plugin.json +++ b/plugins/comptrol/.codex-plugin/plugin.json @@ -1,19 +1,26 @@ { "name": "comptrol", "version": "0.1.67", - "description": "Local Codex computer control with verified browser, desktop, settings, and application routes.", + "description": "Local first, policy bound computer control with verified browser and desktop routes.", "author": { - "name": "Local developer" + "name": "Comptrol contributors", + "url": "https://github.com/Praket7/Comptrol" }, "skills": "./skills/", "interface": { "displayName": "Comptrol", - "shortDescription": "Verified local computer control.", - "longDescription": "Start Comptrol locally from Codex to control supported browser, desktop, mail, Discord, settings, and creative app routes.", - "developerName": "Local developer", + "shortDescription": "Verified local computer control", + "longDescription": "Use policy-bound browser and desktop routes with explicit target identity, recovery, and outcome verification.", + "developerName": "Comptrol contributors", "category": "Productivity", "capabilities": ["Read", "Write"], - "defaultPrompt": "Use the local Comptrol MCP for bounded computer work. Inspect available capabilities and permissions before acting, then verify the changed application or saved artifact." + "websiteURL": "https://github.com/Praket7/Comptrol", + "privacyPolicyURL": "https://github.com/Praket7/Comptrol/blob/main/PRIVACY.md", + "defaultPrompt": [ + "Use Comptrol for a local action only when the exact target and verification level are clear.", + "Prefer a verified programmatic or browser route over screenshot-driven interaction." + ], + "brandColor": "#2457A6" }, "mcpServers": "./.mcp.json", "apps": "./.app.json" diff --git a/plugins/comptrol/.mcp.json b/plugins/comptrol/.mcp.json index 270e6dbe..148455d1 100644 --- a/plugins/comptrol/.mcp.json +++ b/plugins/comptrol/.mcp.json @@ -1,16 +1,14 @@ { "mcpServers": { "comptrol-local": { - "command": "comptrol", - "args": ["mcp"], + "command": "node", + "args": ["scripts/launch-comptrol-mcp.cjs"], + "cwd": "${PLUGIN_ROOT}", "env": { - "COMPTROL_ALLOW_ADAPTERS": "1", - "COMPTROL_ALLOW_MAIL_SEND": "1", "COMPTROL_ALLOW_BROWSER_CDP": "1", - "COMPTROL_ALLOW_CREATIVE_ADAPTERS": "1", - "COMPTROL_ALLOW_APP_LAUNCH": "1", - "COMPTROL_ALLOW_MACOS_AX": "1", - "COMPTROL_ALLOW_SETTINGS": "1" + "COMPTROL_ALLOW_WINDOWS_UIA": "1", + "COMPTROL_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0" } } } diff --git a/plugins/comptrol/CHATGPT_SETUP.md b/plugins/comptrol/CHATGPT_SETUP.md index 0aba3b1b..71dd2079 100644 --- a/plugins/comptrol/CHATGPT_SETUP.md +++ b/plugins/comptrol/CHATGPT_SETUP.md @@ -4,7 +4,16 @@ This folder connects the Comptrol runtime to Codex on your computer. It does not For the npm setup, macOS permission steps, browser connection, and hosted ChatGPT limits, see the [start guide](../../docs/start.md). -To build directly from this repository, install Rust, run `cargo build --release`, then set the command in `.mcp.json` to the full path of `target/release/comptrol`. Set the arguments to `mcp`. Restart Codex after saving the file. +Install the runtime with `npm install -g comptrolling`. From the repository root, register and install the Codex plugin with `codex plugin marketplace add .` and `codex plugin add comptrol@personal`. Check `codex plugin list` for `installed, enabled`, then restart the Codex desktop app so the plugin MCP tools load in new tasks. Enable the bundled server in `~/.codex/config.toml` if it is not already enabled: + +```toml +[plugins."comptrol@personal".mcp_servers.comptrol-local] +enabled = true +``` + +The package opts into browser CDP and Windows UI Automation, and disables automatic Chrome startup. UI Automation remains limited to Windows and the exact process and control supplied in each request. + +If you build directly from this repository, install Rust and run `cargo build --release`. Update both `mcp.json` and `.mcp.json` to launch the full path to `target/release/comptrol` with `args` set to [`mcp`]; retain `"type": "stdio"` in portable `mcp.json`. On macOS, semantic control needs Accessibility permission for the Comptrol executable itself. Open System Settings, choose Privacy and Security, then Accessibility. Add that executable. Restart Codex afterward. Do not grant permission to Terminal in place of Comptrol. diff --git a/plugins/comptrol/mcp.json b/plugins/comptrol/mcp.json index bfcace93..16d6e855 100644 --- a/plugins/comptrol/mcp.json +++ b/plugins/comptrol/mcp.json @@ -2,16 +2,15 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json", "mcpServers": { "comptrol-local": { - "command": "comptrol", - "args": ["mcp"], + "type": "stdio", + "command": "node", + "args": ["scripts/launch-comptrol-mcp.cjs"], + "cwd": "${PLUGIN_ROOT}", "env": { - "COMPTROL_ALLOW_ADAPTERS": "1", - "COMPTROL_ALLOW_MAIL_SEND": "1", "COMPTROL_ALLOW_BROWSER_CDP": "1", - "COMPTROL_ALLOW_CREATIVE_ADAPTERS": "1", - "COMPTROL_ALLOW_APP_LAUNCH": "1", - "COMPTROL_ALLOW_MACOS_AX": "1", - "COMPTROL_ALLOW_SETTINGS": "1" + "COMPTROL_ALLOW_WINDOWS_UIA": "1", + "COMPTROL_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0" } } } diff --git a/plugins/comptrol/scripts/launch-comptrol-mcp.cjs b/plugins/comptrol/scripts/launch-comptrol-mcp.cjs new file mode 100644 index 00000000..8428bd8f --- /dev/null +++ b/plugins/comptrol/scripts/launch-comptrol-mcp.cjs @@ -0,0 +1,55 @@ +#!/usr/bin/env node +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); + +function findLauncher() { + const names = + process.platform === "win32" + ? ["comptrolling.cmd", "comptrolling.exe", "comptrolling"] + : ["comptrolling"]; + for (const directory of (process.env.PATH || "").split(path.delimiter)) { + for (const name of names) { + const candidate = path.join(directory, name); + if (fs.existsSync(candidate)) return candidate; + } + } + throw new Error("The global comptrolling npm launcher was not found on PATH"); +} + +function findServerEntry(launcher) { + if (process.platform === "win32" && launcher.toLowerCase().endsWith(".cmd")) { + return path.join( + path.dirname(launcher), + "node_modules", + "comptrolling", + "bin", + "comptrol-mcp.js", + ); + } + return fs.realpathSync(launcher); +} + +let serverEntry; +try { + serverEntry = findServerEntry(findLauncher()); + if (!fs.statSync(serverEntry).isFile()) { + throw new Error("Comptrol MCP entry is not a file: " + serverEntry); + } +} catch (error) { + process.stderr.write("Comptrol MCP launcher: " + error.message + "\n"); + process.exit(1); +} + +const child = spawn(process.execPath, [serverEntry], { + env: process.env, + stdio: "inherit", +}); + +child.on("error", (error) => { + process.stderr.write("Comptrol MCP process failed: " + error.message + "\n"); + process.exit(1); +}); +child.on("exit", (code) => process.exit(code ?? 1)); diff --git a/scripts/npm_mcp_conformance.py b/scripts/npm_mcp_conformance.py new file mode 100644 index 00000000..8c1eac93 --- /dev/null +++ b/scripts/npm_mcp_conformance.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Exercise the installed npm launcher through MCP initialize, tools/list, and ping.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import time +from pathlib import Path + + +def call(process: subprocess.Popen[str], message: dict) -> dict: + assert process.stdin is not None and process.stdout is not None + process.stdin.write(json.dumps(message) + "\n") + process.stdin.flush() + line = process.stdout.readline() + if not line: + stderr = process.stderr.read() if process.stderr else "" + raise RuntimeError(f"npm launcher closed before replying (exit={process.poll()}): {stderr}") + response = json.loads(line) + if "error" in response: + raise RuntimeError(response) + return response + + +def main() -> None: + root = Path(__file__).resolve().parents[1] + plugin_root = root / "plugins" / "comptrol" + config = json.loads((plugin_root / "mcp.json").read_text(encoding="utf-8")) + server = config["mcpServers"]["comptrol-local"] + node = shutil.which(server["command"]) + if not node: + raise SystemExit(f"Node.js executable not found: {server['command']}") + launcher_script = plugin_root / server["args"][0] + if not launcher_script.is_file(): + raise SystemExit(f"Packaged Comptrol launcher is missing: {launcher_script}") + + with tempfile.TemporaryDirectory(prefix="comptrol-npm-mcp-") as state_dir: + environment = dict(os.environ, **server["env"]) + environment.update( + COMPTROL_STATE_DIR=state_dir, + PLUGIN_ROOT=str(plugin_root), + PLUGIN_DATA=state_dir, + ) + process = subprocess.Popen( + [node, *server["args"]], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + env=environment, + cwd=plugin_root, + ) + try: + initialized = call( + process, + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "comptrol-npm-conformance", "version": "1.0"}, + }, + }, + ) + assert process.stdin is not None + process.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n") + process.stdin.flush() + tools = call(process, {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + ping = call( + process, + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": { + "name": "operate", + "arguments": {"intent": "system.ping", "idempotency_key": "npm-launcher-conformance"}, + }, + }, + ) + names = {tool["name"] for tool in tools["result"]["tools"]} + required = {"operate", "inspect", "watch", "reconcile", "capabilities"} + assert initialized["result"]["serverInfo"]["name"] == "comptrol", initialized + assert required <= names, sorted(names) + assert ping["result"]["structuredContent"]["verification"] == "verified", ping + print( + json.dumps( + { + "launcher": str(launcher_script), + "server": initialized["result"]["serverInfo"], + "registered_tools": sorted(names), + "system_ping": ping["result"]["structuredContent"]["verification"], + }, + sort_keys=True, + ) + ) + finally: + if os.name == "nt": + subprocess.run( + ["taskkill.exe", "/PID", str(process.pid), "/T", "/F"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + time.sleep(0.25) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_plugin_package.py b/scripts/validate_plugin_package.py index b49543ec..777ed056 100644 --- a/scripts/validate_plugin_package.py +++ b/scripts/validate_plugin_package.py @@ -30,27 +30,43 @@ def main(): fail("portable plugin name is not comptrol") if manifest.get("version") != compatibility.get("version"): fail("portable and compatibility versions differ") - if not isinstance(manifest.get("extensions", {}).get("com.openai", {}).get("interface"), dict): + portable_interface = manifest.get("extensions", {}).get("com.openai", {}).get("interface") + if not isinstance(portable_interface, dict): fail("OpenAI interface metadata is missing") + if compatibility.get("name") != manifest.get("name"): + fail("portable and compatibility plugin names differ") + if compatibility.get("description") != manifest.get("description"): + fail("portable and compatibility descriptions differ") + if compatibility.get("author") != manifest.get("author"): + fail("portable and compatibility authors differ") + if compatibility.get("interface") != portable_interface: + fail("portable and compatibility OpenAI interfaces differ") server = mcp.get("mcpServers", {}).get("comptrol-local") if server != { - "command": "comptrol", - "args": ["mcp"], + "type": "stdio", + "command": "node", + "args": ["scripts/launch-comptrol-mcp.cjs"], + "cwd": "${PLUGIN_ROOT}", "env": { - "COMPTROL_ALLOW_ADAPTERS": "1", - "COMPTROL_ALLOW_MAIL_SEND": "1", "COMPTROL_ALLOW_BROWSER_CDP": "1", - "COMPTROL_ALLOW_CREATIVE_ADAPTERS": "1", - "COMPTROL_ALLOW_APP_LAUNCH": "1", - "COMPTROL_ALLOW_MACOS_AX": "1", - "COMPTROL_ALLOW_SETTINGS": "1", + "COMPTROL_ALLOW_WINDOWS_UIA": "1", + "COMPTROL_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0", }, }: - fail("plugin must launch the local stdio runtime with the documented local app policy") - if codex_mcp.get("mcpServers", {}).get("comptrol-local") != server: - fail("Codex plugin and portable MCP launch configs differ") + fail("plugin must declare its stdio runtime and only the intended browser and Windows UIA policies") + legacy_server = codex_mcp.get("mcpServers", {}).get("comptrol-local") + portable_launch = {key: value for key, value in server.items() if key != "type"} + if legacy_server != portable_launch: + fail("Codex compatibility and portable MCP launch configs differ") + if mcp.get("$schema") != "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json": + fail("portable MCP schema is missing or unexpected") + if server.get("type") != "stdio": + fail("portable MCP server type must be stdio") if not (plugin / "skills" / "comptrol-verified-control" / "SKILL.md").is_file(): fail("verified control skill is missing") + if not (plugin / "scripts" / "launch-comptrol-mcp.cjs").is_file(): + fail("cross-platform npm MCP launcher is missing") entries = [entry for entry in marketplace.get("plugins", []) if entry.get("name") == "comptrol"] if len(entries) != 1: fail("marketplace must contain exactly one Comptrol entry") @@ -59,7 +75,7 @@ def main(): fail("marketplace source path is not repo relative") if entry.get("policy", {}).get("installation") != "AVAILABLE": fail("marketplace installation policy is not explicit") - print(json.dumps({"plugin": manifest["name"], "version": manifest["version"], "mcp_command": server["command"], "marketplace": True}, sort_keys=True)) + print(json.dumps({"plugin": manifest["name"], "version": manifest["version"], "mcp_command": server["command"], "mcp_cwd": server["cwd"], "marketplace": True}, sort_keys=True)) if __name__ == "__main__": diff --git a/scripts/windows_uia_a02_batch.py b/scripts/windows_uia_a02_batch.py new file mode 100644 index 00000000..b4c52364 --- /dev/null +++ b/scripts/windows_uia_a02_batch.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Repeat A02 against the synthetic Windows UI Automation fixture.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import pathlib +import platform +import queue +import shutil +import statistics +import subprocess +import threading +import time +import uuid + + +def send(process: subprocess.Popen[str], message: dict) -> None: + assert process.stdin is not None + process.stdin.write(json.dumps(message, ensure_ascii=False) + "\n") + process.stdin.flush() + + +def response_reader(process: subprocess.Popen[str], lines: queue.Queue[str]) -> None: + assert process.stdout is not None + for line in process.stdout: + lines.put(line) + + +def receive(lines: queue.Queue[str], request_id: int, timeout: float = 30) -> dict: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + message = json.loads(lines.get(timeout=min(0.25, deadline - time.monotonic()))) + except queue.Empty: + continue + if message.get("id") == request_id: + return message + raise TimeoutError(f"Comptrol did not answer MCP request {request_id}") + + +def run_trial(root: pathlib.Path, output: pathlib.Path, label: str, index: int, case: str) -> dict: + trial_dir = output / "artifacts" / label + state_dir = trial_dir / "runtime-state" + state_dir.mkdir(parents=True, exist_ok=True) + fixture_state = trial_dir / "fixture-state.json" + value = f"{case}-SYNTHETIC-{label.upper()}-{uuid.uuid4().hex[:12]}" + powershell = shutil.which("powershell.exe") or shutil.which("powershell") + if not powershell: + raise RuntimeError("Windows PowerShell is required for A02") + + fixture = subprocess.Popen( + [ + powershell, + "-NoProfile", + "-STA", + "-ExecutionPolicy", + "Bypass", + "-File", + str(root / "fixtures" / "windows" / "UIAutomationFixture.ps1"), + "-StatePath", + str(fixture_state), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + runtime: subprocess.Popen[str] | None = None + started_utc = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + try: + time.sleep(2) + if fixture.poll() is not None: + error = fixture.stderr.read() if fixture.stderr else "" + raise RuntimeError(f"fixture exited before UIA action: {error}") + + env = os.environ.copy() + env.update( + { + "COMPTROL_ALLOW_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0", + "COMPTROL_STATE_DIR": str(state_dir), + } + ) + binary = env.get("COMPTROL_BIN") or str( + pathlib.Path(os.environ["APPDATA"]) + / "npm" + / "node_modules" + / "comptrolling" + / "native" + / "win32-x64" + / "comptrol.exe" + ) + if not pathlib.Path(binary).is_file(): + raise FileNotFoundError(f"Comptrol runtime not found: {binary}") + runtime = subprocess.Popen( + [binary, "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + env=env, + ) + lines: queue.Queue[str] = queue.Queue() + threading.Thread(target=response_reader, args=(runtime, lines), daemon=True).start() + target = { + "process_id": fixture.pid, + "name": "Synthetic value", + "role": "edit", + "value": value, + } + if case == "A02": + target["automation_id"] = "SyntheticValue" + send( + runtime, + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "windows-uia-a02-batch", "version": "1.0"}, + }, + }, + ) + initialized = receive(lines, 1) + server = initialized.get("result", {}).get("serverInfo", {}) + send(runtime, {"jsonrpc": "2.0", "method": "notifications/initialized"}) + + request_started = time.perf_counter() + send( + runtime, + { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "operate", + "arguments": { + "intent": "windows.uia.set_value", + "idempotency_key": f"{case}-{label}-{fixture.pid}-{uuid.uuid4().hex}", + "params": target, + "postcondition": {"attribute": "value", "equals": value}, + }, + }, + }, + ) + response = receive(lines, 2) + latency_ms = (time.perf_counter() - request_started) * 1000 + structured = response.get("result", {}).get("structuredContent", {}) + independent_state = json.loads(fixture_state.read_text(encoding="utf-8")) + independent_pass = ( + independent_state.get("field_value") == value + and independent_state.get("status") == "Ready" + and independent_state.get("submission_count") == 0 + and independent_state.get("duplicate_one_count") == 0 + and independent_state.get("duplicate_two_count") == 0 + and independent_state.get("disabled_count") == 0 + ) + verified = structured.get("verification") == "verified" + completed_utc = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return { + "task_id": case, + "iteration": label, + "started_utc": started_utc, + "ended_utc": completed_utc, + "server_version": server.get("version"), + "route": structured.get("route"), + "intent": "windows.uia.set_value", + "target_process_id": fixture.pid, + "target_name": "Synthetic value", + "target_automation_id": "SyntheticValue" if case == "A02" else None, + "result_code": structured.get("error", {}).get("code") if structured.get("error") else "ok", + "verification": structured.get("verification"), + "independent_verifier": independent_pass, + "latency_ms": round(latency_ms, 3), + "tool_call_count": 1, + "model_turns": 0, + "retries": 0, + "recovery_actions": 0, + "fixture_state_artifact": str(fixture_state), + "synthetic_value": value, + "independent_state": independent_state, + "success": bool(verified and independent_pass and structured.get("effect") == "changed"), + } + finally: + if runtime is not None: + if runtime.poll() is None: + runtime.terminate() + try: + runtime.wait(timeout=5) + except subprocess.TimeoutExpired: + runtime.kill() + runtime.wait(timeout=5) + if fixture.poll() is None: + fixture.terminate() + try: + fixture.wait(timeout=5) + except subprocess.TimeoutExpired: + fixture.kill() + fixture.wait(timeout=5) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--case", choices=["A02", "A03"], default="A02") + parser.add_argument("--iterations", type=int, default=20) + parser.add_argument("--output", type=pathlib.Path, required=True) + args = parser.parse_args() + if platform.system() != "Windows": + raise SystemExit("A02/A03 requires an interactive Windows desktop") + root = pathlib.Path(__file__).resolve().parents[1] + args.output.mkdir(parents=True, exist_ok=False) + tasks_path = args.output / "tasks.jsonl" + results: list[dict] = [] + with tasks_path.open("w", encoding="utf-8", newline="\n") as tasks: + for label, index in [("warmup", 0)] + [(f"measured-{i:02d}", i) for i in range(1, args.iterations + 1)]: + try: + result = run_trial(root, args.output, label, index, args.case) + except Exception as error: # preserve each attempted run in the raw log + result = { + "task_id": args.case, + "iteration": label, + "result_code": type(error).__name__, + "error": str(error), + "independent_verifier": False, + "success": False, + } + tasks.write(json.dumps(result, ensure_ascii=False) + "\n") + tasks.flush() + if label != "warmup": + results.append(result) + print(json.dumps({"iteration": label, "success": result.get("success"), "verification": result.get("verification"), "latency_ms": result.get("latency_ms"), "error": result.get("error")}), flush=True) + + latencies = [float(row["latency_ms"]) for row in results if row.get("latency_ms") is not None] + ordered = sorted(latencies) + p95 = ordered[max(0, math.ceil(0.95 * len(ordered)) - 1)] if ordered else None + summary = { + "task_id": args.case, + "scope": "direct native Comptrol MCP runtime; not Codex end-to-end", + "package_version": "0.1.67", + "warmup_count": 1, + "measured_trials": len(results), + "verified_successes": sum(bool(row.get("success")) for row in results), + "independent_fixture_successes": sum(bool(row.get("independent_verifier")) for row in results), + "false_verified_successes": sum(row.get("verification") == "verified" and not row.get("independent_verifier") for row in results), + "median_runtime_latency_ms": statistics.median(latencies) if latencies else None, + "p95_runtime_latency_ms_nearest_rank": p95, + "min_runtime_latency_ms": min(latencies) if latencies else None, + "max_runtime_latency_ms": max(latencies) if latencies else None, + "all_measured_trials": results, + } + (args.output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + (args.output / "summary.md").write_text( + f"# {args.case} Windows UI Automation result\n\n" + f"- Package: `comptrolling@{summary['package_version']}`\n" + f"- Scope: {summary['scope']}\n" + f"- Verified success: {summary['verified_successes']}/{summary['measured_trials']}\n" + f"- Independent fixture success: {summary['independent_fixture_successes']}/{summary['measured_trials']}\n" + f"- False verified success: {summary['false_verified_successes']}\n" + f"- Runtime latency, n={len(latencies)}: median {summary['median_runtime_latency_ms']}, p95 {p95}, min {summary['min_runtime_latency_ms']}, max {summary['max_runtime_latency_ms']} ms\n", + encoding="utf-8", + ) + return 0 if len(results) == args.iterations and summary["verified_successes"] == args.iterations else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/windows_uia_conformance.py b/scripts/windows_uia_conformance.py index 08bf6d64..162daf69 100644 --- a/scripts/windows_uia_conformance.py +++ b/scripts/windows_uia_conformance.py @@ -33,10 +33,16 @@ def call(process, identifier, method, params): root = pathlib.Path(__file__).resolve().parents[1] binary = os.environ.get("COMPTROL_BIN", str(root / "target" / "debug" / "comptrol.exe")) -fixture = subprocess.Popen([powershell, "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", str(root / "fixtures" / "windows" / "UIAutomationFixture.ps1")]) +state_dir = pathlib.Path(os.environ.get("COMPTROL_STATE_DIR", str(root / "target" / "windows-uia-state"))) +state_dir.mkdir(parents=True, exist_ok=True) +fixture_state = pathlib.Path(os.environ.get("COMPTROL_FIXTURE_STATE", str(state_dir / "fixture-state.json"))) +fixture = subprocess.Popen([ + powershell, "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", + str(root / "fixtures" / "windows" / "UIAutomationFixture.ps1"), "-StatePath", str(fixture_state), +]) runtime = None try: - time.sleep(1) + time.sleep(3) if fixture.poll() is not None: raise RuntimeError("Windows UI Automation fixture exited before it could be inspected") runtime = subprocess.Popen( @@ -47,14 +53,16 @@ def call(process, identifier, method, params): env={ **os.environ, "COMPTROL_ALLOW_WINDOWS_UIA": "1", - "COMPTROL_STATE_DIR": str(root / "target" / "windows-uia-state"), + "COMPTROL_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0", + "COMPTROL_STATE_DIR": str(state_dir), }, ) call(runtime, 1, "initialize", {}) result = call(runtime, 2, "tools/call", {"name": "operate", "arguments": { "intent": "windows.uia.press", - "idempotency_key": "windows-uia-press", - "params": {"process_id": fixture.pid, "name": "Submit", "role": "button"}, + "idempotency_key": f"windows-uia-press-{fixture.pid}", + "params": {"process_id": fixture.pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"}, "postcondition": {"attribute": "name", "equals": "Submitted"}, }}) structured = result["result"]["structuredContent"] @@ -62,6 +70,14 @@ def call(process, identifier, method, params): raise RuntimeError(structured) assert structured["verification"] == "verified", structured assert structured["route"] == "windows_uia_direct", structured + fixture_result = json.loads(fixture_state.read_text(encoding="utf-8")) + assert fixture_result.get("status") == "Submitted", fixture_result + assert fixture_result.get("submission_count") == 1, fixture_result + assert fixture_result.get("field_value") == "", fixture_result + assert fixture_result.get("duplicate_one_count") == 0, fixture_result + assert fixture_result.get("duplicate_two_count") == 0, fixture_result + assert fixture_result.get("disabled_count") == 0, fixture_result + print(json.dumps({"runtime": structured, "independent_fixture_state": fixture_result}, ensure_ascii=False)) print("Windows UI Automation conformance passed") finally: if runtime is not None: diff --git a/scripts/windows_uia_edge_batch.py b/scripts/windows_uia_edge_batch.py new file mode 100644 index 00000000..8d3ac1d3 --- /dev/null +++ b/scripts/windows_uia_edge_batch.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +"""Run the refusal, stale-target, idempotency, and restart UIA cases.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import pathlib +import platform +import queue +import shutil +import statistics +import subprocess +import threading +import time +import uuid + + +class McpClient: + def __init__(self, binary: str, state_dir: pathlib.Path): + env = os.environ.copy() + env.update( + { + "COMPTROL_ALLOW_WINDOWS_UIA": "1", + "COMPTROL_AUTO_START_CHROME_CDP": "0", + "COMPTROL_STATE_DIR": str(state_dir), + } + ) + self.process = subprocess.Popen( + [binary, "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + bufsize=1, + env=env, + ) + self.lines: queue.Queue[str] = queue.Queue() + threading.Thread(target=self._read_lines, daemon=True).start() + self.request_id = 0 + init = self.request( + "initialize", + { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "windows-uia-edge-batch", "version": "1.0"}, + }, + ) + if init.get("error") or init.get("result", {}).get("serverInfo", {}).get("name") != "comptrol": + raise RuntimeError(f"Comptrol MCP initialization failed: {init}") + self.server_info = init["result"]["serverInfo"] + self.notify("notifications/initialized") + + def _read_lines(self) -> None: + assert self.process.stdout is not None + for line in self.process.stdout: + self.lines.put(line) + + def request(self, method: str, params: dict) -> dict: + self.request_id += 1 + request_id = self.request_id + self.send({"jsonrpc": "2.0", "id": request_id, "method": method, "params": params}) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + message = json.loads(self.lines.get(timeout=min(0.25, deadline - time.monotonic()))) + except queue.Empty: + continue + if message.get("id") == request_id: + return message + raise TimeoutError(f"Comptrol did not answer MCP request {request_id}") + + def send(self, message: dict) -> None: + assert self.process.stdin is not None + self.process.stdin.write(json.dumps(message, ensure_ascii=False) + "\n") + self.process.stdin.flush() + + def notify(self, method: str) -> None: + self.send({"jsonrpc": "2.0", "method": method}) + + def operate(self, intent: str, key: str, params: dict, postcondition: dict | None = None) -> tuple[dict, float]: + arguments = {"intent": intent, "idempotency_key": key, "params": params} + if postcondition is not None: + arguments["postcondition"] = postcondition + started = time.perf_counter() + response = self.request("tools/call", {"name": "operate", "arguments": arguments}) + return unpack(response), (time.perf_counter() - started) * 1000 + + def close(self) -> None: + if self.process.poll() is None: + if self.process.stdin: + self.process.stdin.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + + +def unpack(response: dict) -> dict: + result = response.get("result", {}) + structured = result.get("structuredContent") + if isinstance(structured, dict): + return structured + for item in result.get("content", []): + text = item.get("text") + if text: + try: + return json.loads(text) + except json.JSONDecodeError: + pass + return {"transport_error": response.get("error") or result} + + +def launch_fixture(script: pathlib.Path, state_path: pathlib.Path, powershell: str) -> subprocess.Popen[str]: + state_path.parent.mkdir(parents=True, exist_ok=True) + return subprocess.Popen( + [powershell, "-NoProfile", "-STA", "-ExecutionPolicy", "Bypass", "-File", str(script), "-StatePath", str(state_path)], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + + +def stop_process(process: subprocess.Popen[str] | None) -> None: + if process is None or process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def state(path: pathlib.Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def ready_state() -> dict: + return { + "status": "Ready", + "submission_count": 0, + "field_value": "", + "duplicate_one_count": 0, + "duplicate_two_count": 0, + "disabled_count": 0, + } + + +def submitted_state() -> dict: + expected = ready_state() + expected["status"] = "Submitted" + expected["submission_count"] = 1 + return expected + + +def refusal_code(result: dict) -> str | None: + error = result.get("error") + return error.get("code") if isinstance(error, dict) else None + + +def trial(root: pathlib.Path, output: pathlib.Path, case: str, label: str, binary: str, powershell: str) -> dict: + artifacts = output / "artifacts" / label + artifacts.mkdir(parents=True, exist_ok=True) + state_dir = artifacts / "runtime-state" + fixture_script = root / "fixtures" / "windows" / "UIAutomationFixture.ps1" + fixture_state = artifacts / "fixture-state.json" + main: subprocess.Popen[str] | None = None + decoy: subprocess.Popen[str] | None = None + runtime: McpClient | None = None + call_latencies: list[float] = [] + start_utc = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + response_summaries: list[dict] = [] + recovery_ms: float | None = None + try: + main = launch_fixture(fixture_script, fixture_state, powershell) + time.sleep(2) + if main.poll() is not None: + raise RuntimeError("main fixture exited before action") + runtime = McpClient(binary, state_dir) + main_pid = main.pid + + if case == "A04": + result, latency = runtime.operate( + "windows.uia.press", + f"{case}-{label}-{uuid.uuid4().hex}", + {"process_id": main_pid, "name": "Duplicate", "role": "button"}, + ) + call_latencies.append(latency) + fixture = state(fixture_state) + code = refusal_code(result) + safe = code == "target_ambiguous" and fixture == ready_state() + response_summaries.append({"result_code": code, "effect": result.get("effect"), "recovery": result.get("recovery")}) + independent = fixture == ready_state() + success = safe + + elif case == "A05": + decoy_state = artifacts / "decoy-state.json" + decoy = launch_fixture(root / "fixtures" / "windows" / "UIAutomationDecoyFixture.ps1", decoy_state, powershell) + time.sleep(1) + if decoy.poll() is not None: + raise RuntimeError("decoy fixture exited before wrong-process test") + result, latency = runtime.operate( + "windows.uia.press", + f"{case}-{label}-{uuid.uuid4().hex}", + {"process_id": decoy.pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"}, + ) + call_latencies.append(latency) + main_state, decoy_value = state(fixture_state), state(decoy_state) + code = refusal_code(result) + independent = main_state == ready_state() and decoy_value == {"status": "Ready", "decoy_count": 0} + success = code in ("target_gone", "target_missing") and independent + response_summaries.append({"result_code": code, "effect": result.get("effect"), "recovery": result.get("recovery")}) + + elif case == "A06": + result, latency = runtime.operate( + "windows.uia.press", + f"{case}-{label}-{uuid.uuid4().hex}", + {"process_id": main_pid, "name": "Disabled", "automation_id": "DisabledButton", "role": "button"}, + ) + call_latencies.append(latency) + fixture = state(fixture_state) + code = refusal_code(result) + independent = fixture == ready_state() + success = code in ("not_actionable", "target_gone", "target_missing") and independent + response_summaries.append({"result_code": code, "effect": result.get("effect"), "recovery": result.get("recovery")}) + + elif case == "A07": + recovery_started = time.perf_counter() + old_pid = main_pid + stop_process(main) + main = launch_fixture(fixture_script, artifacts / "reopened-fixture-state.json", powershell) + time.sleep(2) + if main.poll() is not None: + raise RuntimeError("reopened fixture exited before stale target test") + new_state_path = artifacts / "reopened-fixture-state.json" + if main.pid == old_pid: + raise RuntimeError("operating system reused the stale fixture process ID") + stale, latency = runtime.operate( + "windows.uia.press", + f"{case}-stale-{label}-{uuid.uuid4().hex}", + {"process_id": old_pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"}, + {"attribute": "name", "equals": "Submitted"}, + ) + call_latencies.append(latency) + stale_state = state(new_state_path) + fresh_value = f"A07-FRESH-{uuid.uuid4().hex[:12]}" + fresh, latency = runtime.operate( + "windows.uia.set_value", + f"{case}-fresh-{label}-{uuid.uuid4().hex}", + {"process_id": main.pid, "name": "Synthetic value", "automation_id": "SyntheticValue", "role": "edit", "value": fresh_value}, + {"attribute": "value", "equals": fresh_value}, + ) + call_latencies.append(latency) + recovery_ms = (time.perf_counter() - recovery_started) * 1000 + final_state = state(new_state_path) + code = refusal_code(stale) + fresh_expected = ready_state() + fresh_expected["field_value"] = fresh_value + independent = stale_state == ready_state() and final_state == fresh_expected + success = code in ("target_gone", "target_missing") and fresh.get("verification") == "verified" and independent + response_summaries.extend( + [ + {"step": "stale", "result_code": code, "verification": stale.get("verification"), "effect": stale.get("effect")}, + {"step": "fresh", "route": fresh.get("route"), "verification": fresh.get("verification"), "effect": fresh.get("effect")}, + ] + ) + + elif case == "A08": + request_key = f"{case}-{label}-{uuid.uuid4().hex}" + args = {"process_id": main_pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"} + first, latency = runtime.operate( + "windows.uia.press", request_key, args, {"attribute": "name", "equals": "Submitted"} + ) + call_latencies.append(latency) + replay, latency = runtime.operate( + "windows.uia.press", request_key, args, {"attribute": "name", "equals": "Submitted"} + ) + call_latencies.append(latency) + fixture = state(fixture_state) + replay_identified = replay.get("recovery") == "idempotent_replay" + independent = fixture == submitted_state() + first_verified = first.get("verification") == "verified" and first.get("effect") == "changed" + success = first_verified and replay_identified and independent + response_summaries.extend( + [ + {"step": "first", "verification": first.get("verification"), "effect": first.get("effect"), "verified_success": first_verified}, + {"step": "replay", "verification": replay.get("verification"), "effect": replay.get("effect"), "recovery": replay.get("recovery")}, + ] + ) + + elif case == "A09": + result, latency = runtime.operate( + "windows.uia.press", + f"{case}-{label}-{uuid.uuid4().hex}", + {"process_id": main_pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"}, + {"attribute": "name", "equals": "Deliberately-wrong-postcondition"}, + ) + call_latencies.append(latency) + fixture = state(fixture_state) + independent = fixture == submitted_state() + success = result.get("verification") != "verified" and independent + response_summaries.append({"verification": result.get("verification"), "effect": result.get("effect"), "data": result.get("data")}) + + elif case == "A10": + request_key = f"{case}-{label}-{uuid.uuid4().hex}" + args = {"process_id": main_pid, "name": "Submit", "automation_id": "SubmitButton", "role": "button"} + first, latency = runtime.operate( + "windows.uia.press", request_key, args, {"attribute": "name", "equals": "Submitted"} + ) + call_latencies.append(latency) + recovery_started = time.perf_counter() + runtime.close() + runtime = None + runtime = McpClient(binary, state_dir) + replay, latency = runtime.operate( + "windows.uia.press", request_key, args, {"attribute": "name", "equals": "Submitted"} + ) + call_latencies.append(latency) + recovery_ms = (time.perf_counter() - recovery_started) * 1000 + fixture = state(fixture_state) + replay_identified = replay.get("recovery") == "idempotent_replay" + independent = fixture == submitted_state() + success = replay_identified and independent + response_summaries.extend( + [ + {"step": "before_restart", "verification": first.get("verification"), "effect": first.get("effect")}, + {"step": "after_restart", "verification": replay.get("verification"), "effect": replay.get("effect"), "recovery": replay.get("recovery")}, + ] + ) + else: + raise ValueError(f"unsupported case {case}") + + return { + "task_id": case, + "iteration": label, + "started_utc": start_utc, + "ended_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "server_version": runtime.server_info.get("version") if runtime else "0.1.67", + "route": "windows_uia_direct", + "result_code": response_summaries[-1].get("result_code", "ok"), + "verification": response_summaries[-1].get("verification"), + "independent_verifier": independent, + "safe_refusal": case in ("A04", "A05", "A06", "A07") and success, + "replay_identified": case in ("A08", "A10") and success, + "latencies_ms": [round(value, 3) for value in call_latencies], + "latency_ms": round(max(call_latencies), 3) if call_latencies else None, + "recovery_ms": round(recovery_ms, 3) if recovery_ms is not None else None, + "tool_call_count": len(call_latencies), + "model_turns": 0, + "retries": 0, + "recovery_actions": 1 if case == "A07" else (1 if case == "A10" else 0), + "response_summary": response_summaries, + "fixture_state_artifact": str(fixture_state), + "success": bool(success), + } + finally: + if runtime: + runtime.close() + stop_process(main) + stop_process(decoy) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--case", required=True, choices=["A04", "A05", "A06", "A07", "A08", "A09", "A10"]) + parser.add_argument("--iterations", type=int, default=20) + parser.add_argument("--output", type=pathlib.Path, required=True) + args = parser.parse_args() + if platform.system() != "Windows": + raise SystemExit("These cases require an interactive Windows desktop") + root = pathlib.Path(__file__).resolve().parents[1] + powershell = shutil.which("powershell.exe") or shutil.which("powershell") + if not powershell: + raise SystemExit("Windows PowerShell is required") + binary = os.environ.get("COMPTROL_BIN") or str( + pathlib.Path(os.environ["APPDATA"]) / "npm" / "node_modules" / "comptrolling" / "native" / "win32-x64" / "comptrol.exe" + ) + if not pathlib.Path(binary).is_file(): + raise SystemExit(f"Comptrol runtime not found: {binary}") + args.output.mkdir(parents=True, exist_ok=False) + results = [] + with (args.output / "tasks.jsonl").open("w", encoding="utf-8", newline="\n") as tasks: + for label in ["warmup"] + [f"measured-{i:02d}" for i in range(1, args.iterations + 1)]: + try: + row = trial(root, args.output, args.case, label, binary, powershell) + except Exception as error: + row = {"task_id": args.case, "iteration": label, "result_code": type(error).__name__, "error": str(error), "independent_verifier": False, "success": False} + tasks.write(json.dumps(row, ensure_ascii=False) + "\n") + tasks.flush() + if label != "warmup": + results.append(row) + print(json.dumps({"iteration": label, "success": row.get("success"), "verification": row.get("verification"), "latency_ms": row.get("latency_ms"), "error": row.get("error")}), flush=True) + + latencies = [latency for row in results for latency in row.get("latencies_ms", [])] + ordered = sorted(latencies) + p95 = ordered[max(0, math.ceil(0.95 * len(ordered)) - 1)] if ordered else None + summary = { + "task_id": args.case, + "scope": "direct native Comptrol MCP runtime; not Codex end-to-end", + "package_version": "0.1.67", + "warmup_count": 1, + "measured_trials": len(results), + "successful_trials": sum(bool(row.get("success")) for row in results), + "safe_refusals": sum(bool(row.get("safe_refusal")) for row in results), + "identified_replays": sum(bool(row.get("replay_identified")) for row in results), + "independent_fixture_successes": sum(bool(row.get("independent_verifier")) for row in results), + "false_verified_successes": sum( + any(step.get("verification") == "verified" and not row.get("independent_verifier") for step in row.get("response_summary", [])) + for row in results + ), + "runtime_calls": len(latencies), + "median_runtime_latency_ms": statistics.median(latencies) if latencies else None, + "p95_runtime_latency_ms_nearest_rank": p95, + "min_runtime_latency_ms": min(latencies) if latencies else None, + "max_runtime_latency_ms": max(latencies) if latencies else None, + "recovery_ms_by_trial": [row["recovery_ms"] for row in results if row.get("recovery_ms") is not None], + "all_measured_trials": results, + } + (args.output / "summary.json").write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + (args.output / "summary.md").write_text( + f"# {args.case} Windows UI Automation result\n\n" + f"- Package: `comptrolling@{summary['package_version']}`\n" + f"- Scope: {summary['scope']}\n" + f"- Successful trials: {summary['successful_trials']}/{summary['measured_trials']}\n" + f"- Safe refusals: {summary['safe_refusals']}/{summary['measured_trials']}\n" + f"- Identified replays: {summary['identified_replays']}/{summary['measured_trials']}\n" + f"- Independent fixture success: {summary['independent_fixture_successes']}/{summary['measured_trials']}\n" + f"- False verified success: {summary['false_verified_successes']}\n" + f"- Runtime latency, n={len(latencies)}: median {summary['median_runtime_latency_ms']}, p95 {p95}, min {summary['min_runtime_latency_ms']}, max {summary['max_runtime_latency_ms']} ms\n", + encoding="utf-8", + ) + return 0 if len(results) == args.iterations and summary["successful_trials"] == args.iterations else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From b2d36d97dbd42f50eaa44733f52e51b7932aed01 Mon Sep 17 00:00:00 2001 From: Praket7 <149203433+Praket7@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:23:25 -0500 Subject: [PATCH 2/2] fix: negotiate standard MCP handshake versions --- crates/comptrol-core/src/mcp/mod.rs | 24 +++++++++++++++++++++--- scripts/npm_mcp_conformance.py | 3 ++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/comptrol-core/src/mcp/mod.rs b/crates/comptrol-core/src/mcp/mod.rs index 44bd01f0..7c39f1eb 100644 --- a/crates/comptrol-core/src/mcp/mod.rs +++ b/crates/comptrol-core/src/mcp/mod.rs @@ -1,6 +1,12 @@ #![deny(unsafe_code)] pub const LEGACY_VERSION: &str = "2025-03-26"; +pub const HANDSHAKE_VERSIONS: &[&str] = &[ + "2025-11-25", + "2025-06-18", + LEGACY_VERSION, + "2024-11-05", +]; pub const CURRENT_VERSION: &str = "2026-07-28"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -11,17 +17,23 @@ pub enum ProtocolMode { pub fn negotiate(requested: Option<&str>) -> Result<(&'static str, ProtocolMode), String> { match requested { - None | Some(LEGACY_VERSION) => Ok((LEGACY_VERSION, ProtocolMode::Legacy)), Some(CURRENT_VERSION) => Ok((CURRENT_VERSION, ProtocolMode::Current)), - Some(other) => Err(format!("unsupported MCP protocol version {other}")), + Some(version) => HANDSHAKE_VERSIONS + .iter() + .copied() + .find(|supported| *supported == version) + .map(|supported| (supported, ProtocolMode::Legacy)) + .ok_or_else(|| format!("unsupported MCP protocol version {version}")), + None => Ok((LEGACY_VERSION, ProtocolMode::Legacy)), } } pub fn from_header(value: Option<&str>) -> Result { match value { - None | Some(LEGACY_VERSION) => Ok(ProtocolMode::Legacy), Some(CURRENT_VERSION) => Ok(ProtocolMode::Current), + Some(version) if HANDSHAKE_VERSIONS.contains(&version) => Ok(ProtocolMode::Legacy), Some(other) => Err(format!("unsupported MCP protocol version {other}")), + None => Ok(ProtocolMode::Legacy), } } @@ -32,10 +44,16 @@ mod tests { #[test] fn supports_legacy_and_current_modes_without_accepting_unknown_versions() { assert_eq!(negotiate(None).unwrap().1, ProtocolMode::Legacy); + for version in HANDSHAKE_VERSIONS { + assert_eq!(negotiate(Some(version)).unwrap(), (*version, ProtocolMode::Legacy)); + assert_eq!(from_header(Some(version)).unwrap(), ProtocolMode::Legacy); + } assert_eq!( negotiate(Some(CURRENT_VERSION)).unwrap().1, ProtocolMode::Current ); + assert_eq!(from_header(Some(CURRENT_VERSION)).unwrap(), ProtocolMode::Current); assert!(negotiate(Some("future")).is_err()); + assert!(from_header(Some("future")).is_err()); } } diff --git a/scripts/npm_mcp_conformance.py b/scripts/npm_mcp_conformance.py index 8c1eac93..e9ac67d8 100644 --- a/scripts/npm_mcp_conformance.py +++ b/scripts/npm_mcp_conformance.py @@ -63,7 +63,7 @@ def main() -> None: "id": 1, "method": "initialize", "params": { - "protocolVersion": "2025-03-26", + "protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "comptrol-npm-conformance", "version": "1.0"}, }, @@ -88,6 +88,7 @@ def main() -> None: names = {tool["name"] for tool in tools["result"]["tools"]} required = {"operate", "inspect", "watch", "reconcile", "capabilities"} assert initialized["result"]["serverInfo"]["name"] == "comptrol", initialized + assert initialized["result"]["protocolVersion"] == "2025-06-18", initialized assert required <= names, sorted(names) assert ping["result"]["structuredContent"]["verification"] == "verified", ping print(