From e8e10aa06363463f6910735de425df59a096c6d9 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 23 Oct 2026 22:22:49 +0200 Subject: [PATCH 1/6] fix(ax-tool): a recording leaves out the windows --exclude-window-ids names (ScreenCaptureKit ignores sharingType none since macOS 15), and a frame the PNG cap keeps out still counts as motion, so a long movie no longer drops to the idle rate --- native/ax-tool/Sources/IsolatedCapture.swift | 29 +++++++++++++------- native/ax-tool/Sources/Record.swift | 3 +- native/ax-tool/Sources/main.swift | 1 + native/ax-tool/Tests/FrameSamplerTests.swift | 18 ++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/native/ax-tool/Sources/IsolatedCapture.swift b/native/ax-tool/Sources/IsolatedCapture.swift index 767c4a1175..ffc3d05634 100644 --- a/native/ax-tool/Sources/IsolatedCapture.swift +++ b/native/ax-tool/Sources/IsolatedCapture.swift @@ -17,6 +17,10 @@ struct CaptureSelectionOptions { let transparent: Bool let codec: String let indicator: Bool + /// Windows the movie must never show, such as the host's own recording controls: a display capture leaves them + /// out of its content filter and an isolated capture never selects them. `NSWindow.sharingType = .none` does not + /// do it, because ScreenCaptureKit ignores it on macOS 15 and later. + let excludedWindowIDs: Set func dimensions(points: CGSize, nativeScale: CGFloat, video: Bool) -> CGSize { let requested = outputSize ?? CGSize(width: (points.width * (outputScale ?? nativeScale)).rounded(), @@ -34,15 +38,19 @@ struct CaptureSelectionOptions { } } +/// A comma-separated list of positive CG window ids, or nil when the flag is absent. +func windowIDList(_ flag: String) -> [CGWindowID]? { + guard let raw = argValue(flag) else { return nil } + let parts = raw.split(separator: ",", omittingEmptySubsequences: false) + let ids = parts.compactMap { UInt32($0) } + guard ids.count == parts.count, ids.allSatisfy({ $0 > 0 }) else { + errorExit("\(flag) requires positive CG window IDs separated by commas") + } + return ids +} + func captureSelectionOptions(mode: String) -> CaptureSelectionOptions { - var ids: [CGWindowID] = [] - if let raw = argValue("--window-ids") { - let parts = raw.split(separator: ",", omittingEmptySubsequences: false) - ids = parts.compactMap { UInt32($0) } - guard ids.count == parts.count, ids.allSatisfy({ $0 > 0 }) else { - errorExit("--window-ids requires positive CG window IDs separated by commas") - } - } + let ids = windowIDList("--window-ids") ?? [] let args = CommandLine.arguments var apps: [String] = [] for (index, value) in args.enumerated() where value == "--include-app" { @@ -94,7 +102,8 @@ func captureSelectionOptions(mode: String) -> CaptureSelectionOptions { } return CaptureSelectionOptions(windowIDs: Array(Set(ids)).sorted(), apps: apps, canvas: canvas, outputSize: size, outputScale: scale, transparent: transparent, - codec: codec, indicator: !args.contains("--no-indicator")) + codec: codec, indicator: !args.contains("--no-indicator"), + excludedWindowIDs: Set(windowIDList("--exclude-window-ids") ?? [])) } /// Same nonactivating panel contract as CursorOverlay. Core Animation supplies the pulse; @@ -478,7 +487,7 @@ func captureIsolatedWindows(_ options: RecordOptions) { let rows = captureWindowRows() let selected = rows.compactMap { row -> SelectedWindow? in guard let id = row[kCGWindowNumber] as? CGWindowID, let pid = row[kCGWindowOwnerPID] as? pid_t, - explicit.contains(id) || pids.contains(pid), + explicit.contains(id) || pids.contains(pid), !options.selection.excludedWindowIDs.contains(id), let bounds = captureWindowBounds(row), bounds.width > 0, bounds.height > 0 else { return nil } let shareable = content.windows.first { $0.windowID == id } guard let shareable else { return nil } diff --git a/native/ax-tool/Sources/Record.swift b/native/ax-tool/Sources/Record.swift index 36f069ddd8..c954003540 100644 --- a/native/ax-tool/Sources/Record.swift +++ b/native/ax-tool/Sources/Record.swift @@ -557,7 +557,8 @@ func cmdCaptureScreen() { guard let scDisplay = content.displays.first(where: { $0.displayID == display.id }) else { errorExit("display \(display.id) is not shareable") } - filter = SCContentFilter(display: scDisplay, excludingWindows: content.windows.filter { indicator.windowIDs.contains($0.windowID) }) + let excluded = indicator.windowIDs.union(options.selection.excludedWindowIDs) + filter = SCContentFilter(display: scDisplay, excludingWindows: content.windows.filter { excluded.contains($0.windowID) }) pointSize = sourceRect?.size ?? display.bounds.size } let configuration = SCStreamConfiguration() diff --git a/native/ax-tool/Sources/main.swift b/native/ax-tool/Sources/main.swift index e94751b892..07fba1646e 100644 --- a/native/ax-tool/Sources/main.swift +++ b/native/ax-tool/Sources/main.swift @@ -78,6 +78,7 @@ if args.count < 2 || args[1] == "--help" || args[1] == "-h" { isolated: --window-ids ID,ID and/or repeated --include-app NAME|PID; --canvas crop|display [--output-size WxH | --output-scale N] [--transparent --codec prores4444 --video-out f.mov] [--no-indicator] default pulsing border is excluded from capture + [--exclude-window-ids ID,ID] windows the movie never shows (a host's own controls) [--screen-index N] [--region x,y,w,h] --duration [--active-fps 8] [--idle-fps 2] [--threshold 2.5] [--video-out f.mp4] [--out DIR] ScreenCaptureKit recording: change-sampled keep-NNNN.png frames, contact.png, metadata.json; JSON result in the capture-runner shape diff --git a/native/ax-tool/Tests/FrameSamplerTests.swift b/native/ax-tool/Tests/FrameSamplerTests.swift index e202d1838c..f6ab30888e 100644 --- a/native/ax-tool/Tests/FrameSamplerTests.swift +++ b/native/ax-tool/Tests/FrameSamplerTests.swift @@ -109,6 +109,24 @@ final class FrameSamplerTests: XCTestCase { XCTAssertEqual(policy.keptCount, 1) } + func testPastTheCapMotionStillKeepsTheActiveRate() { + var policy = KeepPolicy(thresholdPercent: 0, maxFrames: 1) + var rate = FrameRatePolicy(activeFps: 15, idleFps: 2, stillFramesBeforeIdle: 2) + let shades: [UInt8] = [0, 60, 120, 180, 240, 30] + for shade in shades { + let decision = policy.consider(signature(frame(width: 16, height: 16) { _, _ in (shade, shade, shade) }, width: 16, height: 16)) + XCTAssertTrue(decision.moved, "shade \(shade) changed the frame") + _ = rate.observe(kept: decision.moved) + } + XCTAssertEqual(policy.keptCount, 1, "only the first frame is written") + XCTAssertEqual(rate.currentFps, 15, "a moving recording past the cap stays at the active rate") + let still = signature(frame(width: 16, height: 16) { _, _ in (30, 30, 30) }, width: 16, height: 16) + XCTAssertFalse(policy.consider(still).moved) + _ = rate.observe(kept: false) + _ = rate.observe(kept: policy.consider(still).moved) + XCTAssertEqual(rate.currentFps, 2, "a still screen past the cap still drops to the idle rate") + } + func testTheRatePolicyDropsToIdleOnlyAfterARunOfStillFrames() { var rate = FrameRatePolicy(activeFps: 8, idleFps: 2, stillFramesBeforeIdle: 3) XCTAssertEqual(rate.currentFps, 8) From d227e96dd2ee51197a6f908a5a7ec3efa82ab8d5 Mon Sep 17 00:00:00 2001 From: Martin Date: Sat, 24 Oct 2026 00:00:28 +0200 Subject: [PATCH 2/6] feat(control): under an agent, a call that would bring an app to the front or move the pointer is refused while the user worked in the last 5 s, a background click into a window that is not key says it may have done nothing, and every action takes the window_id get_app_state took --- .../SnapshotSupport/ForegroundGuard.swift | 72 +++++++++++++++++ .../SnapshotSupport/SnapshotToken.swift | 2 + .../SnapshotSupport/UserTakeover.swift | 12 ++- .../SnapshotSupport/WorkflowArguments.swift | 2 +- native/ax-tool/Sources/AppLifecycle.swift | 10 +++ native/ax-tool/Sources/SnapshotWorkflow.swift | 15 +++- native/ax-tool/Sources/commands/Input.swift | 2 +- native/ax-tool/Sources/utils/InputGate.swift | 77 ++++++++++++++++++- native/ax-tool/Sources/utils/Output.swift | 13 +++- .../ax-tool/Tests/ForegroundGuardTests.swift | 56 ++++++++++++++ src/control/commands/workflow.ts | 9 +++ src/control/lib/computer-use/schemas.ts | 18 +++++ src/control/lib/computer-use/session.test.ts | 50 ++++++++++++ src/control/lib/computer-use/session.ts | 23 ++++++ src/control/lib/decision/native.test.ts | 6 ++ src/control/lib/decision/recovery.ts | 12 +++ src/control/lib/runner.test.ts | 10 +++ src/control/lib/runner.ts | 16 +++- 18 files changed, 398 insertions(+), 7 deletions(-) create mode 100644 native/ax-tool/SnapshotSupport/ForegroundGuard.swift create mode 100644 native/ax-tool/Tests/ForegroundGuardTests.swift diff --git a/native/ax-tool/SnapshotSupport/ForegroundGuard.swift b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift new file mode 100644 index 0000000000..efac8e93d8 --- /dev/null +++ b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift @@ -0,0 +1,72 @@ +import Foundation + +/// The quiet time, in seconds, an agent waits for before it may bring an app to the front or move the +/// pointer. Less than this since the user's last key, click, move or scroll means the user is working, +/// and an agent that took focus then typed into their window (2026-10-23: "you're taking my focus quite +/// a lot during this testing"). +public let defaultForegroundIdleSeconds: TimeInterval = 5 + +/// `GENESIS_CONTROL_FOREGROUND_GUARD=1` turns the check on. The TypeScript runner sets it when an agent +/// drives the tool; a person typing a command just pressed Return, and must not be refused for it. +public func foregroundGuardEnabled(_ environment: [String: String]) -> Bool { + environment["GENESIS_CONTROL_FOREGROUND_GUARD"]?.trimmingCharacters(in: .whitespaces) == "1" +} + +/// `GENESIS_CONTROL_FOREGROUND_IDLE=` sets the quiet time; 0 turns the check off. A value that +/// is not a number keeps the default: a guard that a typo disables is not a guard. +public func foregroundIdleThreshold(_ environment: [String: String]) -> TimeInterval { + guard let raw = environment["GENESIS_CONTROL_FOREGROUND_IDLE"]?.trimmingCharacters(in: .whitespaces), + let value = Double(raw), value.isFinite, value >= 0 else { + return defaultForegroundIdleSeconds + } + + return value +} + +/// `GENESIS_CONTROL_ALLOW_FOREGROUND=1` lets a whole run take focus: a demonstration the user asked for. +public func foregroundAllowedByEnvironment(_ environment: [String: String]) -> Bool { + guard let raw = environment["GENESIS_CONTROL_ALLOW_FOREGROUND"]?.trimmingCharacters(in: .whitespaces).lowercased() else { + return false + } + + return ["1", "true", "yes", "on"].contains(raw) +} + +/// True when the newest key, click, move or scroll is the user's and came within `threshold` seconds. +/// The window server counts this tool's own synthetic events as input too, so an event no newer than its +/// last post (`ownInputAgo`, with 0.3 s of slack for the throttled record) is the agent's, not the user's. +public func userIsActive(idleSeconds: TimeInterval, ownInputAgo: TimeInterval?, threshold: TimeInterval) -> Bool { + guard threshold > 0, idleSeconds.isFinite, idleSeconds >= 0, idleSeconds < threshold else { + return false + } + + if let own = ownInputAgo, idleSeconds >= own - 0.3 { + return false + } + + return true +} + +/// The user is at the keyboard or mouse, so this call did not take focus. Never retried automatically. +public struct UserActiveError: Error, LocalizedError { + public let idleSeconds: TimeInterval + public let threshold: TimeInterval + + public init(idleSeconds: TimeInterval, threshold: TimeInterval) { + self.idleSeconds = idleSeconds + self.threshold = threshold + } + + public var category: SnapshotRefusal { .userActive } + + public var errorDescription: String? { + "the user is working on this Mac (their last key, click or pointer move was " + + String(format: "%.1f", idleSeconds) + " s ago, under the " + String(format: "%g", threshold) + + " s quiet time), so no app was brought to the front and the pointer did not move. Act in the " + + "background instead: click without prepare (AXPress or a window-addressed click), press_key with " + + "activate:false, set_value or fill_form without prepare, launch_app with activate:false, open -g " + + "for a URL. Only for a visible demonstration the user asked for: allow_foreground:true " + + "(--allow-foreground, or GENESIS_CONTROL_ALLOW_FOREGROUND=1 for a whole run). " + + "GENESIS_CONTROL_FOREGROUND_IDLE=0 turns this check off" + } +} diff --git a/native/ax-tool/SnapshotSupport/SnapshotToken.swift b/native/ax-tool/SnapshotSupport/SnapshotToken.swift index 96765887c6..ff39c3bfcf 100644 --- a/native/ax-tool/SnapshotSupport/SnapshotToken.swift +++ b/native/ax-tool/SnapshotSupport/SnapshotToken.swift @@ -95,6 +95,8 @@ public enum SnapshotRefusal: String, Codable { case refused /// The user put the pointer in the takeover corner. Terminal: never retried, never recovered. case userTakeover = "user_takeover" + /// The user is at the keyboard or mouse, so an agent's call refused to take focus or the pointer. + case userActive = "user_active" } public enum SnapshotError: Error, LocalizedError { diff --git a/native/ax-tool/SnapshotSupport/UserTakeover.swift b/native/ax-tool/SnapshotSupport/UserTakeover.swift index 686f084994..bf927283d7 100644 --- a/native/ax-tool/SnapshotSupport/UserTakeover.swift +++ b/native/ax-tool/SnapshotSupport/UserTakeover.swift @@ -60,15 +60,20 @@ public final class SyntheticInputGate { private let tookOver: () -> Bool private let sleeper: (TimeInterval) -> Void + /// Runs after every event this gate sends, so the foreground guard can tell the agent's input + /// from the user's. + private let onPost: () -> Void private var held: [(id: Int, release: () -> Void)] = [] private var nextID = 0 /// Events posted through this gate, releases included. public private(set) var posted = 0 public init(tookOver: @escaping () -> Bool, - sleeper: @escaping (TimeInterval) -> Void = { Thread.sleep(forTimeInterval: $0) }) { + sleeper: @escaping (TimeInterval) -> Void = { Thread.sleep(forTimeInterval: $0) }, + onPost: @escaping () -> Void = {}) { self.tookOver = tookOver self.sleeper = sleeper + self.onPost = onPost } /// Stop here when the user took over, after releasing anything still held (latest first). @@ -80,6 +85,7 @@ public final class SyntheticInputGate { while let entry = held.popLast() { entry.release() posted += 1 + onPost() } throw UserTakeoverError(dispatched: posted > 0) } @@ -89,6 +95,7 @@ public final class SyntheticInputGate { try check() send() posted += 1 + onPost() } /// A down event. Its release runs through `release(_:)`, or through `check()` if the user takes @@ -98,6 +105,7 @@ public final class SyntheticInputGate { try check() send() posted += 1 + onPost() let id = nextID nextID += 1 held.append((id, release)) @@ -113,6 +121,7 @@ public final class SyntheticInputGate { let entry = held.remove(at: index) entry.release() posted += 1 + onPost() } /// An event whose check already ran in the caller's own verify step, such as a drag's moves and @@ -120,6 +129,7 @@ public final class SyntheticInputGate { public func deliver(_ send: () -> Void) { send() posted += 1 + onPost() } /// Sleep in slices of at most 0.1 s and check between them, so a long hold or dwell still stops diff --git a/native/ax-tool/SnapshotSupport/WorkflowArguments.swift b/native/ax-tool/SnapshotSupport/WorkflowArguments.swift index ea01ea6f63..12982b01d0 100644 --- a/native/ax-tool/SnapshotSupport/WorkflowArguments.swift +++ b/native/ax-tool/SnapshotSupport/WorkflowArguments.swift @@ -37,7 +37,7 @@ public struct WorkflowArguments { "--revalidate-scope", "--frame", "--by-identifier", "--window-index", "--depth", "--modifiers", "--budget-ms", "--target-ordinal", "--expect-value-sha256", ] - flagOptions = ["--background", "--allow-outside", "--double", "--refresh", "--no-cursor", "--no-image", "--prepare", "--replace", "--hold", "--no-activate"] + flagOptions = ["--background", "--allow-outside", "--double", "--refresh", "--no-cursor", "--no-image", "--prepare", "--replace", "--hold", "--no-activate", "--allow-foreground"] default: throw WorkflowArgumentError.invalid("unknown workflow command \(command)") } diff --git a/native/ax-tool/Sources/AppLifecycle.swift b/native/ax-tool/Sources/AppLifecycle.swift index 3bea837d46..94c32e6750 100644 --- a/native/ax-tool/Sources/AppLifecycle.swift +++ b/native/ax-tool/Sources/AppLifecycle.swift @@ -1,5 +1,6 @@ import AppKit import Foundation +import SnapshotSupport func cmdInstalledApps() { let roots = ["/Applications", "/System/Applications", "/System/Library/CoreServices", @@ -60,6 +61,15 @@ func cmdLaunchApp() { "processLaunch":launch,"frontmost":app.isActive,"alreadyRunning":existing, "dispatchState":"dispatched"]) } + if !args.contains("--background") && !(matches.first?.isActive ?? false) { + do { + try checkForegroundAllowed() + } catch let refusal as UserActiveError { + userActiveExit(refusal) + } catch { + fputs("foreground check failed: \(error.localizedDescription)\n", stderr) + } + } if let existing = matches.first { if !args.contains("--background") && !existing.activate(options: [.activateIgnoringOtherApps]) { jsonOutput(["ok":false,"error":"existing app activation was not accepted; inspect before retrying", diff --git a/native/ax-tool/Sources/SnapshotWorkflow.swift b/native/ax-tool/Sources/SnapshotWorkflow.swift index a9ec87a898..befc82d5a5 100644 --- a/native/ax-tool/Sources/SnapshotWorkflow.swift +++ b/native/ax-tool/Sources/SnapshotWorkflow.swift @@ -72,6 +72,10 @@ private func workflowFailure(_ message: String, category: SnapshotRefusal = .ref } private func workflowFailure(_ error: Error, extras: [String: Any] = [:]) -> Never { + if let active = error as? UserActiveError { + workflowDispatchState = inputGate.posted > 0 ? "uncertain" : "not_started" + workflowFailure(active.localizedDescription, category: active.category, extras: extras) + } if let takeover = error as? UserTakeoverError { workflowDispatchState = takeover.dispatched ? "uncertain" : "not_started" workflowFailure(takeover.localizedDescription, category: takeover.category, extras: extras) @@ -1657,7 +1661,8 @@ func cmdAct(appName _: String) { tree.rows.compactMap { $0["AXDescription"] as? String }) ActionCursor.emit("move", point: point, background: false, target: rawCoords == nil ? "ax" : "pixel") // Checked before the warp: once it runs, the pointer is ours and no longer says - // where the user put it. + // where the user put it. A hover takes the user's pointer, so it waits for them too. + try checkForegroundAllowed() try inputGate.post { _ = CGWarpMouseCursorPosition(point) } if let moved = CGEvent(mouseEventSource: nil, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) { @@ -2077,6 +2082,14 @@ func cmdAct(appName _: String) { // why, rather than wondering whether it was silently skipped. payload["nonActivatingPanel"] = true } + // A window-addressed left click into an app that is not in front lands in a window that is not key. + // A SwiftUI control there takes it as the activation click and does nothing, while the click itself + // was delivered (a sheet's "Cancel" stayed open, 2026-10-23). Say so, so the caller checks the effect. + if action == "click", workflowFlag("--background"), !secondaryClick, !nonActivatingPanel, startingFrontmost != pid { + payload["inactiveWindowClick"] = true + payload["effectNote"] = "delivered to a window that is not key: a SwiftUI control may take it as the activation " + + "click and do nothing. Check the postcondition; AXPress (a click without physical) acts without focus" + } payload.merge(actionExtras) { _, new in new } if workflowFlag("--refresh") { diff --git a/native/ax-tool/Sources/commands/Input.swift b/native/ax-tool/Sources/commands/Input.swift index 543343a4e0..fda66d80ed 100644 --- a/native/ax-tool/Sources/commands/Input.swift +++ b/native/ax-tool/Sources/commands/Input.swift @@ -209,7 +209,7 @@ func secondaryClickFrontGuard(before: pid_t?, target: pid_t) -> [String: Any] { return ["frontmostRestored": false, "menuHoldsKeyboard": frontmostPid() == target] } - let restored = bringFrontmost(before) + let restored = bringFrontmost(before, restoring: true) let name = NSRunningApplication(processIdentifier: before)?.localizedName ?? "pid \(before)" return ["frontmostRestored": restored, "focusNote": restored diff --git a/native/ax-tool/Sources/utils/InputGate.swift b/native/ax-tool/Sources/utils/InputGate.swift index f0d2eeb3ad..1523d80ddf 100644 --- a/native/ax-tool/Sources/utils/InputGate.swift +++ b/native/ax-tool/Sources/utils/InputGate.swift @@ -31,7 +31,82 @@ private func pointerTakeover() -> Bool { } /// The gate every synthetic event of this process passes through. -let inputGate = SyntheticInputGate(tookOver: pointerTakeover) +let inputGate = SyntheticInputGate(tookOver: pointerTakeover, onPost: recordOwnInput) + +private let foregroundEnvironment = ProcessInfo.processInfo.environment +/// Read once per process: the agent's run cannot change these under a running command. +private let foregroundCheckEnabled = foregroundGuardEnabled(foregroundEnvironment) +private let foregroundThreshold = foregroundIdleThreshold(foregroundEnvironment) +private let foregroundAllowed = CommandLine.arguments.contains("--allow-foreground") + || foregroundAllowedByEnvironment(foregroundEnvironment) + +/// Seconds since the last key, click, pointer move or scroll the window server saw. +private func inputIdleSeconds() -> TimeInterval { + let types: [CGEventType] = [.keyDown, .flagsChanged, .mouseMoved, .leftMouseDown, .rightMouseDown, + .otherMouseDown, .leftMouseDragged, .scrollWheel] + return types.map { CGEventSource.secondsSinceLastEventType(.hidSystemState, eventType: $0) }.min() ?? .infinity +} + +/// When any process of this tool last posted a synthetic event. The window server counts those as +/// input, and an agent's own previous click must not read as the user working. +private let ownInputFile: URL = { + let home = foregroundEnvironment["GENESIS_TOOLS_HOME"].map { URL(fileURLWithPath: $0) } + ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".genesis-tools") + return home.appendingPathComponent("control").appendingPathComponent("last-synthetic-input") +}() +private var ownInputWrittenAt: TimeInterval = 0 + +/// At most one write per quarter second; the guard's 0.3 s slack covers the posts in between. +private func recordOwnInput() { + guard foregroundCheckEnabled else { + return + } + + let now = Date().timeIntervalSince1970 + guard now - ownInputWrittenAt >= 0.25 else { + return + } + + ownInputWrittenAt = now + do { + try FileManager.default.createDirectory(at: ownInputFile.deletingLastPathComponent(), + withIntermediateDirectories: true) + try String(now).write(to: ownInputFile, atomically: true, encoding: .utf8) + } catch { + fputs("could not record the time of this tool's own input: \(error.localizedDescription)\n", stderr) + } +} + +private func ownInputAgo() -> TimeInterval? { + guard let text = try? String(contentsOf: ownInputFile, encoding: .utf8), + let at = Double(text.trimmingCharacters(in: .whitespacesAndNewlines)) else { + return nil + } + + return Date().timeIntervalSince1970 - at +} + +/// Refuses to bring an app to the front or move the pointer while the user is at the keyboard or mouse, +/// when an agent drives this tool. Call it before the first change, so a refusal sent nothing. +func checkForegroundAllowed() throws { + guard foregroundCheckEnabled, !foregroundAllowed else { + return + } + + let idle = inputIdleSeconds() + guard userIsActive(idleSeconds: idle, ownInputAgo: ownInputAgo(), threshold: foregroundThreshold) else { + return + } + + throw UserActiveError(idleSeconds: idle, threshold: foregroundThreshold) +} + +/// The refusal for commands that have no error channel of their own: print it and stop. +func userActiveExit(_ error: UserActiveError) -> Never { + jsonOutput(["ok": false, "error": error.localizedDescription, "refusal": error.category.rawValue, + "dispatchState": inputGate.posted > 0 ? "uncertain" : "not_started"]) + exit(1) +} /// The refusal for commands that have no error channel of their own: print it and stop. func takeoverExit(_ takeover: UserTakeoverError) -> Never { diff --git a/native/ax-tool/Sources/utils/Output.swift b/native/ax-tool/Sources/utils/Output.swift index 0ededf2ae2..c0a03ee0a2 100644 --- a/native/ax-tool/Sources/utils/Output.swift +++ b/native/ax-tool/Sources/utils/Output.swift @@ -40,7 +40,18 @@ func frontmostPid() -> pid_t? { /// call + fixed sleep races when another activation is still in flight — /// re-request each poll (up to ~3s), pumping the runloop so NSWorkspace /// state can update too. -func bringFrontmost(_ pid: pid_t) -> Bool { +/// +/// `restoring`: putting the user's own app back in front, which the foreground guard never refuses. +func bringFrontmost(_ pid: pid_t, restoring: Bool = false) -> Bool { + if !restoring && frontmostPid() != pid { + do { + try checkForegroundAllowed() + } catch let refusal as UserActiveError { + userActiveExit(refusal) + } catch { + fputs("foreground check failed: \(error.localizedDescription)\n", stderr) + } + } let runningApp = NSWorkspace.shared.runningApplications.first { $0.processIdentifier == pid } return activateFrontmost( isFrontmost: { frontmostPid() == pid }, diff --git a/native/ax-tool/Tests/ForegroundGuardTests.swift b/native/ax-tool/Tests/ForegroundGuardTests.swift new file mode 100644 index 0000000000..505b220dfa --- /dev/null +++ b/native/ax-tool/Tests/ForegroundGuardTests.swift @@ -0,0 +1,56 @@ +import XCTest +@testable import SnapshotSupport + +/// An agent must not take focus or the pointer while the user works (2026-10-23: a test run took the +/// user's focus again and again while they typed in another app). +final class ForegroundGuardTests: XCTestCase { + func testRecentUserInputMeansTheUserIsActive() { + XCTAssertTrue(userIsActive(idleSeconds: 0.4, ownInputAgo: nil, threshold: 5)) + XCTAssertTrue(userIsActive(idleSeconds: 4.9, ownInputAgo: 30, threshold: 5)) + } + + func testAQuietUserOrATurnedOffCheckIsNotActive() { + XCTAssertFalse(userIsActive(idleSeconds: 5, ownInputAgo: nil, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 60, ownInputAgo: nil, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 0.1, ownInputAgo: nil, threshold: 0)) + XCTAssertFalse(userIsActive(idleSeconds: .infinity, ownInputAgo: nil, threshold: 5)) + } + + /// The window server counts the agent's own synthetic click as input; it must not read as the user. + func testTheAgentsOwnLastPostIsNotTheUser() { + XCTAssertFalse(userIsActive(idleSeconds: 1.0, ownInputAgo: 1.0, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 0.9, ownInputAgo: 1.1, threshold: 5), "within the 0.3 s slack") + XCTAssertTrue(userIsActive(idleSeconds: 0.5, ownInputAgo: 2.0, threshold: 5), "the user typed after the agent") + } + + func testOnlyTheRunnerTurnsTheGuardOnAndATypoKeepsTheDefault() { + XCTAssertTrue(foregroundGuardEnabled(["GENESIS_CONTROL_FOREGROUND_GUARD": "1"])) + XCTAssertFalse(foregroundGuardEnabled([:]), "a person running the command is never refused") + XCTAssertEqual(foregroundIdleThreshold([:]), defaultForegroundIdleSeconds) + XCTAssertEqual(foregroundIdleThreshold(["GENESIS_CONTROL_FOREGROUND_IDLE": "0"]), 0) + XCTAssertEqual(foregroundIdleThreshold(["GENESIS_CONTROL_FOREGROUND_IDLE": "2.5"]), 2.5) + XCTAssertEqual(foregroundIdleThreshold(["GENESIS_CONTROL_FOREGROUND_IDLE": "five"]), defaultForegroundIdleSeconds) + XCTAssertTrue(foregroundAllowedByEnvironment(["GENESIS_CONTROL_ALLOW_FOREGROUND": "yes"])) + XCTAssertFalse(foregroundAllowedByEnvironment(["GENESIS_CONTROL_ALLOW_FOREGROUND": "0"])) + } + + func testTheRefusalNamesTheBackgroundPathsAndTheOverride() { + let message = UserActiveError(idleSeconds: 1.24, threshold: 5).localizedDescription + for expected in ["1.2 s ago", "press_key with activate:false", "launch_app with activate:false", + "allow_foreground:true", "GENESIS_CONTROL_FOREGROUND_IDLE=0"] { + XCTAssertTrue(message.contains(expected), "\(expected) in: \(message)") + } + XCTAssertEqual(UserActiveError(idleSeconds: 1, threshold: 5).category.rawValue, "user_active") + } + + func testTheGateReportsEveryPostSoTheAgentsInputIsKnown() throws { + var reports = 0 + let gate = SyntheticInputGate(tookOver: { false }, onPost: { reports += 1 }) + try gate.post {} + let held = try gate.press({}, release: {}) + gate.release(held) + gate.deliver {} + XCTAssertEqual(reports, 4) + XCTAssertEqual(gate.posted, 4) + } +} diff --git a/src/control/commands/workflow.ts b/src/control/commands/workflow.ts index b4dbf96959..9d8d1b727d 100644 --- a/src/control/commands/workflow.ts +++ b/src/control/commands/workflow.ts @@ -65,6 +65,7 @@ interface WorkflowOptions { frame?: string; background?: boolean; allowOutside?: boolean; + allowForeground?: boolean; prepare?: boolean; replace?: boolean; targetKey?: string; @@ -517,6 +518,10 @@ export function registerWorkflowCommands(program: Command): void { "--allow-outside", "drag: the path and --to may leave the snapshot window, and the window may move with the drag (a panel dragged by its body). Only the start is checked against the snapshot; each later step checks that the window is still on screen and that you have not taken the pointer. The result reports windowBoundsAfter." ) + .option( + "--allow-foreground", + "under an agent, take focus or the pointer even while the user is working (a visible demonstration the user asked for); without it such a call is refused while the user typed, clicked or moved the pointer in the last 5 s (GENESIS_CONTROL_FOREGROUND_IDLE)" + ) .option("--duration ", "drag: duration from 0.1 to 5 seconds") .option("--pages ", "scroll: 1–20 receiving AX scroll-area viewport pages; if unavailable use --pixels") .option("--pixels ", "scroll: exact synthetic wheel pixels from 1 to 10000; mutually exclusive with --pages") @@ -688,6 +693,10 @@ export function registerWorkflowCommands(program: Command): void { args.push("--allow-outside"); } + if (opts.allowForeground) { + args.push("--allow-foreground"); + } + if (opts.refresh) { args.push("--refresh"); } diff --git a/src/control/lib/computer-use/schemas.ts b/src/control/lib/computer-use/schemas.ts index a9f32fe95b..13e3c4598a 100644 --- a/src/control/lib/computer-use/schemas.ts +++ b/src/control/lib/computer-use/schemas.ts @@ -20,10 +20,23 @@ const common = { timeout_ms: z.number().int().min(1).max(120000).optional(), note: noteSchema, }; +/** + * The window a call means, as get_app_state takes it. Refused unless the app's latest observation is of that + * window, so a call never acts on another window's references or pixels. + */ +const windowIdSchema = z.number().int().positive().optional(); +/** + * While the user works (a key, click, move or scroll in the last few seconds), an agent's call refuses to bring + * the app to the front or move the pointer. true takes focus anyway: only for a visible demonstration the user + * asked for. + */ +const allowForegroundSchema = z.boolean().default(false); const target = { ...common, element_index: z.number().int().nonnegative().optional(), element_ref: z.string().min(1).max(100).optional(), + window_id: windowIdSchema, + allow_foreground: allowForegroundSchema, }; const preparedTarget = { ...target, prepare: z.boolean().default(false) }; const point = { @@ -151,7 +164,9 @@ export const computerSchemas = { .object({ bundle_id: appSchema.optional(), path: z.string().min(1).max(4096).optional(), + /** false: launch or find the app without bringing it to the front. */ activate: z.boolean().default(true), + allow_foreground: allowForegroundSchema, timeout_ms: z.number().int().min(1).max(120000).optional(), }) .strict() @@ -192,6 +207,8 @@ export const computerSchemas = { coordinate_space: z.enum(["image", "screen"]).default("image"), duration: z.number().min(0.1).max(5).default(0.3), background: z.boolean().default(true), + window_id: windowIdSchema, + allow_foreground: allowForegroundSchema, /** The destination may lie outside the observed window, and the window may move with the drag. */ allow_outside: z.boolean().default(false), }) @@ -261,6 +278,7 @@ export const computerSchemas = { query: z.string().min(1).max(300), role: z.string().max(100).optional(), limit: z.number().int().min(1).max(100).default(30), + window_id: windowIdSchema, }) .strict(), resolve_visual_target: z diff --git a/src/control/lib/computer-use/session.test.ts b/src/control/lib/computer-use/session.test.ts index fc7931b57c..1b82d0b696 100644 --- a/src/control/lib/computer-use/session.test.ts +++ b/src/control/lib/computer-use/session.test.ts @@ -1920,3 +1920,53 @@ test("menu-bar-only observations explicitly select the native bar surface", asyn await session.observe({ app: "Fixture", top_menu: "File", limit: 40 }); expect(calls[2]).toEqual(["menu-see", "--app", "Fixture", "--menu", "File"]); }); + +test("a call that names a window acts only on that window's observation, and allow_foreground reaches ax-tool", async () => { + const f = fixture(); + const state = await f.computer.get_app_state({ app: "Fixture", image: false }); + const ref = state.elements[1].ref; + const sent = f.calls.length; + await expect(f.computer.click({ app: "Fixture", element_ref: ref, window_id: 11 })).rejects.toThrow( + "get_app_state({app, window_id: 11})" + ); + expect(() => f.computer.find({ app: "Fixture", query: "Fixture", window_id: 11 })).toThrow("not 11"); + expect(f.calls).toHaveLength(sent); + const result = await f.computer.click({ + app: "Fixture", + element_ref: ref, + window_id: 10, + prepare: true, + allow_foreground: true, + }); + expect(result.ok).toBe(true); + expect(f.calls.at(-1)).toContain("--allow-foreground"); + const plain = await f.computer.get_app_state({ app: "Fixture", image: false }); + await f.computer.click({ app: "Fixture", element_ref: plain.elements[1].ref, prepare: true }); + expect(f.calls.at(-1)).not.toContain("--allow-foreground"); +}); + +test("launch_app passes allow_foreground to ax-tool only when asked", async () => { + const f = fixture(); + f.native.run = async (call) => { + f.calls.push(call.args); + return { ok: true, requestAccepted: true }; + }; + await f.computer.launch_app({ bundle_id: "example.fixture" }); + await f.computer.launch_app({ bundle_id: "example.fixture", allow_foreground: true }); + expect(f.calls).toEqual([ + ["launch-app", "--bundle-id", "example.fixture"], + ["launch-app", "--bundle-id", "example.fixture", "--allow-foreground"], + ]); +}); + +test("a background click into a window that is not key carries the native effect note to the caller", async () => { + const f = fixture(); + const state = await f.computer.get_app_state({ app: "Fixture", image: false }); + const run = f.native.run; + f.native.run = async (call) => { + const result = await run(call); + return call.args[0] === "act" ? { ...result, inactiveWindowClick: true, effectNote: "not key" } : result; + }; + const result = await f.computer.click({ app: "Fixture", element_ref: state.elements[1].ref }); + expect(result.effectNote).toBe("not key"); +}); diff --git a/src/control/lib/computer-use/session.ts b/src/control/lib/computer-use/session.ts index 9c1f55e672..baf50987a5 100644 --- a/src/control/lib/computer-use/session.ts +++ b/src/control/lib/computer-use/session.ts @@ -134,6 +134,18 @@ export interface ComputerState { } | null; observedAt: string; } +/** A call that names a window must act on that window's observation, never on another window's refs or pixels. */ +function assertObservedWindow({ record, app, windowId }: { record: AppRecord; app: string; windowId?: number }) { + if (windowId === undefined || record.snapshot.window.id === windowId) { + return; + } + + throw new ComputerUseError( + "WRONG_WINDOW", + `The latest observation of ${app} is window ${record.snapshot.window.id}, not ${windowId}. Observe it first: get_app_state({app, window_id: ${windowId}}).` + ); +} + export class ComputerUseError extends Error { constructor( readonly code: string, @@ -937,6 +949,7 @@ export class ComputerUse { "launch-app", ...(options.bundle_id ? ["--bundle-id", options.bundle_id] : ["--path", options.path!]), ...(!options.activate ? ["--background"] : []), + ...(options.allow_foreground ? ["--allow-foreground"] : []), ], timeoutMs: options.timeout_ms ?? 12000, signal, @@ -1064,6 +1077,8 @@ export class ComputerUse { revision?: string; timeout_ms?: number; note?: string; + window_id?: number; + allow_foreground?: boolean; }; build: (record: AppRecord) => string[]; signal?: AbortSignal; @@ -1073,6 +1088,7 @@ export class ComputerUse { app: input.app, revision: input.revision, }); + assertObservedWindow({ record, app: input.app, windowId: input.window_id }); const argv = build(record); this.menus.close(input.app); const args = [ @@ -1082,6 +1098,7 @@ export class ComputerUse { "--snapshot", record.snapshot.snapshot, ...argv, + ...(input.allow_foreground ? ["--allow-foreground"] : []), "--refresh", ...(!record.image ? ["--no-image"] : []), ]; @@ -1167,6 +1184,9 @@ export class ComputerUse { clipboardRestore: result.clipboardRestore, recovery: result.recovery, note: typeof result.note === "string" ? result.note : undefined, + // A window-addressed click into a window that is not key: delivered, but a SwiftUI control may + // have taken it as the activation click. The caller checks the postcondition. + effectNote: typeof result.effectNote === "string" ? result.effectNote : undefined, windowBoundsAfter: windowBoundsSchema.safeParse(result.windowBoundsAfter).data, }; }); @@ -1423,6 +1443,8 @@ export class ComputerUse { revision?: string; timeout_ms?: number; prepare?: boolean; + window_id?: number; + allow_foreground?: boolean; }; action: string; flags: string[]; @@ -1612,6 +1634,7 @@ export class ComputerUse { const record = this.record({ app: options.app, }); + assertObservedWindow({ record, app: options.app, windowId: options.window_id }); const query = options.query.toLocaleLowerCase(); const elements: ComputerElement[] = []; let total = 0; diff --git a/src/control/lib/decision/native.test.ts b/src/control/lib/decision/native.test.ts index ef597a6c07..aa6e6c9434 100644 --- a/src/control/lib/decision/native.test.ts +++ b/src/control/lib/decision/native.test.ts @@ -216,6 +216,12 @@ describe("OCR reuse", () => { }); }); +describe("user active", () => { + test("is its own refusal, so recovery never retries the call in front", () => { + expect(actionRefusal({ ok: false, dispatchState: "not_started", refusal: "user_active" })).toBe("user_active"); + }); +}); + describe("user takeover", () => { test("is its own refusal whatever the dispatch state, and recovery stops before any read or model call", async () => { expect(actionRefusal({ ok: false, dispatchState: "uncertain", refusal: "user_takeover" })).toBe( diff --git a/src/control/lib/decision/recovery.ts b/src/control/lib/decision/recovery.ts index 672de9224c..ff5ec97a6c 100644 --- a/src/control/lib/decision/recovery.ts +++ b/src/control/lib/decision/recovery.ts @@ -20,6 +20,8 @@ export const refusalSchema = z.enum([ "refused", /** The user put the pointer in the takeover corner; ax-tool stopped and released every held input. */ "user_takeover", + /** The user is at the keyboard or mouse, so an agent's call refused to take focus or the pointer. */ + "user_active", ]); export type Refusal = z.infer; export function actionRefusal(result: AxResult): Refusal { @@ -29,6 +31,10 @@ export function actionRefusal(result: AxResult): Refusal { return "user_takeover"; } + if (result.refusal === "user_active") { + return "user_active"; + } + if (result.dispatchState !== "not_started") { return "transport_uncertainty"; } @@ -155,6 +161,12 @@ export class RecoveryController { return null; } + if (category === "user_active") { + attempt.reason = + "The user is working, so taking focus was refused; act in the background instead of retrying in front."; + return null; + } + if (["permission", "authentication", "scope_changed", "transport_uncertainty", "refused"].includes(category)) { attempt.reason = "This refusal requires inspection or user input; no automatic recovery."; return null; diff --git a/src/control/lib/runner.test.ts b/src/control/lib/runner.test.ts index ee787a5367..ee1e26d3a4 100644 --- a/src/control/lib/runner.test.ts +++ b/src/control/lib/runner.test.ts @@ -14,6 +14,7 @@ import { axCommandLine, DEFAULT_AX_RUN_BOUNDARY, ensureBinary, + foregroundGuardEnvironment, getBinaryPath, nativeEnvironment, REAL_AX_TOOL_IN_TESTS, @@ -756,3 +757,12 @@ test("every ax-tool spawn carries the overlay note: one call's note over the com } ); }); + +test("foregroundGuardEnvironment turns the guard on under an agent only, and an explicit setting wins", () => { + expect(foregroundGuardEnvironment({ CLAUDECODE: "1" })).toEqual({ GENESIS_CONTROL_FOREGROUND_GUARD: "1" }); + expect(foregroundGuardEnvironment({ CODEX_THREAD_ID: "thread" })).toEqual({ + GENESIS_CONTROL_FOREGROUND_GUARD: "1", + }); + expect(foregroundGuardEnvironment({})).toEqual({}); + expect(foregroundGuardEnvironment({ CLAUDECODE: "1", GENESIS_CONTROL_FOREGROUND_GUARD: "0" })).toEqual({}); +}); diff --git a/src/control/lib/runner.ts b/src/control/lib/runner.ts index ae78a58301..42c0e088c8 100644 --- a/src/control/lib/runner.ts +++ b/src/control/lib/runner.ts @@ -1,7 +1,7 @@ import { type ChildProcess, spawn, spawnSync } from "node:child_process"; import { appendFileSync, existsSync, readFileSync } from "node:fs"; import { join, resolve } from "node:path"; -import { agentSessionIds } from "@genesiscz/utils/agent/host"; +import { agentSessionIds, resolveAgentHost } from "@genesiscz/utils/agent/host"; import { untilAborted } from "@genesiscz/utils/async"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; @@ -717,12 +717,26 @@ export function nativeEnvironment(note?: string): NodeJS.ProcessEnv { return { ...environment, + ...foregroundGuardEnvironment(environment), ...cursorEnvironment({ note: note ?? cursorNote ?? environment[CURSOR_NOTE_ENV], session: environment[CURSOR_SESSION_ENV] ?? sessionLabel ?? undefined, }), }; } + +/** + * Under an agent, ax-tool refuses to take focus or the pointer while the user works (2026-10-23: a test run took + * the user's focus again and again). A person running a command just pressed Return, so outside an agent the guard + * stays off. An explicit GENESIS_CONTROL_FOREGROUND_GUARD in the environment wins either way. + */ +export function foregroundGuardEnvironment(environment: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + if (environment.GENESIS_CONTROL_FOREGROUND_GUARD !== undefined || !resolveAgentHost(environment).isInAgent) { + return {}; + } + + return { GENESIS_CONTROL_FOREGROUND_GUARD: "1" }; +} function nativeArguments(args: string[]): string[] { const mutating = [ "act", From f7926eee819e3fe43ce7b844bbffa19eef0b9bc9 Mon Sep 17 00:00:00 2001 From: Martin Date: Sat, 24 Oct 2026 00:00:42 +0200 Subject: [PATCH 3/6] docs(skills, macos-control, control): the macos-control skill puts the background first, names the calls that take focus and the new refusal, and records the window_id, list_windows, drag and inactive-window facts agents tripped over --- .../genesis-tools/skills/macos-control/SKILL.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/genesis-tools/skills/macos-control/SKILL.md b/plugins/genesis-tools/skills/macos-control/SKILL.md index 5722099f00..712806da55 100644 --- a/plugins/genesis-tools/skills/macos-control/SKILL.md +++ b/plugins/genesis-tools/skills/macos-control/SKILL.md @@ -35,12 +35,22 @@ Check the command named by help, not just exit zero. Do not guess command plural Read [automation-playbooks.md](references/automation-playbooks.md) before browser input, modal recovery, a Jev demonstration or multi-app execution. It contains complete examples and the failure-to-fix table. Read [native-cli.md](references/native-cli.md) for selector/snapshot/coordinate/plan contracts. [providers.md](references/providers.md) maps the standalone API, MCP and REPL. +## Don't take the user's focus + +The user keeps working while an agent drives their Mac. A call that brings an app to the front or moves the pointer interrupts them, and the keys they type next land in the agent's window. + +- **The background is the default.** `click` without `physical` or `prepare` uses AXPress or a window-addressed click. Use `press_key({..., activate:false})`, `set_value` without `prepare`, `fill_form({..., activate:false})` and `launch_app({path, activate:false})` (or `bundle_id`). `scroll` posts to the window. Open URLs and files with `open -g`, never with a bare `open -a` or `open -R`. +- **These take focus or the pointer:** `prepare:true`, `physical:true` on a browser page, `press_key` and `launch_app` with their default `activate:true`, `drag({background:false})`, and hover. Use them only for a visible demonstration the user asked for. +- **Under an agent, ax-tool refuses them while the user works.** A key, click, move or scroll in the last 5 s gives refusal `user_active` with `dispatchState:"not_started"`: nothing was sent. Do not retry in front; take the background path the error names. For a demonstration the user asked for, pass `allow_foreground:true` (CLI `--allow-foreground`, or `GENESIS_CONTROL_ALLOW_FOREGROUND=1` for a whole run). `GENESIS_CONTROL_FOREGROUND_IDLE=` sets the quiet time, and `0` turns the check off. A person running a command is never refused. +- **A background click can do nothing.** A window-addressed click into an app that is not in front lands in a window that is not key. A SwiftUI control there (a sheet's "Cancel") takes it as the activation click and does nothing, yet the call returns `ok:true` with an `effectNote`. AXPress (a click without `physical`) acts without focus, but it sends no mouse event, so an app's NSEvent monitor does not see it. Check the postcondition after every background click. +- **Start apps with `launch_app`.** It goes through LaunchServices, as a Finder double-click does, so the app is its own process: its menu-bar item and its privacy prompts belong to it, not to the terminal. Do not run an app's binary from a shell, and do not open it through Finder's `AXOpen` (AX -25205 on a list item, and `/Applications` exceeds the observation budget). + ## Execute the requested task 1. Observe running apps and current windows. Pick a unique observed `window_id`; indexes reorder. Without one, `get_app_state` takes the only real window and ignores popups (a translate bubble, a hover card); with two real windows it refuses and names each `window_id`. Use `scope:"chrome", image:false` for browser tabs and address controls, `scope:"window"` for page content. Never turn a read timeout, oversized tree or permission error into evidence that a window is empty. When a page or an open panel exceeds the snapshot limit, observe one element with `get_app_state({app, window_id, query, role})`: the whole window is walked, only matches and their ancestors become rows, the refs work with `click`/`press_key`, and the state text says how many subtrees depth left unwalked. 2. Check for a visible `AXSheet` or explicitly modal container before choosing background controls. Snapshot dispatch blocks targets behind it. If the fast tree has empty roles/geometry, it is incomplete evidence: the backend falls back to normal AX reads. Inspect the dialog's text and act only within the user's authorization; no automatic acceptance of installs, authentication or permissions. 3. Construct bounded candidates locally. Exact identity/filtering comes first; Jev decides meaning when requested. Distinguish repeated labels by subtree, URL, role and nearby text. A `Source` link for React differs from TypeScript Source; an update page may contain two `Info` buttons. Do not send supplied form values to Jev. -4. Act through the newest reference. Use `prepare:true` for observed browser controls or focused keyboard input. A prepared native action with its original target fingerprint can automatically recover from a confirmed `not_started` stale/focus refusal, at most twice within a 1.5-second admission retry window. Every attempt uses the remaining original deadline, including action readback. This repeats admission, not completed workflow steps or Jev decisions. When a retry finds the target changed or gone, the call reports `effect:"unknown"` with every attempt's refusal (and what held focus): no attempt posted the input, but the UI moved after the first attempt focused it, so observe before repeating. Native AXPress/value writes can remain unfocused; high-level tasks prepare automatically where needed. For a visible demonstration, deliberately foreground the exact window and keep cursor feedback on. +4. Act through the newest reference, in the background (see above). `prepare:true` focuses the window first: use it only for a visible demonstration the user asked for, or for a browser control that ignores background input while the user is away. A prepared native action with its original target fingerprint can automatically recover from a confirmed `not_started` stale/focus refusal, at most twice within a 1.5-second admission retry window. Every attempt uses the remaining original deadline, including action readback. This repeats admission, not completed workflow steps or Jev decisions. When a retry finds the target changed or gone, the call reports `effect:"unknown"` with every attempt's refusal (and what held focus): no attempt posted the input, but the UI moved after the first attempt focused it, so observe before repeating. Native AXPress/value writes can remain unfocused; high-level tasks prepare automatically where needed. For a visible demonstration, deliberately foreground the exact window and keep cursor feedback on. 5. Verify a fresh postcondition. Dispatch is not completion. Use exact URL/title/value/state where available; `expected_url` pins compound browser tasks. Unknown delivery stops mutations. Read fresh state to discover what happened; do not repeat the input just because readback failed. ## Contracts to remember @@ -49,6 +59,8 @@ Read [automation-playbooks.md](references/automation-playbooks.md) before browse - **Fill without taking the user's focus.** `set_value({app, element_ref, value})` without `prepare`, or `fill_form({..., activate:false})`, writes AXValue inside the app: no activation, no clipboard, no keys, so the user can keep typing. It works on Chromium/Electron pages too (measured on T3 Code, 2026-10-13); the readback waits up to 1 s because Chromium applies the write late. The cursor label shows the text written (a secure field shows bullets). A field that saves its draft on blur or Return keeps the text unsaved while the app is in the back; fill that one in front with `paste({..., replace:true, prepare:true})`. - **Jev abstention stays an abstention.** Do not lower thresholds or retry the same question until it accepts. Improve genuinely missing evidence, narrow an ambiguous scope, or return a host packet. An independently justified exact host selection must be reported separately from a model choice. Host replies revalidate fresh evidence and consume no extra model call. - **Bound every loop.** `assist_task`, `fill_form`, `run_workflow` and `await_condition` share deadlines and request/action caps. Waits may use exact local evidence without Jev. Supply an `evidence_scope` to ignore changing unrelated panels; missing/ambiguous scope stops. +- **Name the window.** `click`, `press_key`, `set_value`, `type_text`, `paste`, `scroll`, `drag` and `find` take the `window_id` that `get_app_state` took, and refuse with `WRONG_WINDOW` when the app's latest observation is another window. `list_windows` returns `{windows:[…]}`, not an array. `drag` needs a `get_app_state` observation first, because its coordinates are that observation's pixels. +- **A changing label refuses a pointer click.** A physical click on a panel whose label ticks (a running clock) can refuse with "observed target changed"; AXPress on the same button is pinned by identity and works. Right after a sheet button, the tree can briefly show no `AXSheet` while the sheet is still up, so wait for the change with `await_condition` or a fresh `get_app_state` before the next click. - **Scrolling never takes focus, and can move like a person.** `control scroll --direction` posts to the window, not the frontmost app. To reproduce scroll bugs use `--time` (with `--repeat`/`--alternate`), not a loop of single wheel events; details in [native-cli.md](references/native-cli.md#scrolling-control-scroll). - **Native dropdowns are exact.** `AXPopUpButton` selects one enabled observed menu title and reads it back. Duplicate/missing choices refuse. `AXInvalid` stops subsequent form writes where the app exposes it. Arbitrary custom web dropdowns need their own observed workflow. - **Reference freshness and visibility are different.** Pin process/window/document, reobserve after changes, and use one-use pixel evidence for coordinates. Multiple displays can have negative origins and different scales. API coordinates default to screenshot pixels; native `act --coords` uses global screen points. From 1bb65ba72c0906472a8c87947d9c2d037235396f Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 26 Oct 2026 02:19:17 +0100 Subject: [PATCH 4/6] fix(control): #591 followup - every global click or press passes the foreground gate even into the frontmost app, synthetic input never shortens the user's quiet time, a restore is gated too, and window mode refuses an excluded target --- .claude-plugin/marketplace.json | 4 +- .../SnapshotSupport/ForegroundGuard.swift | 31 +++++++---- .../SnapshotSupport/IsolatedComposition.swift | 7 +++ native/ax-tool/Sources/Record.swift | 5 ++ native/ax-tool/Sources/commands/Harness.swift | 12 +++- .../Sources/commands/SnapshotRestore.swift | 3 + native/ax-tool/Sources/utils/InputGate.swift | 46 ++-------------- .../ax-tool/Tests/ForegroundGuardTests.swift | 55 +++++++++++++++---- .../Tests/IsolatedCompositionTests.swift | 8 +++ .../genesis-tools/.claude-plugin/plugin.json | 2 +- src/control/lib/native-platform.test.ts | 41 ++++++++++++++ 11 files changed, 147 insertions(+), 67 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c741905f35..63b098bd5d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,7 +1,7 @@ { "$schema": "https://anthropic.com/claude-code/marketplace.schema.json", "name": "genesis-tools", - "version": "1.0.116", + "version": "1.0.117", "description": "Plugins for GenesisTools CLI development and management", "owner": { "name": "genesiscz", @@ -11,7 +11,7 @@ { "name": "genesis-tools", "description": "Skills and utilities for working with GenesisTools CLI toolkit. Provides guidance for discovering, executing, and troubleshooting genesis tools with integrated workflow support.", - "version": "1.0.116", + "version": "1.0.117", "author": { "name": "GenesisTools" }, diff --git a/native/ax-tool/SnapshotSupport/ForegroundGuard.swift b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift index efac8e93d8..d1304d662b 100644 --- a/native/ax-tool/SnapshotSupport/ForegroundGuard.swift +++ b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift @@ -32,19 +32,30 @@ public func foregroundAllowedByEnvironment(_ environment: [String: String]) -> B return ["1", "true", "yes", "on"].contains(raw) } -/// True when the newest key, click, move or scroll is the user's and came within `threshold` seconds. -/// The window server counts this tool's own synthetic events as input too, so an event no newer than its -/// last post (`ownInputAgo`, with 0.3 s of slack for the throttled record) is the agent's, not the user's. -public func userIsActive(idleSeconds: TimeInterval, ownInputAgo: TimeInterval?, threshold: TimeInterval) -> Bool { - guard threshold > 0, idleSeconds.isFinite, idleSeconds >= 0, idleSeconds < threshold else { - return false - } +/// Any recent input keeps the guard closed. The window server's idle counter includes synthetic +/// posts, so subtracting our last post could hide a human event immediately before or after it. +/// Conservatively, synthetic input can extend the quiet time but can never shorten it. +public func userIsActive(idleSeconds: TimeInterval, threshold: TimeInterval) -> Bool { + threshold > 0 && idleSeconds.isFinite && idleSeconds >= 0 && idleSeconds < threshold +} - if let own = ownInputAgo, idleSeconds >= own - 0.3 { - return false +/// Admit a command's first global event independently of whether its app is already frontmost. +/// Later events in that command must not refuse on the idle counter its own first event reset. +public final class ForegroundInputGate { + private let check: () throws -> Void + private var admitted = false + + public init(check: @escaping () throws -> Void) { + self.check = check } - return true + public func dispatch(background: Bool, _ send: () throws -> T) throws -> T { + if !background && !admitted { + try check() + admitted = true + } + return try send() + } } /// The user is at the keyboard or mouse, so this call did not take focus. Never retried automatically. diff --git a/native/ax-tool/SnapshotSupport/IsolatedComposition.swift b/native/ax-tool/SnapshotSupport/IsolatedComposition.swift index 7a6e1a3c24..c8a6ee763a 100644 --- a/native/ax-tool/SnapshotSupport/IsolatedComposition.swift +++ b/native/ax-tool/SnapshotSupport/IsolatedComposition.swift @@ -3,6 +3,13 @@ import CoreImage import CoreMedia import Foundation +/// A single-window stream cannot filter its own target out. Refuse before starting capture. +public func validateRecordingWindow(_ id: CGWindowID, excluding excluded: Set) throws { + guard !excluded.contains(id) else { + throw WorkflowArgumentError.invalid("window \(id) is excluded by --exclude-window-ids; choose another capture target") + } +} + /// One selected window of an isolated recording. `Frame` is the last complete ScreenCaptureKit /// surface in the recorder and a synthetic image in tests: the visibility, readiness and /// stacking rules never look inside it. diff --git a/native/ax-tool/Sources/Record.swift b/native/ax-tool/Sources/Record.swift index c954003540..4be429aaad 100644 --- a/native/ax-tool/Sources/Record.swift +++ b/native/ax-tool/Sources/Record.swift @@ -520,6 +520,11 @@ func cmdCaptureScreen() { switch options.mode { case "window": let target = resolveRecordWindow(options) + do { + try validateRecordingWindow(target.id, excluding: options.selection.excludedWindowIDs) + } catch { + errorExit(error.localizedDescription) + } windowTarget = target display = displayContaining(CGPoint(x: target.bounds.midX, y: target.bounds.midY)) case "region": diff --git a/native/ax-tool/Sources/commands/Harness.swift b/native/ax-tool/Sources/commands/Harness.swift index a9f79d36c5..a25b6edc8a 100644 --- a/native/ax-tool/Sources/commands/Harness.swift +++ b/native/ax-tool/Sources/commands/Harness.swift @@ -21,13 +21,21 @@ import Vision extension CGEvent { /// One checked event (a wheel step, a move). A user takeover ends the command before it posts. func postRouted() { - gatedOrExit { try inputGate.post(deliverRouted) } + gatedOrExit { + try foregroundInputGate.dispatch(background: argValue("--to-pid") != nil) { + try inputGate.post(deliverRouted) + } + } } /// A checked down whose up is `release`. The up goes out through `inputGate.release`, or /// through the gate itself if the user takes over first, so nothing is left held. func pressRouted(release up: CGEvent) -> SyntheticInputGate.Held { - gatedOrExit { try inputGate.press(deliverRouted, release: up.deliverRouted) } + gatedOrExit { + try foregroundInputGate.dispatch(background: argValue("--to-pid") != nil) { + try inputGate.press(deliverRouted, release: up.deliverRouted) + } + } } /// The raw delivery, with no takeover check: only the gate calls this. diff --git a/native/ax-tool/Sources/commands/SnapshotRestore.swift b/native/ax-tool/Sources/commands/SnapshotRestore.swift index 8ce23b8e6d..4f56b0c959 100644 --- a/native/ax-tool/Sources/commands/SnapshotRestore.swift +++ b/native/ax-tool/Sources/commands/SnapshotRestore.swift @@ -36,6 +36,9 @@ func cmdRestore(snapshotJson: String) { errorExit("invalid snapshot JSON") } + // Restoring a saved snapshot can take focus or move the pointer while the user works too. + gatedOrExit { try checkForegroundAllowed() } + if let mouse = snap["mouse"] as? [String: Double], let mx = mouse["x"], let my = mouse["y"] { let point = CGPoint(x: mx, y: my) diff --git a/native/ax-tool/Sources/utils/InputGate.swift b/native/ax-tool/Sources/utils/InputGate.swift index 1523d80ddf..4dea31a44b 100644 --- a/native/ax-tool/Sources/utils/InputGate.swift +++ b/native/ax-tool/Sources/utils/InputGate.swift @@ -31,7 +31,8 @@ private func pointerTakeover() -> Bool { } /// The gate every synthetic event of this process passes through. -let inputGate = SyntheticInputGate(tookOver: pointerTakeover, onPost: recordOwnInput) +let inputGate = SyntheticInputGate(tookOver: pointerTakeover) +let foregroundInputGate = ForegroundInputGate(check: checkForegroundAllowed) private let foregroundEnvironment = ProcessInfo.processInfo.environment /// Read once per process: the agent's run cannot change these under a running command. @@ -47,45 +48,6 @@ private func inputIdleSeconds() -> TimeInterval { return types.map { CGEventSource.secondsSinceLastEventType(.hidSystemState, eventType: $0) }.min() ?? .infinity } -/// When any process of this tool last posted a synthetic event. The window server counts those as -/// input, and an agent's own previous click must not read as the user working. -private let ownInputFile: URL = { - let home = foregroundEnvironment["GENESIS_TOOLS_HOME"].map { URL(fileURLWithPath: $0) } - ?? FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent(".genesis-tools") - return home.appendingPathComponent("control").appendingPathComponent("last-synthetic-input") -}() -private var ownInputWrittenAt: TimeInterval = 0 - -/// At most one write per quarter second; the guard's 0.3 s slack covers the posts in between. -private func recordOwnInput() { - guard foregroundCheckEnabled else { - return - } - - let now = Date().timeIntervalSince1970 - guard now - ownInputWrittenAt >= 0.25 else { - return - } - - ownInputWrittenAt = now - do { - try FileManager.default.createDirectory(at: ownInputFile.deletingLastPathComponent(), - withIntermediateDirectories: true) - try String(now).write(to: ownInputFile, atomically: true, encoding: .utf8) - } catch { - fputs("could not record the time of this tool's own input: \(error.localizedDescription)\n", stderr) - } -} - -private func ownInputAgo() -> TimeInterval? { - guard let text = try? String(contentsOf: ownInputFile, encoding: .utf8), - let at = Double(text.trimmingCharacters(in: .whitespacesAndNewlines)) else { - return nil - } - - return Date().timeIntervalSince1970 - at -} - /// Refuses to bring an app to the front or move the pointer while the user is at the keyboard or mouse, /// when an agent drives this tool. Call it before the first change, so a refusal sent nothing. func checkForegroundAllowed() throws { @@ -94,7 +56,7 @@ func checkForegroundAllowed() throws { } let idle = inputIdleSeconds() - guard userIsActive(idleSeconds: idle, ownInputAgo: ownInputAgo(), threshold: foregroundThreshold) else { + guard userIsActive(idleSeconds: idle, threshold: foregroundThreshold) else { return } @@ -122,6 +84,8 @@ func gatedOrExit(_ body: () throws -> T) -> T { return try body() } catch let takeover as UserTakeoverError { takeoverExit(takeover) + } catch let refusal as UserActiveError { + userActiveExit(refusal) } catch { errorExit(error.localizedDescription) } diff --git a/native/ax-tool/Tests/ForegroundGuardTests.swift b/native/ax-tool/Tests/ForegroundGuardTests.swift index 505b220dfa..33041c606b 100644 --- a/native/ax-tool/Tests/ForegroundGuardTests.swift +++ b/native/ax-tool/Tests/ForegroundGuardTests.swift @@ -5,22 +5,55 @@ import XCTest /// user's focus again and again while they typed in another app). final class ForegroundGuardTests: XCTestCase { func testRecentUserInputMeansTheUserIsActive() { - XCTAssertTrue(userIsActive(idleSeconds: 0.4, ownInputAgo: nil, threshold: 5)) - XCTAssertTrue(userIsActive(idleSeconds: 4.9, ownInputAgo: 30, threshold: 5)) + XCTAssertTrue(userIsActive(idleSeconds: 0.4, threshold: 5)) + XCTAssertTrue(userIsActive(idleSeconds: 4.9, threshold: 5)) } func testAQuietUserOrATurnedOffCheckIsNotActive() { - XCTAssertFalse(userIsActive(idleSeconds: 5, ownInputAgo: nil, threshold: 5)) - XCTAssertFalse(userIsActive(idleSeconds: 60, ownInputAgo: nil, threshold: 5)) - XCTAssertFalse(userIsActive(idleSeconds: 0.1, ownInputAgo: nil, threshold: 0)) - XCTAssertFalse(userIsActive(idleSeconds: .infinity, ownInputAgo: nil, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 5, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 60, threshold: 5)) + XCTAssertFalse(userIsActive(idleSeconds: 0.1, threshold: 0)) + XCTAssertFalse(userIsActive(idleSeconds: .infinity, threshold: 5)) } - /// The window server counts the agent's own synthetic click as input; it must not read as the user. - func testTheAgentsOwnLastPostIsNotTheUser() { - XCTAssertFalse(userIsActive(idleSeconds: 1.0, ownInputAgo: 1.0, threshold: 5)) - XCTAssertFalse(userIsActive(idleSeconds: 0.9, ownInputAgo: 1.1, threshold: 5), "within the 0.3 s slack") - XCTAssertTrue(userIsActive(idleSeconds: 0.5, ownInputAgo: 2.0, threshold: 5), "the user typed after the agent") + func testSyntheticInputNeverClearsRecentHumanActivity() { + // Human input at t=0, then a background synthetic post at t=1. The server reports + // the latter's age, which must not erase the human activity still inside the quiet time. + XCTAssertTrue(userIsActive(idleSeconds: 0.01, threshold: 5)) + XCTAssertTrue(userIsActive(idleSeconds: 0.1, threshold: 5), "human input just after a post is active too") + XCTAssertFalse(userIsActive(idleSeconds: 5, threshold: 5), "a full quiet period still permits foreground work") + } + + func testGlobalDispatchRefusesEvenWhenTheTargetIsAlreadyFrontmost() throws { + var sent = 0 + var checked = 0 + let gate = ForegroundInputGate(check: { + checked += 1 + throw UserActiveError(idleSeconds: 0.1, threshold: 5) + }) + XCTAssertThrowsError(try gate.dispatch(background: false) { sent += 1 }) + XCTAssertEqual(checked, 1) + XCTAssertEqual(sent, 0) + try gate.dispatch(background: true) { sent += 1 } + XCTAssertEqual(checked, 1, "contained background input is allowed while the user works") + XCTAssertEqual(sent, 1) + XCTAssertThrowsError(try gate.dispatch(background: false) { sent += 1 }) + XCTAssertEqual(sent, 1, "a background post must not admit a later global post") + } + + func testAnAdmittedCommandCanFinishItsDoubleClickOrKeyChord() throws { + var checked = 0 + var sent = 0 + let gate = ForegroundInputGate(check: { + checked += 1 + if checked > 1 { + throw UserActiveError(idleSeconds: 0.01, threshold: 5) + } + }) + try gate.dispatch(background: false) { sent += 1 } + try gate.dispatch(background: false) { sent += 1 } + XCTAssertEqual(checked, 1, "the first post resets the server's idle counter") + XCTAssertEqual(sent, 2) } func testOnlyTheRunnerTurnsTheGuardOnAndATypoKeepsTheDefault() { diff --git a/native/ax-tool/Tests/IsolatedCompositionTests.swift b/native/ax-tool/Tests/IsolatedCompositionTests.swift index 4941cdeec2..225af847e6 100644 --- a/native/ax-tool/Tests/IsolatedCompositionTests.swift +++ b/native/ax-tool/Tests/IsolatedCompositionTests.swift @@ -7,6 +7,14 @@ import XCTest /// synthetic frames. A whole-recording stall from one window shows up here as `readyToRender` /// or `paintOrder` going wrong, without ScreenCaptureKit. final class IsolatedCompositionTests: XCTestCase { + func testSingleWindowCaptureRefusesAnExcludedTarget() { + XCTAssertThrowsError(try validateRecordingWindow(42, excluding: [42, 43])) { error in + XCTAssertTrue(error.localizedDescription.contains("window 42 is excluded by --exclude-window-ids")) + } + XCTAssertNoThrow(try validateRecordingWindow(42, excluding: [43])) + XCTAssertNoThrow(try validateRecordingWindow(42, excluding: [])) + } + private let left = CGRect(x: 0, y: 0, width: 10, height: 10) private let right = CGRect(x: 5, y: 0, width: 10, height: 10) diff --git a/plugins/genesis-tools/.claude-plugin/plugin.json b/plugins/genesis-tools/.claude-plugin/plugin.json index f7e21e794f..849dd8c208 100644 --- a/plugins/genesis-tools/.claude-plugin/plugin.json +++ b/plugins/genesis-tools/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "genesis-tools", - "version": "1.0.116", + "version": "1.0.117", "description": "Skills and utilities for working with GenesisTools CLI toolkit. Provides guidance for discovering, executing, and troubleshooting genesis tools with integrated workflow support.", "author": { "name": "GenesisTools" diff --git a/src/control/lib/native-platform.test.ts b/src/control/lib/native-platform.test.ts index 4f57d5922a..c7d91bea94 100644 --- a/src/control/lib/native-platform.test.ts +++ b/src/control/lib/native-platform.test.ts @@ -45,3 +45,44 @@ test("explicit AXShowMenu checks takeover after its baseline and immediately bef /let baseline = workflowMenuBaseline\([^\n]+\)\s+try inputGate\.check\(\)\s+let outcome = workflowShowMenu\([^;]+baseline: baseline\)/ ); }); + +test("legacy global input checks foreground permission independently of app activation", () => { + const source = readFileSync( + join(import.meta.dir, "../../../native/ax-tool/Sources/commands/Harness.swift"), + "utf8" + ); + for (const method of ["postRouted()", "pressRouted(release up: CGEvent)"]) { + const start = source.indexOf(`func ${method}`); + expect(start).toBeGreaterThan(-1); + const body = source.slice(start, source.indexOf("\n }", start)); + expect(body).toMatch( + /foregroundInputGate\.dispatch\(background: argValue\("--to-pid"\) != nil\)\s*\{\s*try inputGate\.(?:post|press)\(/ + ); + } + const gate = readFileSync(join(import.meta.dir, "../../../native/ax-tool/Sources/utils/InputGate.swift"), "utf8"); + expect(gate).toContain("ForegroundInputGate(check: checkForegroundAllowed)"); + expect(gate).toMatch(/catch let refusal as UserActiveError\s*\{\s*userActiveExit\(refusal\)/); + expect(gate).not.toContain("last-synthetic-input"); +}); + +test("snapshot restore checks foreground permission before pointer or app changes", () => { + const source = readFileSync( + join(import.meta.dir, "../../../native/ax-tool/Sources/commands/SnapshotRestore.swift"), + "utf8" + ); + const restore = source.slice(source.indexOf("func cmdRestore(")); + const guard = restore.indexOf("gatedOrExit { try checkForegroundAllowed() }"); + expect(guard).toBeGreaterThan(-1); + expect(restore.indexOf("CGWarpMouseCursorPosition")).toBeGreaterThan(guard); + expect(restore.indexOf("runningApp.activate(")).toBeGreaterThan(guard); +}); + +test("single-window recording validates exclusions before creating a capture stream", () => { + const source = readFileSync(join(import.meta.dir, "../../../native/ax-tool/Sources/Record.swift"), "utf8"); + const guard = source.indexOf( + "try validateRecordingWindow(target.id, excluding: options.selection.excludedWindowIDs)" + ); + expect(guard).toBeGreaterThan(-1); + expect(source.indexOf("windowTarget = target")).toBeGreaterThan(guard); + expect(source.indexOf("SCContentFilter(desktopIndependentWindow: window)")).toBeGreaterThan(guard); +}); From 53c4da4826208d0eb0424b7afc3275c6e4519e98 Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 26 Oct 2026 02:53:05 +0100 Subject: [PATCH 5/6] fix(ax-tool): #591 followup - an app already in front is gated too before a window raise or a focus move, and only a parsed --allow-foreground flag lifts the guard, never a typed value --- .../SnapshotSupport/ForegroundGuard.swift | 35 +++++++++++++++++++ native/ax-tool/Sources/SnapshotWorkflow.swift | 2 ++ native/ax-tool/Sources/utils/InputGate.swift | 2 +- native/ax-tool/Sources/utils/Output.swift | 5 +-- .../ax-tool/Tests/ForegroundGuardTests.swift | 15 ++++++++ 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/native/ax-tool/SnapshotSupport/ForegroundGuard.swift b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift index d1304d662b..3d798f4c25 100644 --- a/native/ax-tool/SnapshotSupport/ForegroundGuard.swift +++ b/native/ax-tool/SnapshotSupport/ForegroundGuard.swift @@ -32,6 +32,41 @@ public func foregroundAllowedByEnvironment(_ environment: [String: String]) -> B return ["1", "true", "yes", "on"].contains(raw) } +/// `--allow-foreground` lifts the guard only as a flag its command parsed, never as the value of another option: +/// `act --text --allow-foreground` types those words and keeps the guard. `arguments` is the whole command line +/// (program, command, options). `act` answers from its own parser; `launch-app` reads its two value options; every +/// other command never gets the flag from the runner and never honours it. +public func foregroundAllowedByArguments(_ arguments: [String]) -> Bool { + guard arguments.count > 2 else { + return false + } + + let command = arguments[1] + let options = Array(arguments.dropFirst(2)) + switch command { + case "act": + return (try? WorkflowArguments(options, command: command))?.flags.contains("--allow-foreground") == true + case "launch-app": + var index = 0 + while index < options.count { + if options[index] == "--bundle-id" || options[index] == "--path" { + index += 2 + continue + } + + if options[index] == "--allow-foreground" { + return true + } + + index += 1 + } + + return false + default: + return false + } +} + /// Any recent input keeps the guard closed. The window server's idle counter includes synthetic /// posts, so subtracting our last post could hide a human event immediately before or after it. /// Conservatively, synthetic input can extend the quiet time but can never shorten it. diff --git a/native/ax-tool/Sources/SnapshotWorkflow.swift b/native/ax-tool/Sources/SnapshotWorkflow.swift index befc82d5a5..3d55adfd14 100644 --- a/native/ax-tool/Sources/SnapshotWorkflow.swift +++ b/native/ax-tool/Sources/SnapshotWorkflow.swift @@ -1139,6 +1139,8 @@ func cmdAct(appName _: String) { guard (axAttribute(element,"AXEnabled") as? NSNumber)?.boolValue != false else { workflowFailure("selected target is disabled") } + // Preparation raises the window and moves keyboard focus, also inside an app that is already in front. + gatedOrExit { try checkForegroundAllowed() } let before = tree.rows[elementIndex] if !secondaryClick { workflowFocus(window, pid:pid, element:window.ax) diff --git a/native/ax-tool/Sources/utils/InputGate.swift b/native/ax-tool/Sources/utils/InputGate.swift index 4dea31a44b..3b7068e744 100644 --- a/native/ax-tool/Sources/utils/InputGate.swift +++ b/native/ax-tool/Sources/utils/InputGate.swift @@ -38,7 +38,7 @@ private let foregroundEnvironment = ProcessInfo.processInfo.environment /// Read once per process: the agent's run cannot change these under a running command. private let foregroundCheckEnabled = foregroundGuardEnabled(foregroundEnvironment) private let foregroundThreshold = foregroundIdleThreshold(foregroundEnvironment) -private let foregroundAllowed = CommandLine.arguments.contains("--allow-foreground") +private let foregroundAllowed = foregroundAllowedByArguments(CommandLine.arguments) || foregroundAllowedByEnvironment(foregroundEnvironment) /// Seconds since the last key, click, pointer move or scroll the window server saw. diff --git a/native/ax-tool/Sources/utils/Output.swift b/native/ax-tool/Sources/utils/Output.swift index c0a03ee0a2..fe890fb2a2 100644 --- a/native/ax-tool/Sources/utils/Output.swift +++ b/native/ax-tool/Sources/utils/Output.swift @@ -41,9 +41,10 @@ func frontmostPid() -> pid_t? { /// re-request each poll (up to ~3s), pumping the runloop so NSWorkspace /// state can update too. /// -/// `restoring`: putting the user's own app back in front, which the foreground guard never refuses. +/// `restoring`: putting the user's own app back in front, which the foreground guard never refuses. An app that is +/// already in front is checked too: its callers go on to raise a window and move keyboard focus inside it. func bringFrontmost(_ pid: pid_t, restoring: Bool = false) -> Bool { - if !restoring && frontmostPid() != pid { + if !restoring { do { try checkForegroundAllowed() } catch let refusal as UserActiveError { diff --git a/native/ax-tool/Tests/ForegroundGuardTests.swift b/native/ax-tool/Tests/ForegroundGuardTests.swift index 33041c606b..9090823d99 100644 --- a/native/ax-tool/Tests/ForegroundGuardTests.swift +++ b/native/ax-tool/Tests/ForegroundGuardTests.swift @@ -24,6 +24,21 @@ final class ForegroundGuardTests: XCTestCase { XCTAssertFalse(userIsActive(idleSeconds: 5, threshold: 5), "a full quiet period still permits foreground work") } + func testTheOverrideIsAFlagTheCommandParsedNeverAValue() { + let act = ["ax-tool", "act", "--app", "Notes", "--snapshot", "s1", "--element", "3", "--action", "type"] + XCTAssertTrue(foregroundAllowedByArguments(act + ["--text", "hi", "--allow-foreground"])) + XCTAssertFalse(foregroundAllowedByArguments(act + ["--text", "--allow-foreground"]), + "text that reads --allow-foreground is typed, not obeyed") + XCTAssertFalse(foregroundAllowedByArguments(act + ["--value", "--allow-foreground"])) + XCTAssertFalse(foregroundAllowedByArguments(["ax-tool", "see", "--app", "Notes", "--allow-foreground"]), + "see takes no such flag, so its parser refuses the line") + XCTAssertTrue(foregroundAllowedByArguments(["ax-tool", "launch-app", "--bundle-id", "example.fixture", "--allow-foreground"])) + XCTAssertFalse(foregroundAllowedByArguments(["ax-tool", "launch-app", "--path", "--allow-foreground"])) + XCTAssertFalse(foregroundAllowedByArguments(["ax-tool", "type", "--text", "--allow-foreground"]), + "a command the runner never gives the flag does not honour it") + XCTAssertFalse(foregroundAllowedByArguments(act + ["--text", "hi"])) + } + func testGlobalDispatchRefusesEvenWhenTheTargetIsAlreadyFrontmost() throws { var sent = 0 var checked = 0 From 2e6e8d0cb3f52fd8098f51449433f5891e193efc Mon Sep 17 00:00:00 2001 From: Martin Date: Mon, 26 Oct 2026 02:59:33 +0100 Subject: [PATCH 6/6] fix(ax-tool): #591 followup - a prepared right-click stays exempt from the preparation check, because it neither raises nor focuses --- native/ax-tool/Sources/SnapshotWorkflow.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/native/ax-tool/Sources/SnapshotWorkflow.swift b/native/ax-tool/Sources/SnapshotWorkflow.swift index 3d55adfd14..16062819b1 100644 --- a/native/ax-tool/Sources/SnapshotWorkflow.swift +++ b/native/ax-tool/Sources/SnapshotWorkflow.swift @@ -1139,8 +1139,11 @@ func cmdAct(appName _: String) { guard (axAttribute(element,"AXEnabled") as? NSNumber)?.boolValue != false else { workflowFailure("selected target is disabled") } - // Preparation raises the window and moves keyboard focus, also inside an app that is already in front. - gatedOrExit { try checkForegroundAllowed() } + // Preparation raises the window and moves keyboard focus, also inside an app that is already in front. A + // prepared right-click does neither: it skips focus and raise and clicks in the background. + if !secondaryClick { + gatedOrExit { try checkForegroundAllowed() } + } let before = tree.rows[elementIndex] if !secondaryClick { workflowFocus(window, pid:pid, element:window.ax)