diff --git a/.claude/rules/tools.md b/.claude/rules/tools.md index d0e25ed541..e25f0acf3d 100644 --- a/.claude/rules/tools.md +++ b/.claude/rules/tools.md @@ -27,7 +27,7 @@ When creating a new tool and writing helper functions, check if the utility is * - `src/utils/string.ts` - String utilities (glob matching, ANSI stripping) - `src/utils/cli/executor.ts` - CLI helpers: `suggestCommand()`, `isInteractive()`, `buildCommand()`, `Executor`, `enhanceHelp()` - `src/utils/storage/storage.ts` - Config & cache management -- `src/utils/storage/root.ts` - `toolDataDir(tool, ...)` / `genesisToolsDir(...)`: every `~/.genesis-tools/...` path (honours `GENESIS_TOOLS_HOME`, no logger import). Never hand-join `homedir()` or `env.paths.getHome()` with `".genesis-tools"` +- `src/utils/storage/root.ts` - `toolDataDir(tool, ...)` / `genesisToolsDir(...)`: every `~/.genesis-tools/...` path (honours `GENESIS_TOOLS_HOME`, no logger import); `genesisToolsDirIn(home, ...)` for an explicit sandbox or fixture home. Never hand-join any home with `".genesis-tools"`: lint-rules `no-hand-joined-genesis-tools` fails it outside tests and `plugins/**` - `src/utils/async.ts` - Async helpers (concurrency, retry, etc.) - `src/utils/json-schema.ts` - JSON schema inference: `inferSchema()`, `formatSchema(value, "skeleton"|"typescript"|"schema")` - `src/utils/ai/local/device.ts` - ONNX Runtime device detection: `detectDevice()`, `resolveDevice()` (CoreML/CUDA/DML/CPU) diff --git a/CLAUDE.md b/CLAUDE.md index 134454965a..e2ddad716d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,7 +50,7 @@ Run `bash scripts/ci/placeholder-check.sh`. The pre-push hook runs it first, and Every teammate/subagent given its own worktree MUST, before ANY other work: 1. **Verify the base commit.** Isolated worktrees are often cut from `origin/master`, NOT the campaign/feature branch you were briefed on. Run `git log --oneline -1`; if the briefed base commit is not an ancestor, `git fetch origin && git reset --hard `. Building on the wrong base silently invalidates every anchor in your brief. -2. **Run `bun install` in the worktree.** A worktree without its own `node_modules` resolves imports against the MAIN repo's dependency tree and fabricates failures that look like "the branch broke the world" (verified repeatedly; see also Tests above). +2. **Run `bun install` in the worktree.** A worktree without its own `node_modules` resolves imports against the MAIN repo's dependency tree and fabricates failures that look like "the branch broke the world" (verified repeatedly; see also Tests above). Install once, yourself: `./tools` never installs by itself in a linked worktree (`TOOLS_AUTOINSTALL=1` opts in), because every fresh install writes new native binaries that macOS checks at their first start, and with several worktrees that check queued for minutes (2026-10-14). 3. **Never bare `bun test`** — always `bun run test` (the wrapper repairs the dependency tree first). Skipping any of these has cost real sessions hours; there are no exceptions. diff --git a/native/ax-tool/SnapshotSupport/WindowEvents.swift b/native/ax-tool/SnapshotSupport/WindowEvents.swift index df61e26835..15e50993a9 100644 --- a/native/ax-tool/SnapshotSupport/WindowEvents.swift +++ b/native/ax-tool/SnapshotSupport/WindowEvents.swift @@ -71,28 +71,46 @@ public final class WindowEventFactory { public func drag(start: CGPoint, points: [CGPoint], stepDelay: Double, verify: (CGPoint) throws -> Void, post: (CGEvent) -> Void) throws { + try drag(start: start, points: points, stepDelay: stepDelay, liveBounds: nil, verify: verify, post: post) + } + + /// `liveBounds` is the `--allow-outside` drag: the path may leave the snapshot window, and the window may move with + /// it (a panel dragged by its body). Each step reads where the window is now and computes the window-local location + /// against that, so an app that converts `locationInWindow` to screen gets the global point; against the snapshot + /// bounds a moving window saw every step off by the distance it had already moved. Only the start must lie inside. + public func drag(start: CGPoint, points: [CGPoint], stepDelay: Double, liveBounds: (() -> CGRect?)?, + verify: (CGPoint) throws -> Void, post: (CGEvent) -> Void) throws { let down = try mouse(type: .leftMouseDown, point: start, clickCount: 1) var release = try mouse(type: .leftMouseUp, point: start, clickCount: 1) + func postRelease() { + if let liveBounds { + // Posting the last move can move the window after its release event was allocated. + // Refresh the local field without risking a new allocation after mouse-down. + setLocation(release, snapshotWindowPoint(release.location, in: liveBounds() ?? bounds)) + } + post(release) + } try verify(start) post(down) do { for point in points { Thread.sleep(forTimeInterval: stepDelay) try verify(point) - let move = try mouse(type: .leftMouseDragged, point: point, clickCount: 1) - let nextRelease = try mouse(type: .leftMouseUp, point: point, clickCount: 1) + let frame = liveBounds.map { $0() ?? bounds } + let move = try mouse(type: .leftMouseDragged, point: point, clickCount: 1, frame: frame) + let nextRelease = try mouse(type: .leftMouseUp, point: point, clickCount: 1, frame: frame) post(move) release = nextRelease } } catch { - post(release) + postRelease() // A takeover keeps its own type, so the caller reports it as the terminal refusal it is. if error is UserTakeoverError { throw error } throw WindowEventError.unavailable("drag interrupted after mouse-down: \(error.localizedDescription); inspect the partial outcome") } - post(release) + postRelease() } public func scroll(point: CGPoint, deltaX: Int32, deltaY: Int32) throws -> CGEvent { @@ -129,21 +147,24 @@ public final class WindowEventFactory { return event } - public func mouse(type: NSEvent.EventType, point: CGPoint, clickCount: Int) throws -> CGEvent { + /// `frame` nil: the point must lie in the snapshot window and is located against it. A frame: the window's bounds + /// right now, for a step of an `--allow-outside` drag; the point may lie anywhere. + public func mouse(type: NSEvent.EventType, point: CGPoint, clickCount: Int, frame: CGRect? = nil) throws -> CGEvent { // Split from the bounds check on purpose: Int(Double.nan) traps, so a non-finite point must // never reach the formatter that describes the rectangle. guard point.x.isFinite, point.y.isFinite else { throw WindowEventError.unavailable("mouse point \(point.x),\(point.y) is not a finite coordinate") } - guard bounds.contains(point) else { + let window = frame ?? bounds + guard frame != nil || bounds.contains(point) else { let received = "\(Int(point.x.rounded())),\(Int(point.y.rounded()))" let rectangle = "\(Int(bounds.origin.x.rounded())),\(Int(bounds.origin.y.rounded())) " + "\(Int(bounds.width.rounded()))x\(Int(bounds.height.rounded()))" throw WindowEventError.unavailable( "mouse point \(received) is outside snapshot window \(windowID), whose bounds are \(rectangle)") } - let local = NSPoint(x: point.x - bounds.minX, y: bounds.maxY - point.y) + let local = NSPoint(x: point.x - window.minX, y: window.maxY - point.y) guard let event = NSEvent.mouseEvent(with: type, location: local, modifierFlags: [], timestamp: ProcessInfo.processInfo.systemUptime, windowNumber: windowID, context: nil, eventNumber: 0, clickCount: clickCount, pressure: 0)?.cgEvent else { @@ -151,7 +172,7 @@ public final class WindowEventFactory { } // Setting the global position clears the private window-local field. event.location = point - setLocation(event, snapshotWindowPoint(point, in: bounds)) + setLocation(event, snapshotWindowPoint(point, in: window)) event.setIntegerValueField(.mouseEventClickState, value: Int64(clickCount)) event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(windowID)) event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(windowID)) diff --git a/native/ax-tool/SnapshotSupport/WorkflowArguments.swift b/native/ax-tool/SnapshotSupport/WorkflowArguments.swift index 786524e0e4..ea01ea6f63 100644 --- a/native/ax-tool/SnapshotSupport/WorkflowArguments.swift +++ b/native/ax-tool/SnapshotSupport/WorkflowArguments.swift @@ -15,6 +15,13 @@ public struct WorkflowArguments { public let values: [String: String] public let flags: Set + /// Outside geometry must not bypass the chrome token's omitted-web-content boundary. + public static func validateDragScope(scope: String, allowOutside: Bool) throws { + guard !allowOutside || scope != "chrome" else { + throw WorkflowArgumentError.invalid("--allow-outside requires window scope; chrome scope omits web content") + } + } + public init(_ args: [String], command: String) throws { let valueOptions: Set let flagOptions: Set @@ -30,7 +37,7 @@ public struct WorkflowArguments { "--revalidate-scope", "--frame", "--by-identifier", "--window-index", "--depth", "--modifiers", "--budget-ms", "--target-ordinal", "--expect-value-sha256", ] - flagOptions = ["--background", "--double", "--refresh", "--no-cursor", "--no-image", "--prepare", "--replace", "--hold", "--no-activate"] + flagOptions = ["--background", "--allow-outside", "--double", "--refresh", "--no-cursor", "--no-image", "--prepare", "--replace", "--hold", "--no-activate"] default: throw WorkflowArgumentError.invalid("unknown workflow command \(command)") } @@ -184,6 +191,7 @@ public struct WorkflowArguments { // hover deliberately excluded from --background: the whole point is to move the REAL // pointer, and a window-addressed event does not. try reject(["--background"], unless: ["click", "move", "drag", "scroll"]) + try reject(["--allow-outside"], unless: ["drag"]) try reject(["--coords", "--region"], unless: ["click", "move", "drag", "scroll", "hover"]) try reject(["--frame"], unless: ["click", "move", "drag", "scroll", "hover"]) if let frame = values["--frame"] { diff --git a/native/ax-tool/Sources/IsolatedCapture.swift b/native/ax-tool/Sources/IsolatedCapture.swift index 3a240848a0..767c4a1175 100644 --- a/native/ax-tool/Sources/IsolatedCapture.swift +++ b/native/ax-tool/Sources/IsolatedCapture.swift @@ -512,7 +512,9 @@ func captureIsolatedWindows(_ options: RecordOptions) { compositor.updateSizes() } let indicator = RecordingIndicator(enabled: options.selection.indicator) - indicator.update(displayCanvas.map { [$0] } ?? selected.map(\.bounds)) + // The border marks what the movie holds: the selected windows, also on a display canvas, where framing the whole + // display read as "the whole screen is recorded" (Martin, 2026-10-14). + indicator.update(selected.map(\.bounds)) defer { indicator.close() } var startedStreams: [SCStream] = [] do { @@ -541,7 +543,7 @@ func captureIsolatedWindows(_ options: RecordOptions) { ($0[kCGWindowOwnerPID] as? pid_t) == target.pid }) else { return nil } return captureWindowBounds(row) } - indicator.update(displayCanvas.map { [$0] } ?? bounds) + indicator.update(bounds) compositor.refresh(rows) return compositor.queue.sync { compositor.error == nil } } diff --git a/native/ax-tool/Sources/SnapshotWorkflow.swift b/native/ax-tool/Sources/SnapshotWorkflow.swift index 1722b9e5d0..a9ec87a898 100644 --- a/native/ax-tool/Sources/SnapshotWorkflow.swift +++ b/native/ax-tool/Sources/SnapshotWorkflow.swift @@ -96,6 +96,20 @@ private func describeOutsideWindow(point: CGPoint, window: ObservedWindow) -> St + "both are global logical points, and a window that moved since see needs a fresh see" } +/// Where a window is right now, while it is on screen and still owned by `pid`; nil once it closed, hid or changed owner. +/// An `--allow-outside` drag reads it at every step: the window may move with the drag, but it must stay this window. +private func onscreenWindowBounds(_ id: CGWindowID, pid: pid_t) -> CGRect? { + let listed = CGWindowListCopyWindowInfo(.optionIncludingWindow, id) as? [[CFString: Any]] ?? [] + for info in listed { + guard (info[kCGWindowNumber] as? CGWindowID) == id, (info[kCGWindowOwnerPID] as? pid_t) == pid, + (info[kCGWindowIsOnscreen] as? Bool) == true, + let raw = info[kCGWindowBounds] as? NSDictionary, + let bounds = CGRect(dictionaryRepresentation: raw) else { continue } + return bounds + } + return nil +} + private var workflowInput: WorkflowArguments? /// The caller's deadline (`--budget-ms`), less a margin for printing the result and exiting. @@ -1431,7 +1445,7 @@ func cmdAct(appName _: String) { // error — just a click in the wrong place. A caller who wants a point that survives the // window moving opts in with --frame window, and that is the one this documents. let coordinateFrame = workflowArgument("--frame") ?? "screen" - func parsePoint(_ raw: String) throws -> CGPoint { + func parsePoint(_ raw: String, requireInside: Bool = true) throws -> CGPoint { let parts = raw.split(separator: ",", omittingEmptySubsequences: false) let numbers = parts.compactMap { Double($0.trimmingCharacters(in: .whitespaces)) } guard parts.count == 2, numbers.count == 2, numbers.allSatisfy({ $0.isFinite }) else { @@ -1444,7 +1458,7 @@ func cmdAct(appName _: String) { let point = coordinateFrame == "window" ? CGPoint(x: window.bounds.origin.x + given.x, y: window.bounds.origin.y + given.y) : given - guard window.bounds.contains(point) else { + guard !requireInside || window.bounds.contains(point) else { throw WindowEventError.unavailable(describeOutsideWindow(point: point, window: window)) } @@ -1616,7 +1630,7 @@ func cmdAct(appName _: String) { guard rawCoords != nil || row["visible"] as? Bool == true || visiblePart != nil else { throw WindowEventError.unavailable("element is outside its window/scroll clip") } - let point = try rawCoords.map(parsePoint) ?? visiblePart ?? CGPoint(x: frame.midX, y: frame.midY) + let point = try rawCoords.map { try parsePoint($0) } ?? visiblePart ?? CGPoint(x: frame.midX, y: frame.midY) // A caller that passed a window-relative point cannot otherwise tell WHERE it landed, // and that is the number to compare against a screenshot or a later observation. if let raw = rawCoords { @@ -1725,10 +1739,17 @@ func cmdAct(appName _: String) { Thread.sleep(forTimeInterval: 0.1) } else if action == "drag" { guard let destination = workflowArgument("--to") else { - throw WindowEventError.unavailable("drag requires --to x,y in the snapshot window") + throw WindowEventError.unavailable("drag requires --to x,y in the snapshot window, or anywhere with --allow-outside") + } + // --allow-outside: the destination and the path may leave the window, and the window may move with + // the drag (a panel dragged by its body). The start keeps every check below; each later step checks + // that the user has not taken the pointer and that the window is still this window, on screen. + let allowOutside = workflowFlag("--allow-outside") + try WorkflowArguments.validateDragScope(scope: token.effectiveScope, allowOutside: allowOutside) + let end = try parsePoint(destination, requireInside: !allowOutside) + if !allowOutside { + _ = try verifyPoint(end, pin: .window) } - let end = try parsePoint(destination) - _ = try verifyPoint(end, pin: .window) let duration = Double(workflowArgument("--duration") ?? "0.3") ?? 0 guard duration.isFinite, (0.1...5).contains(duration) else { throw WindowEventError.unavailable("--duration must be 0.1–5 seconds") @@ -1742,16 +1763,29 @@ func cmdAct(appName _: String) { try validateAfterFeedback() // The takeover check runs in verify, which precedes every post but the closing // release; the factory posts that release itself when verify throws. - try factory.drag(start: point, points: points, stepDelay: duration / Double(steps), + let liveBounds: (() -> CGRect?)? = allowOutside ? { onscreenWindowBounds(window.id, pid: pid) } : nil + try factory.drag(start: point, points: points, stepDelay: duration / Double(steps), liveBounds: liveBounds, verify: { try inputGate.check() - _ = try verifyPoint($0, pin: WindowEventFactory.dragVerifyTarget(point: $0, start: point)) + if allowOutside && $0 != point { + guard onscreenWindowBounds(window.id, pid: pid) != nil else { + throw WindowEventError.unavailable("window \(window.id) closed or left the screen during the drag") + } + } else { + _ = try verifyPoint($0, pin: WindowEventFactory.dragVerifyTarget(point: $0, start: point)) + } try admitVisualAction() }, post: { event in ActionCursor.emit("drag", point: event.location, background: background, target: rawCoords == nil ? "ax" : "pixel", waitForPresentation: false) inputGate.deliver { event.postToPid(pid) } }) Thread.sleep(forTimeInterval: 0.05) + if allowOutside { + // Nothing checked what lay under the path, so say where the window ended: a drag that moved it + // shows here, and a caller compares it with the bounds it saw in see. + let after = onscreenWindowBounds(window.id, pid: pid) + actionExtras["windowBoundsAfter"] = after.map { ["x": $0.minX, "y": $0.minY, "width": $0.width, "height": $0.height] } ?? NSNull() + } } else if secondaryClick { // 🛑 A right-click is delivered in the background three ways, in this order, and each later way // runs only while the app shows no menu at all: the NSEvent-built window event, a plain CGEvent diff --git a/native/ax-tool/Tests/WindowEventTests.swift b/native/ax-tool/Tests/WindowEventTests.swift index 3ea2ef8ec5..e0fdadbc60 100644 --- a/native/ax-tool/Tests/WindowEventTests.swift +++ b/native/ax-tool/Tests/WindowEventTests.swift @@ -163,4 +163,88 @@ final class WindowEventTests: XCTestCase { XCTAssertEqual(event.type, .leftMouseDown) XCTAssertEqual(event.location, CGPoint(x: -100, y: -500)) } + + private func windowLocation(_ event: CGEvent) throws -> CGPoint { + typealias GetLocation = @convention(c) (CGEvent) -> CGPoint + let handle = try XCTUnwrap(dlopen("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", RTLD_LAZY)) + defer { dlclose(handle) } + let symbol = try XCTUnwrap(dlsym(handle, "CGEventGetWindowLocation")) + return unsafeBitCast(symbol, to: GetLocation.self)(event) + } + + func testDragOutsideTheWindowLocatesEachStepAgainstWhereTheWindowIsNow() throws { + // A panel dragged by its body moves with the drag. Each step's window-local location must be + // measured from where the window is at that step, or an app that converts it to screen + // coordinates sees the pointer off by the distance the window already moved. + let factory = try WindowEventFactory(windowID: 456, bounds: CGRect(x: 100, y: 100, width: 500, height: 452)) + var moved = CGRect(x: 100, y: 100, width: 500, height: 452) + var posted: [CGEvent] = [] + try factory.drag(start: CGPoint(x: 150, y: 150), points: [CGPoint(x: 400, y: 150), CGPoint(x: 900, y: 150)], + stepDelay: 0, liveBounds: { moved }, + verify: { point in moved.origin.x = 100 + (point.x - 150) }, + post: { posted.append($0) }) + + XCTAssertEqual(posted.map(\.type), [.leftMouseDown, .leftMouseDragged, .leftMouseDragged, .leftMouseUp]) + XCTAssertEqual(posted.map(\.location), [CGPoint(x: 150, y: 150), CGPoint(x: 400, y: 150), + CGPoint(x: 900, y: 150), CGPoint(x: 900, y: 150)]) + // The grab point stays 50,50 inside the window all the way, as it does under a real hand. + XCTAssertEqual(try posted.map(windowLocation), Array(repeating: CGPoint(x: 50, y: 50), count: 4)) + XCTAssertEqual(try XCTUnwrap(NSEvent(cgEvent: posted[2])).windowNumber, 456) + } + + func testCompletedDragReleaseUsesTheFrameMovedByPostingTheLastDrag() throws { + try assertDragReleaseUsesPostMoveFrame(interrupted: false) + } + + func testInterruptedDragReleaseUsesTheFrameMovedByPostingTheLastDrag() throws { + try assertDragReleaseUsesPostMoveFrame(interrupted: true) + } + + private func assertDragReleaseUsesPostMoveFrame(interrupted: Bool) throws { + let initial = CGRect(x: 100, y: 100, width: 500, height: 452) + let factory = try WindowEventFactory(windowID: 456, bounds: initial) + var moved = initial + var posted: [CGEvent] = [] + let action = { + try factory.drag(start: CGPoint(x: 150, y: 150), + points: [CGPoint(x: 250, y: 150), CGPoint(x: 350, y: 150)], + stepDelay: 0, liveBounds: { moved }, + verify: { point in + if interrupted && point.x == 350 { + throw WindowEventError.unavailable("window changed") + } + }, post: { event in + posted.append(event) + // Delivery moves the window; verification does not mutate its frame. + if event.type == .leftMouseDragged { + moved.origin.x = event.location.x - 50 + } + }) + } + if interrupted { + XCTAssertThrowsError(try action()) + } else { + try action() + } + let release = try XCTUnwrap(posted.last) + XCTAssertEqual(release.type, .leftMouseUp) + XCTAssertEqual(posted.filter { $0.type == .leftMouseUp }.count, 1) + XCTAssertEqual(release.location, CGPoint(x: interrupted ? 250 : 350, y: 150)) + XCTAssertEqual(try windowLocation(release), CGPoint(x: 50, y: 50)) + XCTAssertEqual(try XCTUnwrap(NSEvent(cgEvent: release)).windowNumber, 456) + } + + func testDragWithoutLiveBoundsStillStopsAtTheWindowEdge() throws { + // The default drag keeps its old contract: a step outside the snapshot window is refused and + // the button is released where the drag last was. + let factory = try WindowEventFactory(windowID: 456, bounds: CGRect(x: 0, y: 0, width: 500, height: 500)) + var posted: [CGEvent] = [] + XCTAssertThrowsError(try factory.drag(start: CGPoint(x: 10, y: 10), + points: [CGPoint(x: 400, y: 10), CGPoint(x: 700, y: 10)], stepDelay: 0, + verify: { _ in }, post: { posted.append($0) })) { error in + XCTAssertTrue(error.localizedDescription.contains("outside snapshot window"), error.localizedDescription) + } + XCTAssertEqual(posted.map(\.type), [.leftMouseDown, .leftMouseDragged, .leftMouseUp]) + XCTAssertEqual(posted.last?.location, CGPoint(x: 400, y: 10)) + } } diff --git a/native/ax-tool/Tests/WorkflowArgumentsTests.swift b/native/ax-tool/Tests/WorkflowArgumentsTests.swift index c133266d2b..0103e801be 100644 --- a/native/ax-tool/Tests/WorkflowArgumentsTests.swift +++ b/native/ax-tool/Tests/WorkflowArgumentsTests.swift @@ -2,6 +2,15 @@ import XCTest @testable import SnapshotSupport final class WorkflowArgumentsTests: XCTestCase { + func testChromeScopeRefusesOutsideToolbarToPageDragsBeforeDispatch() throws { + XCTAssertThrowsError(try WorkflowArguments.validateDragScope(scope: "chrome", allowOutside: true)) { error in + XCTAssertEqual(error.localizedDescription, + "--allow-outside requires window scope; chrome scope omits web content") + } + XCTAssertNoThrow(try WorkflowArguments.validateDragScope(scope: "chrome", allowOutside: false)) + XCTAssertNoThrow(try WorkflowArguments.validateDragScope(scope: "window", allowOutside: true)) + } + // Regression test: #447 D3 — see had no way to return a tree deeper than --depth func testSeeTakesTruncate() throws { let arguments = try WorkflowArguments(["--app", "Claude", "--truncate"], command: "see") @@ -104,6 +113,22 @@ final class WorkflowArgumentsTests: XCTestCase { XCTAssertThrowsError(try WorkflowArguments(base, command: "act"), "An unpinned index cannot bypass full-window validation") } + func testOnlyADragMayLeaveTheWindow() throws { + // The parser keeps a list of flags per command; a flag missing from it is refused as an unknown + // option before anything runs, which is how the first --allow-outside build failed live. + let drag = try WorkflowArguments([ + "--app", "Genesis", "--snapshot", "token", "--action", "drag", "--coords", "1676,1249", + "--to", "1100,1249", "--duration", "0.4", "--background", "--allow-outside", + ], command: "act") + XCTAssertTrue(drag.flags.contains("--allow-outside")) + + XCTAssertThrowsError(try WorkflowArguments([ + "--app", "Genesis", "--snapshot", "token", "--action", "click", "--coords", "1676,1249", "--allow-outside", + ], command: "act")) { error in + XCTAssertEqual(error.localizedDescription, "option is not valid for click") + } + } + func testRevalidateScopeOffersOnlyTheScopesThatAreImplemented() { // `app` was accepted and behaved exactly like `window`: no app-wide check existed. let base = ["--app", "Fixture", "--snapshot", "token", "--element", "0", "--action", "press"] diff --git a/scripts/ai-proxy/structured-output.ts b/scripts/ai-proxy/structured-output.ts index 712c388840..c6dc08e979 100644 --- a/scripts/ai-proxy/structured-output.ts +++ b/scripts/ai-proxy/structured-output.ts @@ -11,11 +11,11 @@ */ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import { AiProxyClient, type SchemaMode } from "@genesiscz/utils/ai/proxy/AiProxyClient"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + const SCHEMA = { name: "fruit_pick", schema: { @@ -33,7 +33,7 @@ const SCHEMA = { const PROMPT = "Pick the fruit that is a berry botanically-speaking from: apple, banana, cherry."; function lastUsageRecord(model: string): unknown { - const path = join(env.tools.getHome(), ".genesis-tools", "ai-proxy", "usage", "requests.jsonl"); + const path = genesisToolsDir("ai-proxy", "usage", "requests.jsonl"); if (!existsSync(path)) { return "NO requests.jsonl"; } diff --git a/scripts/backfill-product-units.ts b/scripts/backfill-product-units.ts index 77be66ed5a..dda87b7ab2 100644 --- a/scripts/backfill-product-units.ts +++ b/scripts/backfill-product-units.ts @@ -10,15 +10,14 @@ * Usage: bun scripts/backfill-product-units.ts [--dry-run] */ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; import { ShopsDatabase } from "../src/shops/db/ShopsDatabase"; import { refreshMasterDenorm } from "../src/shops/lib/master-denorm"; import { extractFlavorKey, extractPackCount, extractSize, parseUnit, type Unit } from "../src/shops/lib/normalize"; const DRY_RUN = process.argv.includes("--dry-run"); -const DB_PATH = join(env.tools.getHome(), ".genesis-tools/shops/index.db"); +const DB_PATH = toolDataDir("shops", "index.db"); interface Row { id: number; diff --git a/scripts/benchmarks/agent-sessions/session-listing.ts b/scripts/benchmarks/agent-sessions/session-listing.ts index 8946ecc113..b614655189 100644 --- a/scripts/benchmarks/agent-sessions/session-listing.ts +++ b/scripts/benchmarks/agent-sessions/session-listing.ts @@ -27,6 +27,7 @@ import { dirname, join } from "node:path"; import type { BaselineMetrics } from "@app/benchmark/lib"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { finishBench, parseBenchArgs } from "../fs/shared"; const BASELINE_NAME = "agent-sessions-listing"; @@ -36,7 +37,7 @@ const LISTING = ["usage", "sessions", "--json", "--hours", "24", "--min", "10"]; const QUERY = { hours: 24, minRows: 10 }; const SCRATCH_HOME = mkdtempSync(join(tmpdir(), "gt-bench-listing-")); /** Where `sessionRowsCachePath()` points under `GENESIS_TOOLS_HOME=SCRATCH_HOME`. */ -const SCRATCH_CACHE = join(SCRATCH_HOME, ".genesis-tools", "ai", "usage-sessions.json"); +const SCRATCH_CACHE = genesisToolsDirIn(SCRATCH_HOME, "ai", "usage-sessions.json"); function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; diff --git a/scripts/benchmarks/ai-usage/poll-duty.ts b/scripts/benchmarks/ai-usage/poll-duty.ts index 16486f74b1..145b02c5a1 100644 --- a/scripts/benchmarks/ai-usage/poll-duty.ts +++ b/scripts/benchmarks/ai-usage/poll-duty.ts @@ -1,4 +1,5 @@ #!/usr/bin/env bun + /** * What the `ai-usage-poll` daemon costs, read back out of the day logs. * @@ -28,12 +29,11 @@ * were produced with the two landing times of cfc1dc0b7 and bb07e6fa9). */ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { readLinesSync } from "@genesiscz/utils/fs/read-lines"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; import { dayLogPath, listDayLogs } from "@genesiscz/utils/logger/day-logs"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { createBoxTable, renderCliHeader } from "@genesiscz/utils/table"; interface Poll { @@ -234,7 +234,7 @@ if (badSplit !== null) { const days = rangeFrom && rangeTo ? expandRange(rangeFrom, rangeTo) : positional; // Same construction as `createLogger` in src/utils/logger.ts:196. `env.tools.getHome()` // is the USER home, not the tool home, so the `.genesis-tools` segment is not optional. - const logDir = join(env.tools.getHome(), ".genesis-tools", "logs"); + const logDir = genesisToolsDir("logs"); const wanted = days.length > 0 ? days : discoverDays(logDir); const summaries: Summary[] = []; diff --git a/scripts/benchmarks/compress/arms.ts b/scripts/benchmarks/compress/arms.ts index 84bc2e4090..02e8bc3974 100644 --- a/scripts/benchmarks/compress/arms.ts +++ b/scripts/benchmarks/compress/arms.ts @@ -20,7 +20,7 @@ import { join } from "node:path"; import { compressFile, decompressFile } from "@genesiscz/utils/compress/file"; import { PRESETS } from "@genesiscz/utils/compress/presets"; import { formatLocalDate } from "@genesiscz/utils/date"; -import { genesisToolsDir } from "@genesiscz/utils/storage/root"; +import { genesisToolsDir, genesisToolsDirIn } from "@genesiscz/utils/storage/root"; export const BENCH_ROOT = join(tmpdir(), "gt-bench-compress"); export const REAL_HOME = genesisToolsDir(); @@ -92,7 +92,7 @@ export interface ArmInfo { /** Builds (or reuses) one arm home. */ export async function buildArm(arm: Arm, { rebuild = false, now = new Date() } = {}): Promise { const home = armHome(arm); - const tools = join(home, ".genesis-tools"); + const tools = genesisToolsDirIn(home); const done = join(home, ".built"); if (!existsSync(done) || rebuild) { diff --git a/scripts/benchmarks/compress/read-paths.ts b/scripts/benchmarks/compress/read-paths.ts index 1083bd3606..2602719f2c 100644 --- a/scripts/benchmarks/compress/read-paths.ts +++ b/scripts/benchmarks/compress/read-paths.ts @@ -26,6 +26,7 @@ import type { BaselineMetrics } from "@app/benchmark/lib"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { createBoxTable, renderCliHeader } from "@genesiscz/utils/table"; import { finishBench, median, parseBenchArgs } from "../fs/shared"; import { ARMS, type Arm, armHome, benchDays, buildArm, TRANSCRIPTS } from "./arms"; @@ -58,7 +59,7 @@ function day(offset: number): string { } function logFile(arm: Arm, name: string): string { - const plain = join(armHome(arm), ".genesis-tools", "logs", name); + const plain = genesisToolsDirIn(armHome(arm), "logs", name); return arm === "zst" ? `${plain}.zst` : plain; } @@ -95,7 +96,7 @@ const SCENARIOS: Scenario[] = [ name: "tail-1000", command: (arm) => { const md = join(armHome(arm), "tail.md"); - const path = join(armHome(arm), ".genesis-tools", "logs", `${day(2)}.log`); + const path = genesisToolsDirIn(armHome(arm), "logs", `${day(2)}.log`); return { argv: [TOOLS, "markdown", "resolve", md], before: () => writeFileSync(md, `# tail\n\n{{tail path="${path}" n=1000}}\n`), diff --git a/scripts/benchmarks/dashboards/idle-cost.ts b/scripts/benchmarks/dashboards/idle-cost.ts index c8b0e23155..2de76025bb 100644 --- a/scripts/benchmarks/dashboards/idle-cost.ts +++ b/scripts/benchmarks/dashboards/idle-cost.ts @@ -58,6 +58,7 @@ import { logger, out } from "@genesiscz/utils/logger"; import { findFreePort } from "@genesiscz/utils/net/free-port"; import { PROJECT_ROOT } from "@genesiscz/utils/paths"; import { readProcessCommand } from "@genesiscz/utils/process-identity"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { createBoxTable, renderCliHeader, renderCliSection } from "@genesiscz/utils/table"; import { terminalLocaleEnvRecord } from "@genesiscz/utils/terminal/locale"; import type { Subprocess } from "bun"; @@ -249,7 +250,7 @@ interface RunDirs { function prepareRunDirs(mode: Mode): RunDirs { const root = mkdtempSync(join(tmpdir(), "dashboards-idle-cost-")); const home = join(root, "home"); - mkdirSync(join(home, ".genesis-tools", "dev-dashboard"), { recursive: true }); + mkdirSync(genesisToolsDirIn(home, "dev-dashboard"), { recursive: true }); return { root, home, @@ -265,7 +266,7 @@ function prepareRunDirs(mode: Mode): RunDirs { * app rather than a 401, and the port is the free one this run picked. */ async function writeSandboxConfig(home: string, port: number): Promise { - const path = join(home, ".genesis-tools", "dev-dashboard", "config.json"); + const path = genesisToolsDirIn(home, "dev-dashboard", "config.json"); const config = { port, allowedHosts: [], ttydSessions: [], auth: { enabled: false } }; await Bun.write(path, `${SafeJSON.stringify(config, null, 4)}\n`); return path; diff --git a/scripts/benchmarks/hooks/attribution.ts b/scripts/benchmarks/hooks/attribution.ts index 4614df45c9..403e9a46da 100644 --- a/scripts/benchmarks/hooks/attribution.ts +++ b/scripts/benchmarks/hooks/attribution.ts @@ -37,6 +37,7 @@ import { } from "@app/benchmark/lib"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { formatTable } from "@genesiscz/utils/table"; import { Command } from "commander"; @@ -72,9 +73,9 @@ const SCRATCH = realpathSync(mkdtempSync(join(tmpdir(), "gt-bench-attribution-") process.env.TMPDIR = join(SCRATCH, "tmp"); process.env.GENESIS_TOOLS_HOME = join(SCRATCH, "home"); mkdirSync(process.env.TMPDIR, { recursive: true }); -mkdirSync(join(process.env.GENESIS_TOOLS_HOME, ".genesis-tools", "agents"), { recursive: true }); +mkdirSync(genesisToolsDirIn(process.env.GENESIS_TOOLS_HOME, "agents"), { recursive: true }); writeFileSync( - join(process.env.GENESIS_TOOLS_HOME, ".genesis-tools", "agents", "hooks.json"), + genesisToolsDirIn(process.env.GENESIS_TOOLS_HOME, "agents", "hooks.json"), SafeJSON.stringify({ shadow: false, diff: { highlight: "none" } }) ); diff --git a/scripts/benchmarks/hooks/hook-latency.ts b/scripts/benchmarks/hooks/hook-latency.ts index c6dd640d0a..8d07554387 100644 --- a/scripts/benchmarks/hooks/hook-latency.ts +++ b/scripts/benchmarks/hooks/hook-latency.ts @@ -18,6 +18,8 @@ import { join } from "node:path"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; + const runsIndex = process.argv.indexOf("--runs"); const RUNS = runsIndex >= 0 ? Number(process.argv[runsIndex + 1]) : 20; @@ -284,24 +286,21 @@ function interleave(a: RunSpec, b: RunSpec, runs: number): { a: ArmResult; b: Ar function freshHome(prefix: string): string { const home = mkdtempSync(join(tmpdir(), prefix)); - mkdirSync(join(home, ".genesis-tools", "agents"), { recursive: true }); + mkdirSync(genesisToolsDirIn(home, "agents"), { recursive: true }); // `highlight: "none"` keeps every arm below from paying an unrelated `bat` spawn (measured // elsewhere at tens of ms per changed file). Left at the default it would land on BOTH // sides of a pair and add noise, not signal, to the sink's own delta. - writeFileSync( - join(home, ".genesis-tools", "agents", "hooks.json"), - SafeJSON.stringify({ diff: { highlight: "none" } }) - ); + writeFileSync(genesisToolsDirIn(home, "agents", "hooks.json"), SafeJSON.stringify({ diff: { highlight: "none" } })); return home; } function changesPath(home: string, session: string): string { - return join(home, ".genesis-tools", "agents", session, "changes.jsonl"); + return genesisToolsDirIn(home, "agents", session, "changes.jsonl"); } function agentsSessionDirs(home: string): string[] { - const dir = join(home, ".genesis-tools", "agents"); + const dir = genesisToolsDirIn(home, "agents"); if (!existsSync(dir)) { return []; @@ -363,7 +362,7 @@ function guardHome(prefix: string, customRules: string[]): string { const home = freshHome(prefix); writeFileSync( - join(home, ".genesis-tools", "agents", "hooks.json"), + genesisToolsDirIn(home, "agents", "hooks.json"), SafeJSON.stringify({ shadow: false, diff: { enabled: false }, guard: { customRules } }) ); diff --git a/scripts/benchmarks/hooks/status-ab.ts b/scripts/benchmarks/hooks/status-ab.ts index 2d5447696c..5b9a65eeef 100644 --- a/scripts/benchmarks/hooks/status-ab.ts +++ b/scripts/benchmarks/hooks/status-ab.ts @@ -32,6 +32,7 @@ import { type BaselineMetrics, withSpawnCounter } from "@app/benchmark/lib"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { finishBench, parseBenchArgs } from "../fs/shared"; const args = parseBenchArgs(process.argv); @@ -58,9 +59,9 @@ const SANDBOX_HOME = join(SCRATCH, "home"); process.env.TMPDIR = SANDBOX_TMP; process.env.GENESIS_TOOLS_HOME = SANDBOX_HOME; mkdirSync(SANDBOX_TMP, { recursive: true }); -mkdirSync(join(SANDBOX_HOME, ".genesis-tools", "agents"), { recursive: true }); +mkdirSync(genesisToolsDirIn(SANDBOX_HOME, "agents"), { recursive: true }); writeFileSync( - join(SANDBOX_HOME, ".genesis-tools", "agents", "hooks.json"), + genesisToolsDirIn(SANDBOX_HOME, "agents", "hooks.json"), SafeJSON.stringify({ shadow: false, diff: { highlight: "none" } }) ); diff --git a/scripts/benchmarks/polls/agents-bridge-register.ts b/scripts/benchmarks/polls/agents-bridge-register.ts index 5b4bfe01f8..b4644d42b6 100644 --- a/scripts/benchmarks/polls/agents-bridge-register.ts +++ b/scripts/benchmarks/polls/agents-bridge-register.ts @@ -33,6 +33,7 @@ import { type BaselineMetrics, sampleSelf, withSpawnCounter } from "@app/benchma import { CliAgentsTransport } from "@app/codex/lib/agents-bridge"; import { env } from "@genesiscz/utils/env"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { Command } from "commander"; import { addCommonOptions, type CommonFlags, runPollBenchmark } from "./harness"; @@ -56,7 +57,7 @@ function createReadOnlySession(): SessionFixture { env.testing.set("GENESIS_TOOLS_HOME", home); const session = `bench-${Date.now()}`; - const sessionDir = join(home, ".genesis-tools", "agents", session); + const sessionDir = genesisToolsDirIn(home, "agents", session); mkdirSync(join(sessionDir, "slots"), { recursive: true }); writeFileSync(join(sessionDir, "feed.jsonl"), ""); chmodSync(join(sessionDir, "slots"), 0o500); diff --git a/scripts/benchmarks/polls/agents-request-wait.ts b/scripts/benchmarks/polls/agents-request-wait.ts index c6ecb517c1..bb22650f85 100644 --- a/scripts/benchmarks/polls/agents-request-wait.ts +++ b/scripts/benchmarks/polls/agents-request-wait.ts @@ -34,6 +34,7 @@ import { type BaselineMetrics, sampleSelf } from "@app/benchmark/lib"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { out } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { Command } from "commander"; import { addCommonOptions, type CommonFlags, runPollBenchmark } from "./harness"; @@ -54,7 +55,7 @@ function createFeed(lines: number): Fixture { env.testing.set("GENESIS_TOOLS_HOME", home); const session = `bench-${Date.now()}`; - const sessionDir = join(home, ".genesis-tools", "agents", session); + const sessionDir = genesisToolsDirIn(home, "agents", session); mkdirSync(join(sessionDir, "slots"), { recursive: true }); const ts = new Date().toISOString(); diff --git a/scripts/ci/harvest-placeholder-markers.ts b/scripts/ci/harvest-placeholder-markers.ts index 51c1b12893..80388b3580 100644 --- a/scripts/ci/harvest-placeholder-markers.ts +++ b/scripts/ci/harvest-placeholder-markers.ts @@ -6,13 +6,14 @@ */ import { existsSync, mkdirSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { dirname, join } from "node:path"; +import { dirname } from "node:path"; import { cacheDbPath } from "@app/ms-teams/lib/paths"; import { TeamsCache } from "@app/ms-teams/lib/store"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; const log = logger.scoped("placeholder-check").log; @@ -171,7 +172,7 @@ export function spliceAutoBlock(existing: string, autoLines: string[]): string { } export function markersPath(home = homedir()): string { - return join(home, ".genesis-tools/placeholder-check/markers.txt"); + return genesisToolsDirIn(home, "placeholder-check", "markers.txt"); } function peopleFromConversations(cache: TeamsCache): PersonLike[] { diff --git a/scripts/ci/lint-rules.test.ts b/scripts/ci/lint-rules.test.ts index 5fc135cd0b..3fea4167be 100644 --- a/scripts/ci/lint-rules.test.ts +++ b/scripts/ci/lint-rules.test.ts @@ -9,12 +9,8 @@ import { checkSource, pluginsDisabledFor } from "./lint-rules"; /** * These pin the behaviour of the six GritQL plugins this checker replaced. * - * Every POSITIVE case was verified to be flagged by the original plugins, and - * every NEGATIVE case to be left alone by them, before the plugins were removed - * from biome.json. The negatives matter more than the positives here: a first - * cut of this checker tested string literals without their context and - * condemned 317 lines of correct code, 158 of them the sanctioned - * `join(env.tools.getHome(), ".genesis-tools", …)`. + * Storage-path enforcement now requires the canonical root helper for every home expression, including + * env.tools.getHome(), so explicit sandbox roots and the user's root follow the same construction contract. */ function rules(source: string, file = "src/example.ts"): string[] { @@ -65,19 +61,39 @@ describe("no-hardcoded-user-paths", () => { }); }); -describe("no-homedir-genesis-tools", () => { +describe("no-hand-joined-genesis-tools", () => { test("flags the data directory reached through homedir()", () => { - expect(rules('const d = join(homedir(), ".genesis-tools", "daemon");')).toContain("no-homedir-genesis-tools"); + expect(rules('const d = join(homedir(), ".genesis-tools", "daemon");')).toContain( + "no-hand-joined-genesis-tools" + ); }); test("flags a template that builds the same path", () => { - expect(rules("const d = `${homedir()}/.genesis-tools/daemon`;")).toContain("no-homedir-genesis-tools"); + expect(rules("const d = `${homedir()}/.genesis-tools/daemon`;")).toContain("no-hand-joined-genesis-tools"); }); - test("LEAVES the sanctioned env.tools.getHome() form alone", () => { - // 158 sites use this. Flagging it would condemn the correct code and - // make the rule unusable. - expect(rules('const d = join(env.tools.getHome(), ".genesis-tools", "daemon");')).toEqual([]); + test("flags arbitrary homes and path function forms", () => { + for (const home of ["env.tools.getHome()", "env.paths.getHome()", "home", "resolve(home)"]) { + expect(rules(`const d = join(${home}, ".genesis-tools", "daemon");`)).toContain( + "no-hand-joined-genesis-tools" + ); + } + expect(rules('const d = path.resolve(home, ".genesis-tools/daemon");')).toContain( + "no-hand-joined-genesis-tools" + ); + expect(rules("const d = `${home}/.genesis-tools/daemon`;")).toContain("no-hand-joined-genesis-tools"); + }); + + test("allows helpers, the canonical implementation, tests, plugins and lookalike names", () => { + expect(rules('const d = genesisToolsDir("daemon");')).toEqual([]); + expect(rules('const d = genesisToolsDirIn(home, "daemon");')).toEqual([]); + const direct = 'const d = join(home, ".genesis-tools", "daemon");'; + expect(rules(direct, "src/utils/storage/root.ts")).toEqual([]); + expect(rules(direct, "plugins/example/index.ts")).toEqual([]); + expect(rules(direct, "src/example.test.ts")).toEqual([]); + expect(rules('const d = join(home, ".genesis-tools-backup");')).toEqual([]); + expect(rules(`// lint-rules-ignore: intentional fixture\n${direct}`)).toEqual([]); + expect(rules("const shell = '\"${HOME}/.genesis-tools/cmux\"';")).toEqual([]); }); test("LEAVES LaunchAgent plist names alone", () => { diff --git a/scripts/ci/lint-rules.ts b/scripts/ci/lint-rules.ts index 270dd630dd..276fc659c1 100644 --- a/scripts/ci/lint-rules.ts +++ b/scripts/ci/lint-rules.ts @@ -65,13 +65,34 @@ interface Rule { */ const HARDCODED_TMP = /^["'`]\/tmp\/.+/s; const HARDCODED_USER_PATH = /^["'`]\/Users\/[^/]+\//s; -/** A literal that IS a `.genesis-tools` path segment, or a template that builds one from homedir(). */ -const GENESIS_TOOLS_LITERAL = /^["'`]\.genesis-tools/s; -const HOMEDIR_GENESIS_TOOLS_TEMPLATE = /\$\{\s*homedir\(\)\s*\}\/\.genesis-tools/s; +/** An entire storage segment, including a suffix, but not a similarly named folder or plist. */ +const GENESIS_TOOLS_LITERAL = /^["'`]\.genesis-tools(?:[/\\]|["'`])/s; +const HAND_JOINED_GENESIS_TOOLS_TEMPLATE = /\$\{[^}]+\}[/\\]\.genesis-tools(?:[/\\]|`)/s; +const STORAGE_ROOT_ALLOWED = /(^|\/)(?:plugins\/|src\/utils\/storage\/root\.ts$)/; +const STORAGE_ROOT_RULE = { + name: "no-hand-joined-genesis-tools", + severity: "error" as const, + message: + "Construct storage paths with genesisToolsDir/toolDataDir from @genesiscz/utils/storage/root. " + + "Use genesisToolsDirIn for an explicit fixture or sandbox home.", +}; + +function recordHandJoinedStorage( + nodes: SgNode[], + filenameOf: (node: SgNode) => string, + record: (node: SgNode, rule: typeof STORAGE_ROOT_RULE) => void +): void { + for (const node of nodes) { + if (STORAGE_ROOT_ALLOWED.test(filenameOf(node))) { + continue; + } -const HOMEDIR_MESSAGE = - "`homedir()` bypasses the GENESIS_TOOLS_HOME test sandbox. Use `env.tools.getHome()` from " + - "@genesiscz/utils/env — it falls back to homedir(), so production is unchanged."; + const args = node.getMultipleMatches("ARGS"); + if (args.length > 1 && args.some((arg) => GENESIS_TOOLS_LITERAL.test(arg.text()))) { + record(node, STORAGE_ROOT_RULE); + } + } +} const LITERAL_KINDS = ["string", "template_string"]; @@ -79,6 +100,12 @@ const LITERAL_KINDS = ["string", "template_string"]; const GENESIS_ROUTER_LINK = /genesis\.tools\/|genesis-(?:md|tools):\/\//; const RULES: Rule[] = [ + { + ...STORAGE_ROOT_RULE, + kinds: LITERAL_KINDS, + test: (text) => text.startsWith("`") && HAND_JOINED_GENESIS_TOOLS_TEMPLATE.test(text), + allowedIn: STORAGE_ROOT_ALLOWED, + }, { name: "no-hardcoded-tmp", severity: "error", @@ -111,9 +138,6 @@ const RULES: Rule[] = [ }, ]; -/** Shared so `checkSource` (one file) and `scanAll` (whole repo) cannot drift. */ -const HOMEDIR_RULE = { name: "no-homedir-genesis-tools", severity: "error" as const, message: HOMEDIR_MESSAGE }; - const PROMPT_BACKEND_RULE = { name: "no-direct-prompt-backend", severity: "error" as const, @@ -743,29 +767,12 @@ export function checkSource(file: string, source: string): Finding[] { record(node, rule); } } - - if (HOMEDIR_GENESIS_TOOLS_TEMPLATE.test(text)) { - record(node, HOMEDIR_RULE); - } } recordBusyWaitFindings(root, record); recordProcessToolSpawns(root.findAll(PROCESS_TOOL_CALL_MATCHER), () => file, record); - // The `.genesis-tools` rule is about REACHING that directory through - // `homedir()`, not about the literal. `join(env.tools.getHome(), - // ".genesis-tools", …)` is the sanctioned form and appears 158 times, so a - // rule that only looked at the literal would condemn the correct code. - for (const node of root.findAll({ rule: { pattern: "$FN(homedir(), $$$ARGS)" } })) { - // Start-anchored per argument, exactly as the GritQL rule was. A loose - // `includes(".genesis-tools")` also matches the LaunchAgent plist names - // (`com.genesis-tools.daemon.plist`), which are not the data directory. - const violating = node.getMultipleMatches("ARGS").some((arg) => GENESIS_TOOLS_LITERAL.test(arg.text())); - - if (violating) { - record(node, HOMEDIR_RULE); - } - } + recordHandJoinedStorage(root.findAll({ rule: { pattern: "$FN($$$ARGS)" } }), () => file, record); for (const node of root.findAll({ rule: { pattern: "$OBJ.$METHOD($$$ARGS)" } })) { const object = node.getMatch("OBJ")?.text(); @@ -979,10 +986,6 @@ async function scanAll(files: string[], options: { processToolOnly?: boolean } = push(node, rule); } } - - if (HOMEDIR_GENESIS_TOOLS_TEMPLATE.test(text)) { - push(node, HOMEDIR_RULE); - } } }) ), @@ -1003,13 +1006,9 @@ async function scanAll(files: string[], options: { processToolOnly?: boolean } = processToolPass, findInFiles( lang, - { paths, matcher: { rule: { pattern: "$FN(homedir(), $$$ARGS)" } } }, + { paths, matcher: { rule: { pattern: "$FN($$$ARGS)" } } }, guard((nodes) => { - for (const node of nodes) { - if (node.getMultipleMatches("ARGS").some((arg) => GENESIS_TOOLS_LITERAL.test(arg.text()))) { - push(node, HOMEDIR_RULE); - } - } + recordHandJoinedStorage(nodes, (node) => node.getRoot().filename(), push); }) ), findInFiles( diff --git a/scripts/history/benchmark.ts b/scripts/history/benchmark.ts index 6b36a23edb..30d5b79fae 100644 --- a/scripts/history/benchmark.ts +++ b/scripts/history/benchmark.ts @@ -16,6 +16,7 @@ import { generateHistoryCorpus, type HistoryCorpusManifest } from "@genesiscz/ut import { createFixtureWorld, type HistoryFixtureWorld } from "@genesiscz/utils/agent-sessions/testing/fixture-world"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; export type BenchmarkOperation = | "metadata-list" | "summary-search" @@ -884,7 +885,7 @@ export function createBaselineBenchmarkVariant(options: { const oracle = await createBaselineOracle({ world: options.world, cacheHome: context.cachePath }); invocationOracle = oracle; manifest = oracle.manifest; - storagePaths.add(join(context.cachePath, ".genesis-tools", "claude-history", "index.db")); + storagePaths.add(genesisToolsDirIn(context.cachePath, "claude-history", "index.db")); return { initializationDetails: { ...oracle.startupMeasurement }, execute: (operation) => executeBaseline(oracle, operation, options.corpus), diff --git a/scripts/history/retire-native-cache.ts b/scripts/history/retire-native-cache.ts index 9fdd90b5fc..9a263334de 100644 --- a/scripts/history/retire-native-cache.ts +++ b/scripts/history/retire-native-cache.ts @@ -19,6 +19,7 @@ import { dirname, join, resolve } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { openFiles } from "@genesiscz/utils/process/open-files"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; const EXPECTED_TABLES = [ "_migrations", @@ -48,7 +49,7 @@ export interface NativeCacheRetirementReport { } function retiredTarget(toolsHome: string): string { - return join(resolve(toolsHome), ".genesis-tools", "native-history"); + return genesisToolsDirIn(resolve(toolsHome), "native-history"); } function databaseSchema(path: string): { tables: string[]; migrationIds: string[] } { @@ -163,7 +164,7 @@ export function retireNativeCache(options: { const stamp = new Date().toISOString().replace(/[:.]/g, "-"); const manifestDirectory = options.manifestDirectory ?? - join(resolve(options.toolsHome ?? env.tools.getHome()), ".genesis-tools", "history-retirement"); + genesisToolsDirIn(resolve(options.toolsHome ?? env.tools.getHome()), "history-retirement"); const manifestRoot = resolve(manifestDirectory); if (manifestRoot === report.target || manifestRoot.startsWith(`${report.target}/`)) { throw new Error("Retirement manifest must be outside the directory being deleted"); diff --git a/scripts/session-changes-compare.ts b/scripts/session-changes-compare.ts index 62abe9baed..247215fd7d 100644 --- a/scripts/session-changes-compare.ts +++ b/scripts/session-changes-compare.ts @@ -17,7 +17,6 @@ import { join } from "node:path"; import { rawLogFiles } from "@app/agents/commands/changes"; import { type ChangeEvent, sessionChangesPath } from "@app/agents/lib/changes/log"; import { runTool } from "@genesiscz/utils/cli"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { readJsonlRows } from "@genesiscz/utils/jsonl"; import { out } from "@genesiscz/utils/logger"; @@ -27,6 +26,7 @@ import { loadSessionChanges, type TurnChanges, } from "@genesiscz/utils/session-changes"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { Command } from "commander"; interface Options { @@ -63,7 +63,7 @@ interface SessionReport { } function recentSessions(count: number): string[] { - const dir = join(env.tools.getHome(), ".genesis-tools", "agents"); + const dir = genesisToolsDir("agents"); const found: { id: string; mtime: number }[] = []; for (const id of readdirSync(dir)) { diff --git a/scripts/tools-entrypoint.test.ts b/scripts/tools-entrypoint.test.ts index 809583699d..b76f51d27e 100644 --- a/scripts/tools-entrypoint.test.ts +++ b/scripts/tools-entrypoint.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "bun:test"; import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; +import { skipWorktreeInstall } from "@genesiscz/utils/cli/worktree-install"; +import { shellQuote } from "@genesiscz/utils/shell/quote"; /** * Guards the `tools` dispatcher at the repo root. @@ -116,3 +120,71 @@ describe("native widget dispatcher", () => { expect(invalid.stderr).toContain("--fps"); }); }); + +describe("linked worktree automatic installation", () => { + it("prints a shell-safe repair command for workspace paths with metacharacters", () => { + const workspaceRoot = mkdtempSync(join(tmpdir(), "gt repair '$literal; ")); + try { + const source = readFileSync(join(ROOT, "tools"), "utf8"); + const hint = source.split("\n").find((line) => line.includes("repair it with: cd ")); + expect(hint).toBeDefined(); + const template = hint!.slice(hint!.indexOf("`"), hint!.lastIndexOf("`") + 1); + + const render = new Function("workspaceRoot", "shellQuote", `return ${template}`); + const command = render(workspaceRoot, shellQuote).match(/repair it with: (.*) && bun install/)[1]; + const result = spawnSync("/bin/sh", ["-c", `${command} && pwd -P`], { encoding: "utf8", env: process.env }); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe(realpathSync(workspaceRoot)); + } finally { + rmSync(workspaceRoot, { recursive: true, force: true }); + } + }); + it("reads the auto-install opt-in through the shared environment layer", () => { + const source = readFileSync(join(ROOT, "tools"), "utf8"); + expect(source).toMatch(/autoInstall:\s*env\.get\("TOOLS_AUTOINSTALL"\)/); + expect(source).not.toMatch(/process\.env\.TOOLS_AUTOINSTALL/); + }); + it("reads the marker once and distinguishes linked and main checkouts", () => { + for (const linked of [true, false]) { + let reads = 0; + expect( + skipWorktreeInstall({ + workspaceRoot: ROOT, + stat: () => { + reads++; + return { isFile: () => linked }; + }, + }) + ).toBe(linked); + expect(reads).toBe(1); + } + }); + it("allows only missing-marker errors and propagates access failures", () => { + for (const code of ["ENOENT", "ENOTDIR", "EACCES", "EPERM"]) { + const error = Object.assign(new Error(code), { code }); + const check = () => + skipWorktreeInstall({ + workspaceRoot: ROOT, + stat: () => { + throw error; + }, + }); + if (code === "ENOENT" || code === "ENOTDIR") { + expect(check()).toBe(false); + } else { + expect(check).toThrow(error); + } + } + }); + it("honors explicit auto-install without inspecting the marker", () => { + expect( + skipWorktreeInstall({ + workspaceRoot: ROOT, + autoInstall: "1", + stat: () => { + throw new Error("must not inspect the marker"); + }, + }) + ).toBe(false); + }); +}); diff --git a/src/Internal/commands/reas/cache/index.ts b/src/Internal/commands/reas/cache/index.ts index 8ac0d16b11..1d728b5d91 100644 --- a/src/Internal/commands/reas/cache/index.ts +++ b/src/Internal/commands/reas/cache/index.ts @@ -1,10 +1,11 @@ import { existsSync, mkdirSync, readdirSync, unlinkSync } from "node:fs"; import { join } from "node:path"; import type { CacheEntry } from "@app/Internal/commands/reas/types"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; -const CACHE_DIR = join(env.tools.getHome(), ".genesis-tools", "internal", "reas"); +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + +const CACHE_DIR = genesisToolsDir("internal", "reas"); export const REAS_TTL = 24 * 60 * 60 * 1000; export const SREALITY_TTL = 6 * 60 * 60 * 1000; diff --git a/src/Internal/commands/reas/lib/store.ts b/src/Internal/commands/reas/lib/store.ts index 928f64cf7a..97952f5b84 100644 --- a/src/Internal/commands/reas/lib/store.ts +++ b/src/Internal/commands/reas/lib/store.ts @@ -1,11 +1,11 @@ -import { join } from "node:path"; import { getListingPersistenceDistrict } from "@app/Internal/commands/reas/lib/district-matching"; import type { FullAnalysis } from "@app/Internal/commands/reas/types"; import { BaseDatabase } from "@genesiscz/utils/database"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; -const DEFAULT_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "internal", "reas", "reas.sqlite"); +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + +const DEFAULT_DB_PATH = genesisToolsDir("internal", "reas", "reas.sqlite"); export interface AnalysisHistoryRow { id: number; diff --git a/src/agents/lib/changes/log.ts b/src/agents/lib/changes/log.ts index f925901114..0f3b30eea7 100644 --- a/src/agents/lib/changes/log.ts +++ b/src/agents/lib/changes/log.ts @@ -1,7 +1,7 @@ import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs"; -import { dirname, join, sep } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { dirname, sep } from "node:path"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { namedArguments } from "../hooks/diff/command-paths"; import { safeSegment } from "../hooks/paths"; @@ -56,7 +56,7 @@ export function sessionChangesPath(session: string): string { throw new Error(`not a session id: "${session}"`); } - return join(env.tools.getHome(), ".genesis-tools", "agents", safe, "changes.jsonl"); + return genesisToolsDir("agents", safe, "changes.jsonl"); } /** The harness file tools this log records: Edit and MultiEdit as `edit`, Write as `write`. */ diff --git a/src/agents/lib/changes/objects.ts b/src/agents/lib/changes/objects.ts index 8845d3c5d1..7fe8341fcb 100644 --- a/src/agents/lib/changes/objects.ts +++ b/src/agents/lib/changes/objects.ts @@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { ChangeSink } from "./log"; /** Well inside the hook's own 15 s budget, so a stuck `git` fails the store, not the hook. */ @@ -10,7 +10,7 @@ const GIT_TIMEOUT_MS = 8_000; /** One bare repo for every session. `git hash-object -w` stores identical bytes once. */ export function objectsDir(): string { - return join(env.tools.getHome(), ".genesis-tools", "agents", "_objects"); + return genesisToolsDir("agents", "_objects"); } function ensureRepo(gitDir: string): void { diff --git a/src/agents/lib/hooks/diff/writers.ts b/src/agents/lib/hooks/diff/writers.ts index 74d4e372d5..e8c0440171 100644 --- a/src/agents/lib/hooks/diff/writers.ts +++ b/src/agents/lib/hooks/diff/writers.ts @@ -2,7 +2,7 @@ import { closeSync, existsSync, openSync, readFileSync, readSync, statSync } fro import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; -import { toolDataDir } from "@genesiscz/utils/storage/root"; +import { genesisToolsDirIn, toolDataDir } from "@genesiscz/utils/storage/root"; import { hookDiag } from "../log"; import type { NamedChange } from "./named"; @@ -42,7 +42,7 @@ interface ManifestEntry { export function fableReplaceJournal(): string { const override = env.tools.getFableReplaceHome(); if (override) { - return join(override, ".genesis-tools", "fable-replace", "journal.jsonl"); + return genesisToolsDirIn(override, "fable-replace", "journal.jsonl"); } return toolDataDir("fable-replace", "journal.jsonl"); diff --git a/src/ai-proxy/scripts/verify-openrouter-live.ts b/src/ai-proxy/scripts/verify-openrouter-live.ts index a883d2246e..23828ba385 100644 --- a/src/ai-proxy/scripts/verify-openrouter-live.ts +++ b/src/ai-proxy/scripts/verify-openrouter-live.ts @@ -39,6 +39,7 @@ import { queryUsage, recordUsage } from "@genesiscz/utils/ai/usage"; import { concurrentMap } from "@genesiscz/utils/async"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { matchGlob } from "@genesiscz/utils/string"; import { createBoxTable, formatDotStatus, renderCliHeader, renderCliSection } from "@genesiscz/utils/table"; import { embed, generateText } from "ai"; @@ -183,7 +184,7 @@ function sandboxHome(): string { * never lands in a file this script wrote. */ async function seedAiAccount(home: string): Promise { - const dir = join(home, ".genesis-tools", "ai"); + const dir = genesisToolsDirIn(home, "ai"); mkdirSync(dir, { recursive: true }); writeFileSync( join(dir, "config.json"), diff --git a/src/ask/AIChat.ts b/src/ask/AIChat.ts index 0c9c94a87d..2ad928718e 100644 --- a/src/ask/AIChat.ts +++ b/src/ask/AIChat.ts @@ -1,4 +1,3 @@ -import { resolve } from "node:path"; import { ChatEngine } from "@ask/chat/ChatEngine"; import { ChatEvent } from "@ask/lib/ChatEvent"; import { ChatLog } from "@ask/lib/ChatLog"; @@ -17,7 +16,7 @@ import { modelSelector } from "@ask/providers/ModelSelector"; import { providerManager } from "@ask/providers/ProviderManager"; import type { ChatConfig } from "@ask/types"; import { getLanguageModel } from "@ask/types"; -import { env } from "@genesiscz/utils/env"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; import type { ToolSet } from "ai"; import { tool } from "ai"; @@ -26,7 +25,7 @@ interface EngineWithRestore { restore: () => void; } -const DEFAULT_SESSION_DIR = resolve(env.tools.getHome(), ".genesis-tools/ai-chat/sessions"); +const DEFAULT_SESSION_DIR = toolDataDir("ai-chat", "sessions"); export class AIChat { private _options: AIChatOptions; diff --git a/src/ask/chat/ConversationManager.ts b/src/ask/chat/ConversationManager.ts index 4737327cb4..7b8732d6ab 100644 --- a/src/ask/chat/ConversationManager.ts +++ b/src/ask/chat/ConversationManager.ts @@ -4,13 +4,13 @@ import { join, resolve } from "node:path"; import type { ChatMessage, ChatSession, ConversationMetadata } from "@ask/types"; import { usageCacheReadTokens } from "@ask/utils/helpers"; import { costForCall } from "@genesiscz/utils/ai/catalog/pricing"; -import { env } from "@genesiscz/utils/env"; import { formatCost } from "@genesiscz/utils/format"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { write } from "bun"; -const DEFAULT_CONVERSATIONS_DIR = resolve(env.tools.getHome(), ".genesis-tools", "ask", "conversations"); +const DEFAULT_CONVERSATIONS_DIR = genesisToolsDir("ask", "conversations"); export class ConversationManager { private conversationsDir: string; diff --git a/src/ask/lib/db.ts b/src/ask/lib/db.ts index e48bfaf07a..1934d7f62f 100644 --- a/src/ask/lib/db.ts +++ b/src/ask/lib/db.ts @@ -1,6 +1,5 @@ -import { join } from "node:path"; import { createKyselyClient, type DatabaseClient } from "@genesiscz/utils/database"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { AskDB } from "./db-types"; const BOOTSTRAP: string[] = [ @@ -25,7 +24,7 @@ const BOOTSTRAP: string[] = [ ]; export function defaultAskDbPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "ask.sqlite"); + return genesisToolsDir("ask.sqlite"); } export function openAskDatabase(path?: string): DatabaseClient { diff --git a/src/automate/lib/credentials.ts b/src/automate/lib/credentials.ts index d0cf3c54d7..602f14d566 100644 --- a/src/automate/lib/credentials.ts +++ b/src/automate/lib/credentials.ts @@ -2,13 +2,13 @@ import { existsSync, mkdirSync, readdirSync, unlinkSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { atomicWriteFileSync, isJsonObject, readJsonFileForUpdateSync } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { StoredCredential } from "./types"; -const CREDENTIALS_DIR = join(env.tools.getHome(), ".genesis-tools", "automate", "credentials"); +const CREDENTIALS_DIR = genesisToolsDir("automate", "credentials"); /** Validate credential name to prevent path traversal */ function safeName(name: string): string { diff --git a/src/automate/lib/daemon.ts b/src/automate/lib/daemon.ts index c9dd3c0416..2aa70812e6 100644 --- a/src/automate/lib/daemon.ts +++ b/src/automate/lib/daemon.ts @@ -1,11 +1,10 @@ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { createLogger } from "@genesiscz/utils/logger"; import { clearPidFile, readLivePid, writePidFileWhenFree } from "@genesiscz/utils/process/pidfile"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { closeDb, getDb } from "./db"; import { runSchedulerLoop } from "./scheduler"; -const PID_FILE = join(env.tools.getHome(), ".genesis-tools", "automate", "daemon.pid"); +const PID_FILE = genesisToolsDir("automate", "daemon.pid"); /** * True when a command line is recognisably an automate daemon. Only used to diff --git a/src/automate/lib/db.ts b/src/automate/lib/db.ts index 0ae2fa53c9..409a9a242b 100644 --- a/src/automate/lib/db.ts +++ b/src/automate/lib/db.ts @@ -1,13 +1,12 @@ -import { join } from "node:path"; import { createKyselyClient, type DatabaseClient } from "@genesiscz/utils/database"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { sql } from "kysely"; import type { AutomateDB, RunLogRow, RunRow, ScheduleRow } from "./db-types"; export type { RunLogRow, RunRow, ScheduleRow } from "./db-types"; -const DB_PATH = join(env.tools.getHome(), ".genesis-tools", "automate", "automate.db"); +const DB_PATH = genesisToolsDir("automate", "automate.db"); const SCHEMA_VERSION = 1; const BOOTSTRAP: string[] = [ diff --git a/src/automate/lib/launchd.ts b/src/automate/lib/launchd.ts index 05983c275a..329085824b 100644 --- a/src/automate/lib/launchd.ts +++ b/src/automate/lib/launchd.ts @@ -1,22 +1,22 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; import { launchdPlistNeedsGenesisApp, launchdProgramArgumentsXml } from "@genesiscz/utils/macos/genesis-app"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; const PLIST_PATH = join(homedir(), "Library", "LaunchAgents", "com.genesis-tools.automate.plist"); const LABEL = "com.genesis-tools.automate"; -export const DAEMON_LOG_DIR = join(env.tools.getHome(), ".genesis-tools", "automate", "logs"); +export const DAEMON_LOG_DIR = genesisToolsDir("automate", "logs"); export const DAEMON_STDOUT_LOG = join(DAEMON_LOG_DIR, "daemon-stdout.log"); export const DAEMON_STDERR_LOG = join(DAEMON_LOG_DIR, "daemon-stderr.log"); export function generatePlist(): string { const home = homedir(); const daemonScript = resolve(import.meta.dir, "daemon.ts"); - const logDir = join(home, ".genesis-tools", "automate", "logs"); + const logDir = DAEMON_LOG_DIR; const bunPath = Bun.which("bun") ?? "/usr/local/bin/bun"; return ` @@ -45,7 +45,7 @@ ${launchdProgramArgumentsXml([bunPath, "run", daemonScript])} * unload fails, so a failed migration never leaves the user with no agent at all. */ export async function installLaunchd(): Promise { - mkdirSync(join(env.tools.getHome(), ".genesis-tools", "automate", "logs"), { recursive: true }); + mkdirSync(genesisToolsDir("automate", "logs"), { recursive: true }); const previousPlist = existsSync(PLIST_PATH) ? readFileSync(PLIST_PATH, "utf8") : undefined; diff --git a/src/benchmark/lib/results.ts b/src/benchmark/lib/results.ts index 279bb27202..f4722c1fa7 100644 --- a/src/benchmark/lib/results.ts +++ b/src/benchmark/lib/results.ts @@ -1,10 +1,10 @@ import { existsSync, mkdirSync, readdirSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { SavedResult } from "./types"; -const RESULTS_DIR = join(env.tools.getHome(), ".genesis-tools", "benchmarks"); +const RESULTS_DIR = genesisToolsDir("benchmarks"); export function ensureResultsDir(): void { if (!existsSync(RESULTS_DIR)) { diff --git a/src/cmux/lib/capture-installer.ts b/src/cmux/lib/capture-installer.ts index b2f5848115..a880a1736a 100644 --- a/src/cmux/lib/capture-installer.ts +++ b/src/cmux/lib/capture-installer.ts @@ -24,6 +24,7 @@ import { renderCaptureShell } from "@app/cmux/lib/capture-shell"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { z } from "zod"; const START = "# >>> GenesisTools cmux capture >>>"; @@ -71,7 +72,7 @@ function realpathOr(path: string): string { function paths(options: CaptureInstallOptions) { const home = resolve(options.home ?? env.tools.getHome()); - const root = join(home, ".genesis-tools/cmux"); + const root = genesisToolsDirIn(home, "cmux"); const configuredRc = resolve(options.rcPath ?? join(home, ".zshrc")); let symlink = false; try { diff --git a/src/cmux/lib/capture-journal.ts b/src/cmux/lib/capture-journal.ts index 3d4a83abbf..aa6ad65f23 100644 --- a/src/cmux/lib/capture-journal.ts +++ b/src/cmux/lib/capture-journal.ts @@ -12,9 +12,9 @@ import { writeSync, } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { z } from "zod"; const MAX_JOURNAL_BYTES = 1024 * 1024; @@ -34,7 +34,7 @@ const recordSchema = z.object({ export type CapturedCommand = z.infer; export function captureJournalDirectory(): string { - return join(env.tools.getHome(), ".genesis-tools", "cmux", "command-journal"); + return genesisToolsDir("cmux", "command-journal"); } /** One owning shell per surface writes synchronously before the command starts. */ diff --git a/src/cmux/lib/capture-runtime-logger.ts b/src/cmux/lib/capture-runtime-logger.ts index 74e361ca83..55ca3aec4a 100644 --- a/src/cmux/lib/capture-runtime-logger.ts +++ b/src/cmux/lib/capture-runtime-logger.ts @@ -1,7 +1,7 @@ import { appendFileSync, existsSync, mkdirSync, renameSync, statSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; export function createCaptureRuntimeLogger(options: { directory?: string } = {}) { let previous: string | undefined; @@ -16,7 +16,7 @@ export function createCaptureRuntimeLogger(options: { directory?: string } = {}) return; } - const directory = options.directory ?? join(env.tools.getHome(), ".genesis-tools/cmux"); + const directory = options.directory ?? toolDataDir("cmux"); let line = `${SafeJSON.stringify({ at: new Date().toISOString(), level, message, ...detail })}\n`; if (Buffer.byteLength(line) > 65536) { line = `${SafeJSON.stringify({ at: new Date().toISOString(), level, message: message.slice(0, 1024), detail: "oversized diagnostic omitted" })}\n`; diff --git a/src/cmux/lib/screen-cache.ts b/src/cmux/lib/screen-cache.ts index b9a2dfd7df..cf8973ac3b 100644 --- a/src/cmux/lib/screen-cache.ts +++ b/src/cmux/lib/screen-cache.ts @@ -11,9 +11,9 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { stripAnsi } from "@genesiscz/utils/string"; import { z } from "zod"; @@ -30,7 +30,7 @@ export function preferredScreenText(nativeText: string | undefined, cachedText: } export function screenCacheDirectory(): string { - return join(env.tools.getHome(), ".genesis-tools", "cmux", "screens"); + return genesisToolsDir("cmux", "screens"); } function writeAtomic(path: string, text: string): void { diff --git a/src/control/README.md b/src/control/README.md index 1a6d113ae1..6ee05dc53a 100644 --- a/src/control/README.md +++ b/src/control/README.md @@ -320,7 +320,7 @@ tools control cursor click \ ### Snapshot drag, selection and paste -`drag` uses the left mouse button and accepts `--to X,Y`, `--duration 0.1..5`, `--coords` and `--background`. `click --button left|right|middle` selects a mouse button for clicks. Background drag and right-click passed the dedicated AppKit fixture; receiving apps must accept background events. The tool does not explicitly activate or raise the app, but an app may change its own key window in response. +`drag` uses the left mouse button and accepts `--to X,Y`, `--duration 0.1..5`, `--coords`, `--background` and `--allow-outside`. `--allow-outside` requires a non-chrome observation: chrome-scoped tokens refuse it before dispatch so a toolbar drag cannot enter omitted page content. With `--allow-outside` the path and `--to` may leave the snapshot window and the window may move with the drag (a panel dragged by its body): the start keeps every check, each later step checks only that the window is still on screen and that the user has not taken the pointer, every step's window-local location is measured from where the window is at that step, and the result reports `windowBoundsAfter`. The ComputerUse/MCP adapter also extrapolates image-coordinate destinations outside the screenshot when `allow_outside` is set; screenshot dimensions remain required and the start stays inside the screenshot. Events still go to the target app only, so a path over another app's window delivers nothing to that app. `click --button left|right|middle` selects a mouse button for clicks. Background drag and right-click passed the dedicated AppKit fixture; receiving apps must accept background events. The tool does not explicitly activate or raise the app, but an app may change its own key window in response. 🛑 **A right-click never takes the user's focus.** It opens a context menu, which needs neither the app in front nor the real pointer, so every path delivers it in the background: `act --button right` is always background (with or without `--background`), `--prepare` skips its focus and raise step for it, the computer-use `click` with `mouse_button: "right"` is never prepared, and the legacy `ax-tool click --right` no longer activates the app, raises its window or posts at the global HID tap. A right-click, on an element or a point, is delivered three ways in this order, and each later way runs only if no menu was seen and no menu of the app is open: the NSEvent-built right mouse down and up addressed to the window (`window-event`), a plain CGEvent right mouse down and up posted to the pid with the window number and window-local location set (`post-to-pid`), then `AXShowMenu` on the element (or the element under the point; skipped for a double click or a modifier). After each the tool waits up to one second for a new menu window of the app or an `AXMenu` in its tree. 🛑 `AXShowMenu` is last because SwiftUI answers it with the menu of an ENCLOSING view: on a Genesis Markdown tab button it opened the tab bar's one-item menu (Wrap Tabs), and on a fixture `Button` with its own `.contextMenu` inside a container with another, it opened the container's, while the window event opened the button's own (2026-10-15). Element targets used to go to `AXShowMenu` first. Measured on 2026-10-13 against a background AppKit probe: no form made the probe the active app. The result says how it went: `delivery` (the delivery that opened the menu, or the ones tried), `deliveriesTried`, `menuOpened`, `menuOpenedBy`, and `menuItems`, the first five item titles, so a caller can tell the element's own menu from an enclosing one. `AX_TOOL_CONTEXT_MENU_DELIVERIES` (a comma list, such as `post-to-pid`) limits the deliveries, for testing one. If no menu is seen, the result is `ok: false` with `menuOpened: false` and an error; `dispatchState` stays `dispatched`, because the events were sent. No path moves the pointer. If the target still brought itself to the front, the result has `frontmostRestored: true` with a `focusNote` once the app the user had was put back. An open context menu takes the keyboard until it closes (`menuHoldsKeyboard: true`); pick an item or close it with `--action perform --ax-action AXCancel` promptly. diff --git a/src/control/commands/workflow.test.ts b/src/control/commands/workflow.test.ts index 714be911ba..2d9c7c61f3 100644 --- a/src/control/commands/workflow.test.ts +++ b/src/control/commands/workflow.test.ts @@ -107,6 +107,7 @@ test("action help exposes native drag selection and paste options", async () => expect(result.stdout).toContain("--format"); expect(result.stdout).toContain("256 UTF-16 units"); expect(result.stdout).toContain("--button [name]"); + expect(result.stdout).toContain("--allow-outside"); }); const actOn = ["act", "--app", "nonexistent-control-fixture"]; diff --git a/src/control/commands/workflow.ts b/src/control/commands/workflow.ts index db5a3f78b4..b4dbf96959 100644 --- a/src/control/commands/workflow.ts +++ b/src/control/commands/workflow.ts @@ -64,6 +64,7 @@ interface WorkflowOptions { coords?: string; frame?: string; background?: boolean; + allowOutside?: boolean; prepare?: boolean; replace?: boolean; targetKey?: string; @@ -512,6 +513,10 @@ export function registerWorkflowCommands(program: Command): void { .option("--replace", "paste with --prepare: select all, paste once and verify exact field readback") .option("--button [name]", "click: left, right or middle") .option("--to ", "drag: global destination point") + .option( + "--allow-outside", + "drag: the path and --to may leave the snapshot window, and the window may move with the drag (a panel dragged by its body). Only the start is checked against the snapshot; each later step checks that the window is still on screen and that you have not taken the pointer. The result reports windowBoundsAfter." + ) .option("--duration ", "drag: duration from 0.1 to 5 seconds") .option("--pages ", "scroll: 1–20 receiving AX scroll-area viewport pages; if unavailable use --pixels") .option("--pixels ", "scroll: exact synthetic wheel pixels from 1 to 10000; mutually exclusive with --pages") @@ -679,6 +684,10 @@ export function registerWorkflowCommands(program: Command): void { args.push("--background"); } + if (opts.allowOutside) { + args.push("--allow-outside"); + } + if (opts.refresh) { args.push("--refresh"); } diff --git a/src/control/lib/computer-use/schemas.ts b/src/control/lib/computer-use/schemas.ts index e14359c7a0..a9f32fe95b 100644 --- a/src/control/lib/computer-use/schemas.ts +++ b/src/control/lib/computer-use/schemas.ts @@ -192,6 +192,8 @@ export const computerSchemas = { coordinate_space: z.enum(["image", "screen"]).default("image"), duration: z.number().min(0.1).max(5).default(0.3), background: z.boolean().default(true), + /** The destination may lie outside the observed window, and the window may move with the drag. */ + allow_outside: z.boolean().default(false), }) .strict(), scroll: z diff --git a/src/control/lib/computer-use/session.test.ts b/src/control/lib/computer-use/session.test.ts index 6f39d19a02..fc7931b57c 100644 --- a/src/control/lib/computer-use/session.test.ts +++ b/src/control/lib/computer-use/session.test.ts @@ -920,6 +920,45 @@ test("a paste reports the clipboard restore on every path, and an unreported or expect(replies).toHaveLength(0); }); +test("outside drag returns validated final bounds even when refresh fails", async () => { + for (const bounds of [ + { x: -300, y: 100, width: 800, height: 600 }, + { x: Number.POSITIVE_INFINITY, y: 0, width: 800, height: 600 }, + { x: 0, y: 0, width: -1, height: 600 }, + { x: "invalid", y: 0, width: 800, height: 600 }, + ]) { + const f = fixture(); + const original = f.native.run; + f.native.run = async (call) => + call.args[0] === "act" + ? { + ok: true, + dispatchState: "dispatched", + windowBoundsAfter: bounds, + after: { ok: false }, + note: "refresh failed", + } + : original(call); + const state = await f.computer.get_app_state({ app: "Fixture" }); + const result = await f.computer.drag({ + app: "Fixture", + revision: state.revision, + from_x: 100, + from_y: 100, + to_x: -100, + to_y: 100, + allow_outside: true, + }); + expect(result.ok).toBe(true); + expect(result.state).toBeUndefined(); + expect(result.note).toBe("refresh failed"); + expect(result).toMatchObject({ + windowBoundsAfter: + typeof bounds.x === "number" && Number.isFinite(bounds.x) && bounds.width > 0 ? bounds : undefined, + }); + } +}); + test("a dispatched press whose refresh ran out of budget comes back dispatched with the note, not as a timeout", async () => { const f = fixture(); const original = f.native.run; @@ -1238,6 +1277,97 @@ describe("independent Computer Use session", () => { ); expect(calls).toHaveLength(3); }); + test("allow_outside permits only the drag destination to leave the observed window", async () => { + const { computer, calls } = fixture(); + const state = await computer.get_app_state({ app: "Fixture" }); + const drag = { + app: "Fixture", + revision: state.revision, + coordinate_space: "screen" as const, + from_x: -400, + from_y: 150, + to_x: 100, + to_y: 150, + }; + await expect(computer.drag(drag)).rejects.toThrow("outside the observed window"); + await expect(computer.drag({ ...drag, allow_outside: true, from_x: 100 })).rejects.toThrow( + "outside the observed window" + ); + await expect( + computer.drag({ ...drag, allow_outside: true, coordinate_space: "image", from_x: 801, to_x: 200 }) + ).rejects.toThrow("outside the source screenshot"); + await expect(computer.drag({ ...drag, allow_outside: true, to_x: Number.POSITIVE_INFINITY })).rejects.toThrow(); + expect(calls).toHaveLength(1); + expect((await computer.drag({ ...drag, allow_outside: true })).ok).toBe(true); + expect(calls).toHaveLength(2); + expect(calls[1]).toContain("--allow-outside"); + expect(calls[1].slice(calls[1].indexOf("--coords"), calls[1].indexOf("--coords") + 4)).toEqual([ + "--coords", + "-400,150", + "--to", + "100,150", + ]); + }); + test("allow_outside extrapolates image destinations while keeping the image start bounded", async () => { + for (const [to_x, screen] of [ + [-200, "-600,150"], + [1000, "0,150"], + ] as const) { + const { computer, calls } = fixture(); + const state = await computer.get_app_state({ app: "Fixture" }); + const drag = { app: "Fixture", revision: state.revision, from_x: 200, from_y: 100, to_x, to_y: 100 }; + await expect(computer.drag(drag)).rejects.toThrow("outside the source screenshot"); + await expect(computer.drag({ ...drag, allow_outside: true, from_x: -1 })).rejects.toThrow( + "outside the source screenshot" + ); + expect(calls).toHaveLength(1); + expect((await computer.drag({ ...drag, allow_outside: true })).ok).toBe(true); + expect(calls).toHaveLength(2); + expect(calls[1]).toContain("--allow-outside"); + expect(calls[1].slice(calls[1].indexOf("--coords"), calls[1].indexOf("--coords") + 4)).toEqual([ + "--coords", + "-400,150", + "--to", + screen, + ]); + } + }); + + test("outside image destinations still need screenshot dimensions and finite converted coordinates", async () => { + const missing = fixture(); + const run = missing.native.run; + missing.native.run = async (input) => ({ ...(await run(input)), screenshot: {} }); + const state = await missing.computer.get_app_state({ app: "Fixture" }); + await expect( + missing.computer.drag({ + app: "Fixture", + revision: state.revision, + from_x: 0, + from_y: 0, + to_x: -200, + to_y: 100, + allow_outside: true, + }) + ).rejects.toThrow("image-backed state"); + expect(missing.calls).toHaveLength(1); + + const overflow = fixture(); + overflow.snapshot.screenshot.width = 1; + const observed = await overflow.computer.get_app_state({ app: "Fixture" }); + await expect( + overflow.computer.drag({ + app: "Fixture", + revision: observed.revision, + from_x: 0, + from_y: 0, + to_x: Number.MAX_VALUE, + to_y: 100, + allow_outside: true, + }) + ).rejects.toThrow("not a finite coordinate"); + expect(overflow.calls).toHaveLength(1); + }); + test("literal values, context selection and normalized secondary actions keep native argv separate", async () => { const { computer, calls } = fixture(); let state = await computer.get_app_state({ app: "Fixture", image: false }); diff --git a/src/control/lib/computer-use/session.ts b/src/control/lib/computer-use/session.ts index 8d825a4af3..9c1f55e672 100644 --- a/src/control/lib/computer-use/session.ts +++ b/src/control/lib/computer-use/session.ts @@ -47,6 +47,12 @@ function parseCall( signal?.throwIfAborted(); return { options: computerSchemas[method].parse(args) as ComputerArgs, signal }; } +const windowBoundsSchema = z.object({ + x: z.number().finite(), + y: z.number().finite(), + width: z.number().finite().positive(), + height: z.number().finite().positive(), +}); const snapshotSchema = observationSchema.and( z.object({ window: z.object({ x: z.number(), y: z.number(), width: z.number().positive(), height: z.number().positive() }), @@ -1015,11 +1021,13 @@ export class ComputerUse { x, y, coordinateSpace, + allowOutside = false, }: { record: AppRecord; x: number; y: number; coordinateSpace: "image" | "screen"; + allowOutside?: boolean; }) { const { window, screenshot } = record.snapshot; if (coordinateSpace === "image") { @@ -1029,13 +1037,19 @@ export class ComputerUse { "Get an image-backed state before using screenshot coordinates." ); } - if (x < 0 || y < 0 || x >= screenshot.width || y >= screenshot.height) { + if (!allowOutside && (x < 0 || y < 0 || x >= screenshot.width || y >= screenshot.height)) { throw new ComputerUseError("INVALID_POINT", "Point is outside the source screenshot."); } - x = window.x + (x * window.width) / screenshot.width; - y = window.y + (y * window.height) / screenshot.height; + x = window.x + (x / screenshot.width) * window.width; + y = window.y + (y / screenshot.height) * window.height; } - if (x < window.x || y < window.y || x >= window.x + window.width || y >= window.y + window.height) { + if (!Number.isFinite(x) || !Number.isFinite(y)) { + throw new ComputerUseError("INVALID_POINT", "Converted point is not a finite coordinate."); + } + if ( + !allowOutside && + (x < window.x || y < window.y || x >= window.x + window.width || y >= window.y + window.height) + ) { throw new ComputerUseError("INVALID_POINT", "Point is outside the observed window."); } return `${x},${y}`; @@ -1153,6 +1167,7 @@ export class ComputerUse { clipboardRestore: result.clipboardRestore, recovery: result.recovery, note: typeof result.note === "string" ? result.note : undefined, + windowBoundsAfter: windowBoundsSchema.safeParse(result.windowBoundsAfter).data, }; }); } @@ -1321,10 +1336,12 @@ export class ComputerUse { x: options.to_x, y: options.to_y, coordinateSpace: options.coordinate_space, + allowOutside: options.allow_outside, }), "--duration", String(options.duration), ...(options.background ? ["--background"] : []), + ...(options.allow_outside ? ["--allow-outside"] : []), ]; }, signal, diff --git a/src/control/lib/runner.ts b/src/control/lib/runner.ts index 2b9af9afe8..46ac5bfe38 100644 --- a/src/control/lib/runner.ts +++ b/src/control/lib/runner.ts @@ -13,6 +13,7 @@ import { import { boundedCommand } from "@genesiscz/utils/process/bounded-command"; import { profiler } from "@genesiscz/utils/profile"; import { Stopwatch } from "@genesiscz/utils/Stopwatch"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { isTestProcess } from "@genesiscz/utils/test-process"; import { ancestorNamesOfThisProcess, @@ -88,7 +89,7 @@ export function refuseRealMachineInTest(what: string): void { throw new RealMachineInTestError(what); } -export const RECORD_DIR = join(env.tools.getHome(), ".genesis-tools", "control", "record"); +export const RECORD_DIR = genesisToolsDir("control", "record"); export const RECORD_SESSION = join(RECORD_DIR, "session.json"); const RECORDED_COMMANDS = new Set([ "press", diff --git a/src/control/mcp/server.ts b/src/control/mcp/server.ts index 3be8b0ec30..070748e315 100644 --- a/src/control/mcp/server.ts +++ b/src/control/mcp/server.ts @@ -39,7 +39,7 @@ const descriptions: Record = { quit_app: "Request a normal quit of the currently observed app instance. No force kill; unsaved-document prompts may keep it running.", click: "prepare:true explicitly focuses/reveals an observed element and clicks it in one native invocation. Click a currently observed element/ref or screenshot point. A normal accessible single click uses its observed AXPress; physical/right/double clicks use native events. Pixel evidence is checked and consumed once. No retry or AI call.", - drag: "Drag between points in the current screenshot (or explicit screen coordinates), with native geometry/pixel checks and bounded duration.", + drag: "Drag between points in the current screenshot (or explicit screen coordinates), with native geometry/pixel checks and bounded duration. allow_outside: true lets the destination leave the observed window and the window move with the drag (moving a panel by its body); only the start is checked.", scroll: "Scroll a current element or screenshot point by receiving-viewport pages or exact wheel pixels.", set_value: "Set an observed editable AX value with exact native readback. The supplied value is not sent to an AI model.", diff --git a/src/daemon/lib/config.ts b/src/daemon/lib/config.ts index 4b3c644e2d..2c16be27e5 100644 --- a/src/daemon/lib/config.ts +++ b/src/daemon/lib/config.ts @@ -2,6 +2,7 @@ import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { Storage } from "@genesiscz/utils/storage/storage"; import { parseInterval } from "./interval"; import { timeOnDay } from "./missed-run"; @@ -30,7 +31,7 @@ function resolveBaseDir(): string { return override; } - return join(env.tools.getHome(), ".genesis-tools", "daemon"); + return genesisToolsDir("daemon"); } export function getLogsBaseDir(): string { diff --git a/src/daemon/lib/launchd.ts b/src/daemon/lib/launchd.ts index 319fc194ce..b138d90b44 100644 --- a/src/daemon/lib/launchd.ts +++ b/src/daemon/lib/launchd.ts @@ -1,15 +1,15 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; import { launchdPlistNeedsGenesisApp, launchdProgramArgumentsXml } from "@genesiscz/utils/macos/genesis-app"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; const PLIST_PATH = join(homedir(), "Library", "LaunchAgents", "com.genesis-tools.daemon.plist"); const LABEL = "com.genesis-tools.daemon"; -export const DAEMON_LOG_DIR = join(env.tools.getHome(), ".genesis-tools", "daemon", "logs"); +export const DAEMON_LOG_DIR = genesisToolsDir("daemon", "logs"); export const DAEMON_STDOUT_LOG = join(DAEMON_LOG_DIR, "daemon-stdout.log"); export const DAEMON_STDERR_LOG = join(DAEMON_LOG_DIR, "daemon-stderr.log"); @@ -19,7 +19,7 @@ export const DAEMON_STDERR_LOG = join(DAEMON_LOG_DIR, "daemon-stderr.log"); export function generatePlist(): string { const home = homedir(); const daemonScript = resolve(import.meta.dir, "../daemon.ts"); - const logDir = join(home, ".genesis-tools", "daemon", "logs"); + const logDir = DAEMON_LOG_DIR; const bunPath = Bun.which("bun") ?? "/usr/local/bin/bun"; return ` @@ -48,7 +48,7 @@ ${launchdProgramArgumentsXml([bunPath, "run", daemonScript])} * unload fails, so a failed migration never leaves the user with no agent at all. */ export async function installLaunchd(): Promise { - mkdirSync(join(env.tools.getHome(), ".genesis-tools", "daemon", "logs"), { recursive: true }); + mkdirSync(genesisToolsDir("daemon", "logs"), { recursive: true }); const previousPlist = existsSync(PLIST_PATH) ? readFileSync(PLIST_PATH, "utf8") : undefined; diff --git a/src/debugging-master/core/file-tailer.test.ts b/src/debugging-master/core/file-tailer.test.ts index 7e92ef07f5..f29b460d85 100644 --- a/src/debugging-master/core/file-tailer.test.ts +++ b/src/debugging-master/core/file-tailer.test.ts @@ -1,10 +1,25 @@ import { afterEach, describe, expect, it } from "bun:test"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { isAbsolute, join, resolve } from "node:path"; import { FileTailer } from "@app/debugging-master/core/file-tailer"; +import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +it("relative storage overrides keep session paths absolute and contained", async () => { + await env.testing.withOverrides({ GENESIS_TOOLS_HOME: "relative-session-fixture" }, async () => { + const modulePath = `${import.meta.dir}/paths.ts?relative-storage-regression`; + const { SESSIONS_DIR, sessionFilePath } = await import(modulePath); + expect(isAbsolute(SESSIONS_DIR)).toBe(true); + expect(SESSIONS_DIR).toBe( + resolve("relative-session-fixture", ".genesis-tools", "debugging-master", "sessions") + ); + expect(sessionFilePath("valid-session")).toBe(join(SESSIONS_DIR, "valid-session.jsonl")); + expect(() => sessionFilePath("../escape")).toThrow("Invalid session name"); + expect(() => sessionFilePath(resolve("outside-session"))).toThrow("Invalid session name"); + }); +}); + describe("FileTailer truncation", () => { const dirs: string[] = []; diff --git a/src/debugging-master/core/paths.ts b/src/debugging-master/core/paths.ts index c1aca323a1..7cf355265b 100644 --- a/src/debugging-master/core/paths.ts +++ b/src/debugging-master/core/paths.ts @@ -1,13 +1,13 @@ import { resolve, sep } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** * Single source of truth for where session JSONL files live on disk. * All readers (dashboard-server, http-server, sse-broadcaster, file-tailer) * resolve session paths via this constant — keep them in sync by importing * from here instead of re-deriving the path. */ -export const SESSIONS_DIR = resolve(env.tools.getHome(), ".genesis-tools", "debugging-master", "sessions"); +export const SESSIONS_DIR = resolve(genesisToolsDir("debugging-master", "sessions")); /** * Resolve the JSONL file path for a session by name. Defense-in-depth: diff --git a/src/dev-dashboard/lib/e2e/box.ts b/src/dev-dashboard/lib/e2e/box.ts index 63ef3ad63e..4c9a392d72 100644 --- a/src/dev-dashboard/lib/e2e/box.ts +++ b/src/dev-dashboard/lib/e2e/box.ts @@ -1,8 +1,7 @@ -import { join } from "node:path"; import type { BoxCipher, KeyPair } from "@app/dev-dashboard/contract/box-types"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; import nacl from "tweetnacl"; import naclUtil from "tweetnacl-util"; @@ -26,7 +25,7 @@ export const naclBoxCipher: BoxCipher = { export const toBase64 = naclUtil.encodeBase64; export const fromBase64 = naclUtil.decodeBase64; -const DEFAULT_KEY_PATH = join(env.tools.getHome(), ".genesis-tools", "dev-dashboard", "e2e-keys.json"); +const DEFAULT_KEY_PATH = genesisToolsDir("dev-dashboard", "e2e-keys.json"); interface StoredKeys { publicKey: string; diff --git a/src/dev-dashboard/lib/e2e/pairing-code.ts b/src/dev-dashboard/lib/e2e/pairing-code.ts index a3ed452cf5..ec3a6a9df9 100644 --- a/src/dev-dashboard/lib/e2e/pairing-code.ts +++ b/src/dev-dashboard/lib/e2e/pairing-code.ts @@ -1,10 +1,10 @@ import { randomBytes } from "node:crypto"; import { chmodSync } from "node:fs"; -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + // The managed-tier pairing ADMISSION control. `/api/e2e/pair` stores public keys only, but // without this an attacker who can reach the Agent — including the untrusted vendor relay — // could self-pair its own key and defeat the "we can't see your data" guarantee (the trust @@ -16,7 +16,7 @@ import { logger } from "@genesiscz/utils/logger"; // complementary defence is the safety-number (SAS) check after pairing — see the managed-tier // follow-up. This module is the admission gate; SAS is the MITM detector. -const CODE_PATH = join(env.tools.getHome(), ".genesis-tools", "dev-dashboard", "pairing-code.json"); +const CODE_PATH = genesisToolsDir("dev-dashboard", "pairing-code.json"); const DEFAULT_TTL_MS = 5 * 60_000; const CODE_ALPHABET = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // no 0/O/1/I — unambiguous to read + type const CODE_LENGTH = 8; diff --git a/src/dev-dashboard/lib/system/history-db.ts b/src/dev-dashboard/lib/system/history-db.ts index 58e05f50dc..a9e2e40da7 100644 --- a/src/dev-dashboard/lib/system/history-db.ts +++ b/src/dev-dashboard/lib/system/history-db.ts @@ -1,10 +1,10 @@ import { Database } from "bun:sqlite"; import { mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { dirname } from "node:path"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { PulsePoint } from "./types"; -const DEFAULT_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "dev-dashboard", "pulse.db"); +const DEFAULT_DB_PATH = genesisToolsDir("dev-dashboard", "pulse.db"); export const MAX_SERIES_POINTS = 360; export function downsamplePoints(points: PulsePoint[], maxPoints: number = MAX_SERIES_POINTS): PulsePoint[] { diff --git a/src/dev-dashboard/lib/tunnel/pairing.ts b/src/dev-dashboard/lib/tunnel/pairing.ts index 847f5990ee..1a3ec5b538 100644 --- a/src/dev-dashboard/lib/tunnel/pairing.ts +++ b/src/dev-dashboard/lib/tunnel/pairing.ts @@ -1,7 +1,6 @@ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; // Agent-side pairing: re-export the PURE codec from the contract (the source of truth @@ -24,7 +23,7 @@ export interface PersistedTunnelConfig { pairingUri?: string; } -const TUNNEL_CONFIG_PATH = join(env.tools.getHome(), ".genesis-tools", "dev-dashboard", "tunnel.json"); +const TUNNEL_CONFIG_PATH = genesisToolsDir("dev-dashboard", "tunnel.json"); export function tunnelConfigPath(): string { return TUNNEL_CONFIG_PATH; diff --git a/src/dev-dashboard/server/routes/e2e.ts b/src/dev-dashboard/server/routes/e2e.ts index 18de97b37a..94639ad68f 100644 --- a/src/dev-dashboard/server/routes/e2e.ts +++ b/src/dev-dashboard/server/routes/e2e.ts @@ -1,16 +1,15 @@ -import { join } from "node:path"; import { verifyAndConsumePairingCode } from "@app/dev-dashboard/lib/e2e/pairing-code"; import { errorResult } from "@app/dev-dashboard/server/routes/error"; import type { RouteDef } from "@app/dev-dashboard/server/types"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; // The managed-tier pairing store. Holds device PUBLIC keys ONLY (0600 file) — there is // nothing here the vendor relay could misuse even if it logged the pairing POST, because // a public key is public by definition. The matching secret key never leaves the phone. -const PEERS_PATH = join(env.tools.getHome(), ".genesis-tools", "dev-dashboard", "e2e-peers.json"); +const PEERS_PATH = genesisToolsDir("dev-dashboard", "e2e-peers.json"); interface PeerRecord { publicKey: string; diff --git a/src/doctor/__tests__/developer-tools.test.ts b/src/doctor/__tests__/developer-tools.test.ts new file mode 100644 index 0000000000..bcaadd8d51 --- /dev/null +++ b/src/doctor/__tests__/developer-tools.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "bun:test"; +import { rm } from "node:fs/promises"; +import { + developerToolStatus, + developerToolsDetail, + hostAppBundle, + isDeveloperToolAllowed, + SecurityAnalyzer, +} from "@app/doctor/analyzers/security"; +import { writeCache } from "@app/doctor/lib/cache"; +import { analysisDirFor, cacheFilePath } from "@app/doctor/lib/paths"; +import type { AnalyzerContext, Finding } from "@app/doctor/lib/types"; + +describe("Developer Tools check", () => { + it("discloses reduced protection and restricts advice to trusted hosts in both states", () => { + for (const allowed of [false, true]) { + const detail = developerToolsDetail("Fixture", allowed); + expect(detail).toContain("reduces macOS protection"); + expect(detail).toContain("only to a host you trust"); + } + expect(developerToolsDetail("Fixture", false)).toContain("If you trust Fixture"); + expect(developerToolsDetail("Fixture", true)).toContain("skip the per-launch Gatekeeper scan"); + }); + + it("keeps partial TCC reads unknown unless a readable database proves a grant", () => { + const grant = [{ client: "com.example.host", auth_value: 2 }]; + expect(developerToolStatus([null, []], "com.example.host")).toBeNull(); + expect(developerToolStatus([[], null], "com.example.host")).toBeNull(); + expect(developerToolStatus([null, null], "com.example.host")).toBeNull(); + expect(developerToolStatus([null, grant], "com.example.host")).toBe(true); + expect(developerToolStatus([grant, null], "com.example.host")).toBe(true); + expect(developerToolStatus([[], []], "com.example.host")).toBe(false); + expect(developerToolStatus([[], grant], "com.example.other")).toBe(false); + }); + + it("ignores cached host findings and scans again when the host changes", async () => { + let host = "Editor"; + let scans = 0; + class HostSecurityAnalyzer extends SecurityAnalyzer { + protected async *run(): AsyncIterable { + scans++; + yield { + id: "sec-developer-tools", + analyzerId: this.id, + title: `Developer Tools for ${host}`, + severity: host === "Terminal" ? "safe" : "cautious", + actions: [], + }; + } + } + const analyzer = new HostSecurityAnalyzer(); + const runId = `doctor-security-test-${crypto.randomUUID()}`; + const ctx: AnalyzerContext = { + runId, + opts: { thorough: false, fresh: false, dryRun: true }, + emit: () => {}, + }; + await writeCache(analyzer.id, { + analyzerId: analyzer.id, + findings: [ + { + id: "sec-developer-tools", + analyzerId: analyzer.id, + title: "stale Terminal", + severity: "safe", + actions: [], + }, + ], + durationMs: 0, + error: null, + fromCache: false, + timestamp: new Date().toISOString(), + }); + try { + const editor = await analyzer.analyze(ctx); + expect(editor.fromCache).toBe(false); + expect(editor.findings[0].title).toBe("Developer Tools for Editor"); + expect(editor.findings[0].severity).toBe("cautious"); + host = "Terminal"; + const terminal = await analyzer.analyze(ctx); + expect(terminal.fromCache).toBe(false); + expect(terminal.findings[0].title).toBe("Developer Tools for Terminal"); + expect(terminal.findings[0].severity).toBe("safe"); + expect(scans).toBe(2); + } finally { + await rm(cacheFilePath(analyzer.id), { force: true }); + await rm(analysisDirFor(runId), { recursive: true, force: true }); + } + }); + + it("takes the outermost app up the process tree, not an app executable run as a CLI inside it", () => { + const tree = [ + "/Users/example/Applications/GenesisTools.app/Contents/MacOS/GenesisTools", + "/bin/zsh", + "/Users/example/.bun/bin/bun", + "/Applications/cmux.app/Contents/MacOS/cmux", + "/sbin/launchd", + ]; + expect(hostAppBundle(tree)).toBe("/Applications/cmux.app"); + }); + + it("reads an editor app and gives nothing outside any app", () => { + expect(hostAppBundle(["/Applications/Cursor.app/Contents/MacOS/Cursor"])).toBe("/Applications/Cursor.app"); + expect(hostAppBundle(["/bin/zsh", "/sbin/launchd"])).toBeNull(); + }); + + it("allows only an allowed row for the same bundle id", () => { + const rows = [ + { client: "com.apple.Terminal", auth_value: 2 }, + { client: "com.example.denied", auth_value: 0 }, + ]; + expect(isDeveloperToolAllowed(rows, "com.apple.Terminal")).toBe(true); + expect(isDeveloperToolAllowed(rows, "com.example.denied")).toBe(false); + expect(isDeveloperToolAllowed(rows, "com.cmuxterm.app")).toBe(false); + }); +}); diff --git a/src/doctor/analyzers/app-bloat.ts b/src/doctor/analyzers/app-bloat.ts index 700b7939a7..d24074be6f 100644 --- a/src/doctor/analyzers/app-bloat.ts +++ b/src/doctor/analyzers/app-bloat.ts @@ -30,6 +30,7 @@ import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { openFiles, pidsOf } from "@genesiscz/utils/process/open-files"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; export const APP_BLOAT_ANALYZER_ID = "app-bloat"; const MB = 1024 * 1024; @@ -215,7 +216,7 @@ export class AppBloatAnalyzer extends Analyzer { this.p("Library", "Caches"), this.p(".cache"), this.p(".android", "avd"), - join(this.toolsHome, ".genesis-tools", "logs"), + genesisToolsDirIn(this.toolsHome, "logs"), ]; } @@ -853,7 +854,7 @@ export class AppBloatAnalyzer extends Analyzer { } private async *bigLogs(c: Ctx): AsyncIterable { - const logsDir = join(this.toolsHome, ".genesis-tools", "logs"); + const logsDir = genesisToolsDirIn(this.toolsHome, "logs"); const roots: Array<{ dir: string; match: RegExp }> = [ { dir: this.p(".claude"), match: /\.jsonl$/ }, { dir: logsDir, match: /\.(log|ndjson|jsonl)$/ }, diff --git a/src/doctor/analyzers/security.ts b/src/doctor/analyzers/security.ts index 58ae040e61..53c7ddb24c 100644 --- a/src/doctor/analyzers/security.ts +++ b/src/doctor/analyzers/security.ts @@ -1,6 +1,11 @@ +import { Database } from "bun:sqlite"; +import { homedir } from "node:os"; +import { join } from "node:path"; import { Analyzer } from "@app/doctor/lib/analyzer"; import { run } from "@app/doctor/lib/run"; import type { AnalyzerCategory, AnalyzerContext, Finding } from "@app/doctor/lib/types"; +import { logger } from "@genesiscz/utils/logger"; +import { ancestry, readExecutablePath } from "@genesiscz/utils/process/processes"; interface SecurityCheck { cmd: string; @@ -14,7 +19,8 @@ export class SecurityAnalyzer extends Analyzer { readonly name = "Security"; readonly icon = "Q"; readonly category: AnalyzerCategory = "security"; - readonly cacheTtlMs = 24 * 60 * 60 * 1000; + // Developer Tools depends on this process's host, so machine-wide cached results are unsafe. + readonly cacheTtlMs = 0; protected async *run(_ctx: AnalyzerContext): AsyncIterable { for (const check of securityChecks()) { @@ -33,7 +39,133 @@ export class SecurityAnalyzer extends Analyzer { metadata: { check: check.label, passing }, }; } + + const devTools = await developerToolsFinding(this.id); + if (devTools) { + yield devTools; + } + } +} + +/** + * Whether the app that hosts this shell (cmux, Terminal, Ghostty, an editor) is on the Developer Tools list. + * Off the list, macOS runs Gatekeeper and XProtect on every new program it starts before the program runs: about + * 0.25 s for a tiny one, a full scan for a large one. Each build or test run makes new programs (adhoc, linker-signed, + * with com.apple.provenance), and several at once on 2026-10-14 pinned syspolicyd at 100% and stalled every launch + * until it was killed. Measured that day: a fresh binary's first launch took 260-400 ms under cmux (not listed) and + * 22 ms under Terminal.app (listed). + */ +async function developerToolsFinding(analyzerId: string): Promise { + const host = hostAppBundle(ancestorExecutables(process.pid)); + if (!host) { + return null; + } + + const name = + host + .split("/") + .pop() + ?.replace(/\.app$/, "") ?? host; + const plist = await run("plutil", ["-extract", "CFBundleIdentifier", "raw", join(host, "Contents/Info.plist")]); + const bundleId = plist.status === 0 ? plist.stdout.trim() : ""; + const allowed = bundleId ? developerToolStatus(readDeveloperToolClients(), bundleId) : null; + if (allowed === null) { + return null; + } + return { + id: "sec-developer-tools", + analyzerId, + title: `Developer Tools for ${name}: ${allowed ? "on" : "off"}`, + detail: developerToolsDetail(name, allowed), + severity: allowed ? "safe" : "cautious", + actions: [], + metadata: { check: "Developer Tools", passing: allowed, app: host, bundleId }, + }; +} + +export function developerToolsDetail(name: string, allowed: boolean): string { + const tradeoff = "This reduces macOS protection; grant Developer Tools access only to a host you trust."; + return allowed + ? `${name} is on the Developer Tools list: programs it starts skip the per-launch Gatekeeper scan ` + + `(a new entry counts once ${name} restarts). ${tradeoff}` + : `macOS scans every new program ${name} starts before it runs, so builds and tests can wait on syspolicyd. ` + + `${tradeoff} If you trust ${name}, add it in System Settings > Privacy & Security > Developer Tools, ` + + `then restart ${name}.`; +} + +/** + * The outermost `.app` bundle up the process tree (`executables` runs from this process to the top): the app macOS + * holds responsible for what the shell starts. An app executable run as a CLI inside the tree (the `tools` wrapper + * runs through GenesisTools.app) does not take that over. + */ +export function hostAppBundle(executables: string[]): string | null { + let outermost: string | null = null; + for (const path of executables) { + const match = path.match(/^(\/.+?\.app)\/Contents\/MacOS\//); + if (match) { + outermost = match[1]; + } + } + + return outermost; +} + +/** The bundle ids allowed in TCC's Developer Tools service (auth_value 2 is allowed). */ +export function isDeveloperToolAllowed(rows: Array<{ client: string; auth_value: number }>, bundleId: string): boolean { + return rows.some((row) => row.client === bundleId && row.auth_value === 2); +} + +interface DeveloperToolClient { + client: string; + auth_value: number; +} + +/** A grant proves access; a missing grant proves denial only if every database was readable. */ +export function developerToolStatus(reads: Array, bundleId: string): boolean | null { + if (reads.some((rows) => rows !== null && isDeveloperToolAllowed(rows, bundleId))) { + return true; } + + return reads.length === 0 || reads.some((rows) => rows === null) ? null : false; +} + +/** + * The Developer Tools rows from both TCC databases (System Settings writes them to the system one). + * Each unreadable database remains null so a partial read cannot turn an unknown grant into a denial. + */ +function readDeveloperToolClients(): Array { + const paths = [ + "/Library/Application Support/com.apple.TCC/TCC.db", + join(homedir(), "Library/Application Support/com.apple.TCC/TCC.db"), + ]; + const reads: Array = []; + for (const path of paths) { + try { + const db = new Database(path, { readonly: true }); + try { + const found = db + .query<{ client: string; auth_value: number }, []>( + "SELECT client, auth_value FROM access WHERE service = 'kTCCServiceDeveloperTool'" + ) + .all(); + reads.push(found); + } finally { + db.close(); + } + } catch (error) { + logger.debug({ error, path }, "could not read Developer Tools grants"); + reads.push(null); + } + } + + return reads; +} + +/** `pid` and its ancestors below launchd, nearest first, each as its executable path (its kernel name when unreadable). */ +function ancestorExecutables(pid: number): string[] { + return ancestry(pid, 12) + .filter((entry) => entry.pid > 1) + .map((entry) => readExecutablePath(entry.pid) ?? entry.name); } export function isFileVaultEnabled(out: string): boolean { diff --git a/src/doctor/lib/paths.ts b/src/doctor/lib/paths.ts index 725e1b396a..bc5ef19b71 100644 --- a/src/doctor/lib/paths.ts +++ b/src/doctor/lib/paths.ts @@ -1,9 +1,9 @@ import { mkdirSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { toPosixPath } from "@genesiscz/utils/paths"; -export const DOCTOR_DIR = toPosixPath(join(env.tools.getHome(), ".genesis-tools", "doctor")); +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; +export const DOCTOR_DIR = toPosixPath(genesisToolsDir("doctor")); export const ANALYSIS_DIR = toPosixPath(join(DOCTOR_DIR, "analysis")); export const CACHE_DIR = toPosixPath(join(DOCTOR_DIR, "cache")); export const SNAPSHOTS_DIR = toPosixPath(join(DOCTOR_DIR, "global-packages-snapshots")); diff --git a/src/github/lib/cache.ts b/src/github/lib/cache.ts index f239139c9b..6378c4ccfa 100644 --- a/src/github/lib/cache.ts +++ b/src/github/lib/cache.ts @@ -3,7 +3,6 @@ import { Database } from "bun:sqlite"; import { existsSync, mkdirSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import type { CommentRecord, FetchMetadataRecord, @@ -13,7 +12,9 @@ import type { } from "@genesiscz/utils/github/types"; import { logger } from "@genesiscz/utils/logger"; -const DEFAULT_CACHE_DIR = join(env.tools.getHome(), ".genesis-tools", "github"); +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + +const DEFAULT_CACHE_DIR = genesisToolsDir("github"); const DB_NAME = "cache.db"; let _db: Database | null = null; diff --git a/src/github/lib/stack-restack.test.ts b/src/github/lib/stack-restack.test.ts index 0936efadba..75c425e6a3 100644 --- a/src/github/lib/stack-restack.test.ts +++ b/src/github/lib/stack-restack.test.ts @@ -1,5 +1,17 @@ -import { describe, expect, test } from "bun:test"; -import { createNoopStackRestack, pickOldBase, unknownBaseRefusal } from "./stack-restack"; +import { describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { env } from "@genesiscz/utils/env"; +import { logger } from "@genesiscz/utils/logger"; +import { + createNoopStackRestack, + gitAt, + gitAuthEnv, + localObjectStore, + pickOldBase, + unknownBaseRefusal, +} from "./stack-restack"; describe("pickOldBase", () => { const inHistory = (shas: string[]) => async (sha: string) => shas.includes(sha); @@ -49,3 +61,51 @@ describe("createNoopStackRestack", () => { }); }); }); + +describe("gitAuthEnv", () => { + test("carries the token as an extraheader in the environment, never in a URL", () => { + const auth = gitAuthEnv("tok-123"); + expect(auth?.GIT_CONFIG_KEY_0).toBe("http.https://github.com/.extraheader"); + const header = auth?.GIT_CONFIG_VALUE_0 ?? ""; + expect(Buffer.from(header.replace("AUTHORIZATION: basic ", ""), "base64").toString()).toBe( + "x-access-token:tok-123" + ); + expect(gitAuthEnv(undefined)).toBeUndefined(); + }); +}); + +describe("restack subprocess behavior", () => { + test("disables terminal and askpass prompts with and without a token", async () => { + await env.testing.withOverrides({ GIT_TERMINAL_PROMPT: "1", GIT_ASKPASS: "/fixture/askpass" }, async () => { + for (const token of [undefined, "fixture-token"]) { + const result = await gitAt(process.cwd(), token).exec([ + "-c", + `alias.probe=!printf '%s|%s' "$GIT_TERMINAL_PROMPT" "$GIT_ASKPASS"`, + "probe", + ]); + expect(result.success).toBe(true); + expect(result.stdout.trim()).toBe("0|"); + } + }); + }); + + test("logs an unavailable object store and preserves the successful alternate", async () => { + const cwd = mkdtempSync(join(tmpdir(), "restack-no-git-")); + const debug = spyOn(logger, "debug"); + try { + expect(await localObjectStore(cwd)).toBeUndefined(); + expect(debug).toHaveBeenCalledWith( + expect.objectContaining({ + cwd, + exitCode: expect.any(Number), + stderr: expect.stringContaining("not a git repository"), + }), + "local Git object store unavailable; fetching without an alternate" + ); + } finally { + debug.mockRestore(); + rmSync(cwd, { recursive: true, force: true }); + } + expect(await localObjectStore()).toMatch(/[/\\]objects$/); + }); +}); diff --git a/src/github/lib/stack-restack.ts b/src/github/lib/stack-restack.ts index bcb891fb08..4be08d8ece 100644 --- a/src/github/lib/stack-restack.ts +++ b/src/github/lib/stack-restack.ts @@ -9,7 +9,7 @@ // git rebase --onto // then force-with-lease (same algorithm as github/gh-stack cascadeRebase). -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Executor } from "@genesiscz/utils/cli"; @@ -101,15 +101,24 @@ function writeToken(): string | undefined { return getGhCliToken() ?? env.github.getToken(); } -function authedCloneUrl(owner: string, repo: string, token: string | undefined): string { - if (token) { - return `https://x-access-token:${token}@github.com/${owner}/${repo}.git`; +/** + * The token reaches git through its environment (the `http.extraheader` actions/checkout uses), never through its + * argv, which any local user can read with `ps`, nor through the temp clone's .git/config. + */ +export function gitAuthEnv(token: string | undefined): Record | undefined { + if (!token) { + return undefined; } - return `https://github.com/${owner}/${repo}.git`; + const basic = Buffer.from(`x-access-token:${token}`).toString("base64"); + return { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: "http.https://github.com/.extraheader", + GIT_CONFIG_VALUE_0: `AUTHORIZATION: basic ${basic}`, + }; } -function gitAt(cwd: string): Executor { +export function gitAt(cwd: string, token?: string): Executor { return new Executor({ prefix: "git", cwd, @@ -117,9 +126,28 @@ function gitAt(cwd: string): Executor { verbose: false, debug: false, label: "git-restack", + env: { ...gitAuthEnv(token), GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "" }, }); } +/** + * The object store of the checkout this runs in, borrowed as an alternate so the fetch transfers only what it lacks. + * A cold fetch of both tips' full history timed out at 300 s on a slow link (2026-10-14, #505); with the alternate + * the same fetch took under a second and 4 KB. + */ +export async function localObjectStore(cwd: string = process.cwd()): Promise { + const result = await gitAt(cwd).exec(["rev-parse", "--path-format=absolute", "--git-common-dir"]); + if (!result.success || !result.stdout.trim()) { + logger.debug( + { cwd, exitCode: result.exitCode, stderr: result.stderr }, + "local Git object store unavailable; fetching without an alternate" + ); + return undefined; + } + + return join(result.stdout.trim(), "objects"); +} + async function gitOrThrow(g: Executor, args: string[], label: string, timeout = 120_000): Promise { const result = await g.exec(args, { timeout }); if (!result.success) { @@ -140,15 +168,21 @@ export function createGitStackRestack(options?: { log?: RestackLog }): StackRest return { async restackBranch(input: RestackBranchInput): Promise { const { owner, repo, branch, expectedHeadSha, newBase, oldBaseSha, priorBaseShas } = input; - const token = writeToken(); - const url = authedCloneUrl(owner, repo, token); + const url = `https://github.com/${owner}/${repo}.git`; const tmpRoot = await mkdtemp(join(tmpdir(), "gt-merge-restack-")); - const g = gitAt(tmpRoot); + const g = gitAt(tmpRoot, writeToken()); log?.(` restack: ${branch} onto ${newBase}${oldBaseSha ? ` (drop ≤ ${oldBaseSha.slice(0, 7)})` : ""}`); try { await gitOrThrow(g, ["init", "-q"], "git init"); + const localObjects = await localObjectStore(); + + if (localObjects) { + await mkdir(join(tmpRoot, ".git", "objects", "info"), { recursive: true }); + await writeFile(join(tmpRoot, ".git", "objects", "info", "alternates"), `${localObjects}\n`); + } + await gitOrThrow(g, ["remote", "add", "origin", url], "git remote add"); // Fetch only the refs we need (full history of those tips — no --depth). await gitOrThrow( diff --git a/src/handoff/log-store.ts b/src/handoff/log-store.ts index 7d1385b0be..bb7bd7fc27 100644 --- a/src/handoff/log-store.ts +++ b/src/handoff/log-store.ts @@ -2,13 +2,14 @@ import { appendFileSync, mkdirSync } from "node:fs"; import { dirname, join } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { HandoffEvent } from "./types"; /** Handoffs share the question family's storage root (§6.2) — honor its override too. */ function questionRoot(): string { const logBase = env.question.getLogBase(); - return logBase !== undefined ? dirname(logBase) : join(env.tools.getHome(), ".genesis-tools", "question"); + return logBase !== undefined ? dirname(logBase) : genesisToolsDir("question"); } export function handoffLogDir(base?: string): string { diff --git a/src/handoff/read-model.ts b/src/handoff/read-model.ts index e7c6cabfbd..2735a72543 100644 --- a/src/handoff/read-model.ts +++ b/src/handoff/read-model.ts @@ -1,10 +1,10 @@ import { Database } from "bun:sqlite"; import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { parseJsonlChunk } from "@genesiscz/utils/jsonl"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { applyHandoffEvent } from "./fold"; import { handoffLogDir } from "./log-store"; import type { FoldOutcome, Handoff, HandoffEvent, HandoffPublicEvent } from "./types"; @@ -13,7 +13,7 @@ const log = logger.child({ component: "handoff:read-model" }); /** Same db the /qa dashboard opens — handoffs ARE the QA family (spec §6.2). */ export function handoffDbPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "question", "qa.db"); + return genesisToolsDir("question", "qa.db"); } function ensureColumn(db: Database, table: string, column: string, ddl: string): void { diff --git a/src/indexer/lib/sources/telegram-source.ts b/src/indexer/lib/sources/telegram-source.ts index 3274b33b78..e1bbf07b9b 100644 --- a/src/indexer/lib/sources/telegram-source.ts +++ b/src/indexer/lib/sources/telegram-source.ts @@ -1,8 +1,7 @@ import { Database } from "bun:sqlite"; import { existsSync } from "node:fs"; -import { join } from "node:path"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { type DetectChangesOptions, defaultDetectChanges, @@ -13,7 +12,7 @@ import { type SourceEntry, } from "./source"; -const DEFAULT_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "telegram", "history.db"); +const DEFAULT_DB_PATH = genesisToolsDir("telegram", "history.db"); interface TelegramMessageRow { rowid: number; diff --git a/src/jev/lib/route/cache.ts b/src/jev/lib/route/cache.ts index c35ebd06b2..6cbf932f62 100644 --- a/src/jev/lib/route/cache.ts +++ b/src/jev/lib/route/cache.ts @@ -1,17 +1,17 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { discoverTools } from "@app/tools/lib/discovery"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { profiler } from "@genesiscz/utils/profile"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { buildCatalogue, CATALOGUE_VERSION, type ToolCatalogue } from "./catalogue"; const { log } = logger.scoped("jev-route"); const prof = profiler.scope("jev-route"); export function catalogueCachePath(): string { - return join(env.tools.getHome(), ".genesis-tools", "jev", "route-catalogue.json"); + return genesisToolsDir("jev", "route-catalogue.json"); } function gitHead(srcDir: string): string { diff --git a/src/jev/lib/sessions.ts b/src/jev/lib/sessions.ts index 3a90328e39..48456219ab 100644 --- a/src/jev/lib/sessions.ts +++ b/src/jev/lib/sessions.ts @@ -1,9 +1,8 @@ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { readTextChunks } from "@genesiscz/utils/fs/read-lines"; import { SafeJSON } from "@genesiscz/utils/json"; import { dayLogPath, listDayLogs } from "@genesiscz/utils/logger/day-logs"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** One parsed pino row from a day log. Everything beyond these keys is kept as-is. */ export interface JevLogRow { time: string; @@ -47,7 +46,7 @@ export interface JevDecision { } export function logsDirectory(): string { - return join(env.tools.getHome(), ".genesis-tools", "logs"); + return genesisToolsDir("logs"); } /** Day files only (plain or compressed), newest first; the `-profiling` siblings are a different format. */ diff --git a/src/jev/lib/wake/wake.ts b/src/jev/lib/wake/wake.ts index ce4a249618..2b3f916f00 100644 --- a/src/jev/lib/wake/wake.ts +++ b/src/jev/lib/wake/wake.ts @@ -1,9 +1,10 @@ import { mkdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { dirname } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { isJsonObject, readJsonFileForUpdateSync } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; const { log } = logger.scoped("jev-wake"); @@ -35,7 +36,7 @@ export const DEFAULT_WAKE_GATE: WakeGate = { }; export function wakeConfigPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "jev", "wake.json"); + return genesisToolsDir("jev", "wake.json"); } function isWakeGatePatch(value: unknown): value is Partial { diff --git a/src/learn-from-fable/commands/report.ts b/src/learn-from-fable/commands/report.ts index a07f66ada1..e976f6958e 100644 --- a/src/learn-from-fable/commands/report.ts +++ b/src/learn-from-fable/commands/report.ts @@ -1,9 +1,9 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger, out } from "@genesiscz/utils/logger"; import { renderMarkdownToCli } from "@genesiscz/utils/markdown"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { FABLE_CONFIG_PATH, type FableConfig, packPaths } from "../lib/config"; import { readStageRuns } from "../lib/manifest"; import type { ConsolidatedPrinciple } from "../lib/stages/consolidate"; @@ -249,10 +249,10 @@ export function buildReportMarkdown(config: FableConfig): { markdown: string; da push(`## Where to verify every model call`); push(); push( - `- per-request tokens + write-time cost: ${code(join(env.tools.getHome(), ".genesis-tools/ai-proxy/usage/requests.jsonl"))} (proxy records every call; grep by model id)` + `- per-request tokens + write-time cost: ${code(genesisToolsDir("ai-proxy", "usage", "requests.jsonl"))} (proxy records every call; grep by model id)` ); push( - `- day-stamped debug logs (full prompts on jsonrepair events, drift retries): ${code(join(env.tools.getHome(), ".genesis-tools/logs/"))}` + `- day-stamped debug logs (full prompts on jsonrepair events, drift retries): ${code(genesisToolsDir("logs"))}` ); push(`- episode inputs (context prefixes fed to models) + reference outputs: the per-model episode files above`); push( diff --git a/src/learn-from-fable/lib/config.ts b/src/learn-from-fable/lib/config.ts index 2a96d5ac85..e2c0e9d593 100644 --- a/src/learn-from-fable/lib/config.ts +++ b/src/learn-from-fable/lib/config.ts @@ -1,13 +1,12 @@ import { mkdirSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join, resolve } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { isJsonObject } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; - -export const FABLE_LOCAL_DIR = resolve(env.tools.getHome(), ".genesis-tools", "claude", "fable"); +export const FABLE_LOCAL_DIR = genesisToolsDir("claude", "fable"); export const FABLE_CONFIG_PATH = join(FABLE_LOCAL_DIR, "config.json"); export const FABLE_MODEL = "claude-fable-5-1"; diff --git a/src/macos/GenesisTools/scripts/perf-report.ts b/src/macos/GenesisTools/scripts/perf-report.ts index 25f12184ed..bb7180bbdc 100644 --- a/src/macos/GenesisTools/scripts/perf-report.ts +++ b/src/macos/GenesisTools/scripts/perf-report.ts @@ -1,5 +1,4 @@ #!/usr/bin/env bun -import { join } from "node:path"; // Copied from /Users/Martin/Tresors/Projects/GenesisPlayground/Genesis/scripts/perf-report.ts at 2026-09-24T04:59:16+02:00 at commit hash 0d268a43e86e6e5e0ce16132aed8c862e201923e /** * perf-report.ts — summarise `~/.genesis/logs/perf.log` (PerfLog + MonitorPerf). @@ -12,7 +11,7 @@ import { join } from "node:path"; * (`main-stall recovered after …`) and every `TIMEOUT` / hang-sample mark, so * "what was slow today" is one command instead of an awk session. */ -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; const args = process.argv.slice(2); const flag = (name: string): string | undefined => { @@ -20,7 +19,7 @@ const flag = (name: string): string | undefined => { return i >= 0 ? args[i + 1] : undefined; }; // GenesisTools adaptation: GenesisTools.app's PerfLog file; the whole file is biome-formatted (block ifs). -const file = flag("--file") ?? join(env.tools.getHome(), ".genesis-tools", "logs", "app-perf.log"); +const file = flag("--file") ?? genesisToolsDir("logs", "app-perf.log"); const tail = Number(flag("--tail") ?? 0); const minP95 = Number(flag("--min") ?? 0); diff --git a/src/macos/lib/clones/file-meta-cache.ts b/src/macos/lib/clones/file-meta-cache.ts index dfd88c597f..74140071cd 100644 --- a/src/macos/lib/clones/file-meta-cache.ts +++ b/src/macos/lib/clones/file-meta-cache.ts @@ -1,10 +1,9 @@ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import { createKyselyClient, type DatabaseClient } from "@genesiscz/utils/database/client"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { Stopwatch } from "@genesiscz/utils/Stopwatch"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; import { FILE_META_MIGRATION_CONTEXT, FILE_META_MIGRATIONS } from "./file-meta-migrations"; import type { FileMetaDB } from "./file-meta-schema"; @@ -24,7 +23,7 @@ const log = logger.child({ component: "clones:file-meta-cache" }); /** Single global path: any scan that walks a path looks up its cached metadata * regardless of which root surrounded it. `~/Projects/Foo` then `~/Projects` * reuses rows from the first scan. ONE db per machine, not per scan-root. */ -export const FILE_META_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "macos-clones", "cache", "file-meta.db"); +export const FILE_META_DB_PATH = genesisToolsDir("macos-clones", "cache", "file-meta.db"); /** SQLite caps bound parameters at 32766 (SQLITE_MAX_VARIABLE_NUMBER since * 3.32.0). Our flush rows have 6 columns, so 32766 / 6 = 5461 is the hard diff --git a/src/macos/lib/swap/scanner.ts b/src/macos/lib/swap/scanner.ts index c6aa162b30..618f7a3718 100644 --- a/src/macos/lib/swap/scanner.ts +++ b/src/macos/lib/swap/scanner.ts @@ -1,8 +1,8 @@ import { mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { dirname } from "node:path"; import { SafeJSON } from "@genesiscz/utils/json"; import { listProcessesWithUsage, readExecutablePath } from "@genesiscz/utils/process/processes"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; import type { ProcessSwap, ScanOptions, ScanResult, SystemSwap } from "./types"; const UNIT_MULT: Record = { @@ -62,7 +62,7 @@ const VMMAP_CONCURRENCY = 32; const RSS_FLOOR_BYTES = 25 * 1024 * 1024; const CACHE_TTL_MS = 2 * 60 * 1000; const START_TIME_TOLERANCE_MS = 5000; -const CACHE_PATH = join(env.tools.getHome(), ".genesis-tools", "macos-swap", "cache.json"); +const CACHE_PATH = toolDataDir("macos-swap", "cache.json"); interface CachedEntry { pid: number; diff --git a/src/mcp-manager/lib/gateway/login-runner.ts b/src/mcp-manager/lib/gateway/login-runner.ts index f58967c357..14c9d9308a 100644 --- a/src/mcp-manager/lib/gateway/login-runner.ts +++ b/src/mcp-manager/lib/gateway/login-runner.ts @@ -21,6 +21,7 @@ import { env } from "@genesiscz/utils/env"; import { watchFileFeed } from "@genesiscz/utils/fs/file-feed-watcher"; import { logger } from "@genesiscz/utils/logger"; import { sendNotification } from "@genesiscz/utils/macos/notifications"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { serverAuth } from "../auth/policy.ts"; import { autoLoginRefusal, createLoginLauncher, type LoginLauncher } from "./auto-login.ts"; import { pendingLoginPath, readPendingLogin, takeLivePendingLogin, writePendingLogin } from "./login-state.ts"; @@ -30,7 +31,7 @@ import { gatewayRepoRoot } from "./service.ts"; const URL_WAIT_MS = 30_000; export function loginLogFile(server: string): string { - return join(env.tools.getHome(), ".genesis-tools", "logs", `mcp-login-${encodeURIComponent(server)}.log`); + return genesisToolsDir("logs", `mcp-login-${encodeURIComponent(server)}.log`); } export function loginSpawnArgs(server: string, clientName?: string): string[] { diff --git a/src/mcp-manager/lib/gateway/service.ts b/src/mcp-manager/lib/gateway/service.ts index fd801db773..549400803a 100644 --- a/src/mcp-manager/lib/gateway/service.ts +++ b/src/mcp-manager/lib/gateway/service.ts @@ -21,14 +21,13 @@ import { startLaunchd, uninstallLaunchd, } from "@genesiscz/utils/DashboardApp/launchd"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { gatewayHealth } from "./health.ts"; - export const GATEWAY_LAUNCHD_LABEL = "com.genesis-tools.mcp-gateway"; export function gatewayLogFile(): string { - return join(env.tools.getHome(), ".genesis-tools", "logs", "mcp-gateway.log"); + return genesisToolsDir("logs", "mcp-gateway.log"); } export function gatewayPlistPath(): string { diff --git a/src/monitor/lib/db.ts b/src/monitor/lib/db.ts index d210ff89cc..b3240cd725 100644 --- a/src/monitor/lib/db.ts +++ b/src/monitor/lib/db.ts @@ -1,8 +1,8 @@ import { join } from "node:path"; import { createKyselyClient, type DatabaseClient, type Migration, nowUtcIso } from "@genesiscz/utils/database"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { sql } from "kysely"; import type { CheckRow, FeedItemRow, IncidentRow, MonitorDB, NotifyTargetRow, WatcherRow } from "./db-types"; import { @@ -30,10 +30,9 @@ import { type WatcherStatus, type WatcherSummary, } from "./types"; - // Root via `env.tools.getHome()` so the test sandbox's GENESIS_TOOLS_HOME is // honoured and no test ever writes the real watcher list. -export const MONITOR_DIR = join(env.tools.getHome(), ".genesis-tools", "monitor"); +export const MONITOR_DIR = genesisToolsDir("monitor"); export const DEFAULT_DB_PATH = join(MONITOR_DIR, "monitor.db"); const RECENT_POINTS = 40; diff --git a/src/node-repl/lib/trust.ts b/src/node-repl/lib/trust.ts index 9bbcf1c104..0833c17ff7 100644 --- a/src/node-repl/lib/trust.ts +++ b/src/node-repl/lib/trust.ts @@ -1,9 +1,9 @@ import { existsSync, readFileSync, realpathSync } from "node:fs"; import { isBuiltin } from "node:module"; -import { join, sep } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { sep } from "node:path"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** * What a REPL turn may `import()`. Read once when the worker starts, never changed by a turn: * the allowlist is the one chokepoint every dynamic import passes through, so it is decided @@ -31,7 +31,7 @@ export const DEFAULT_IMPORT_POLICY: ImportPolicy = { }; export function importPolicyPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "node-repl", "trust.json"); + return genesisToolsDir("node-repl", "trust.json"); } /** diff --git a/src/question/lib/config.ts b/src/question/lib/config.ts index 2b838d2c9d..6e4a7b09a2 100644 --- a/src/question/lib/config.ts +++ b/src/question/lib/config.ts @@ -1,5 +1,4 @@ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; import type { SoundChoice } from "@genesiscz/utils/audio/runner.server"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; import { env } from "@genesiscz/utils/env"; @@ -7,6 +6,7 @@ import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { atomicWriteFileSync, isJsonObject } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; export interface QuestionConfig { sinks: { obsidian: boolean; @@ -55,7 +55,7 @@ const DEFAULT: QuestionConfig = { }; export function configPath(): string { - return env.question.getConfigPath() ?? join(env.tools.getHome(), ".genesis-tools", "question", "config.json"); + return env.question.getConfigPath() ?? genesisToolsDir("question", "config.json"); } /** diff --git a/src/question/lib/decisions/read.ts b/src/question/lib/decisions/read.ts index 1cdeb712fd..060c56f10e 100644 --- a/src/question/lib/decisions/read.ts +++ b/src/question/lib/decisions/read.ts @@ -2,8 +2,8 @@ import { join } from "node:path"; import { gatherHarnessPoster } from "@genesiscz/utils/agent/runtime"; import { routerLink } from "@genesiscz/utils/browser-router/router-link"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; -import { env } from "@genesiscz/utils/env"; import { json2md } from "@genesiscz/utils/json2md"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { isTestProcess } from "@genesiscz/utils/test-process"; import { DeliveryUnknownError, NotDeliveredError } from "./deliver"; import { @@ -26,7 +26,6 @@ import { readDecisions, restoreUndelivered, } from "./store"; - /** Session id of the harness posting this process, or null for a human CLI and for tests. */ export function currentHarnessSession(): string | null { if (isTestProcess()) { @@ -42,7 +41,7 @@ export function currentHarnessSession(): string | null { return poster.sessionId; } -export function decisionFiles(root = join(env.tools.getHome(), ".genesis-tools", "question", "decisions")): { +export function decisionFiles(root = genesisToolsDir("question", "decisions")): { file: string; events: string; } { diff --git a/src/question/lib/log-store.ts b/src/question/lib/log-store.ts index 960a196b44..4917e5e256 100644 --- a/src/question/lib/log-store.ts +++ b/src/question/lib/log-store.ts @@ -2,10 +2,10 @@ import { appendFileSync, mkdirSync } from "node:fs"; import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { QaEntry } from "./types"; - export function logDir(base?: string): string { - return base ?? env.question.getLogBase() ?? join(env.tools.getHome(), ".genesis-tools", "question", "log"); + return base ?? env.question.getLogBase() ?? genesisToolsDir("question", "log"); } export function logFilePathFor(entry: Pick, base?: string): string { diff --git a/src/question/lib/pending/events.ts b/src/question/lib/pending/events.ts index 327ca02990..d65890af67 100644 --- a/src/question/lib/pending/events.ts +++ b/src/question/lib/pending/events.ts @@ -1,16 +1,15 @@ import { appendFileSync, mkdirSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { AskForm, PendingEvent, PendingEventKind } from "./types"; - /** * Lifecycle events are appended to a day-stamped JSONL, exactly like the Q→A log and the * handoff log. The dashboard tails the file, so a form created by a CLI process in another * terminal reaches the browser with no shared memory and no daemon between them. */ export function pendingEventDir(base?: string): string { - return base ?? join(env.tools.getHome(), ".genesis-tools", "question", "pending"); + return base ?? genesisToolsDir("question", "pending"); } export function pendingEventFileFor(ts: number, base?: string): string { diff --git a/src/question/lib/pending/store.ts b/src/question/lib/pending/store.ts index de5e76e917..b26770834c 100644 --- a/src/question/lib/pending/store.ts +++ b/src/question/lib/pending/store.ts @@ -1,13 +1,13 @@ import { Database } from "bun:sqlite"; import { mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { dirname } from "node:path"; import { createTranscriptAnchor, transcriptAnchorSchema } from "@genesiscz/utils/agent/source-anchor"; import { type Migration, runMigrations } from "@genesiscz/utils/database/migrations"; import { applyPragmas } from "@genesiscz/utils/database/pragmas"; import { withDatabaseReadSnapshot } from "@genesiscz/utils/database/read-snapshot"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { ANSWER_CLAIM_TTL_MS, type AskAnswer, type AskForm, type AskFormStatus, type AskItem } from "./types"; const log = logger.child({ component: "question:pending-store" }); @@ -70,7 +70,7 @@ PENDING_MIGRATIONS.push({ }); export function defaultPendingDbPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "question", "qa.db"); + return genesisToolsDir("question", "qa.db"); } export function openPendingStore(dbPath: string = defaultPendingDbPath()): Database { diff --git a/src/question/lib/transclude.ts b/src/question/lib/transclude.ts index 05aa7d736f..4a6f73b785 100644 --- a/src/question/lib/transclude.ts +++ b/src/question/lib/transclude.ts @@ -1,13 +1,13 @@ import { existsSync } from "node:fs"; -import { basename, join } from "node:path"; +import { basename } from "node:path"; import { gatherHarnessPoster } from "@genesiscz/utils/agent/runtime"; -import { env } from "@genesiscz/utils/env"; import { type ImageAttachmentInput, parseImageAttachmentInputs, readValidatedImage, } from "@genesiscz/utils/image/attachments"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { isTestProcess } from "@genesiscz/utils/test-process"; import { defaultTransclusionRegistry, @@ -19,7 +19,6 @@ import { import type { QuestionItemInput } from "./decisions/items"; import type { DecisionSource, StoredTransclusion } from "./decisions/schema"; import type { AskChoice } from "./pending/types"; - /** One token of an item, with the field it was written in (`promptMarkdown`, `reasoning`, `choices[1]`). */ export type FieldTransclusion = TransclusionToken & { field: string }; @@ -45,7 +44,7 @@ export function questionTokenRegistry(): TransclusionRegistry { /** Where `{{image}}` copies pictures, beside the decision log. */ export function questionAssetDir(): string { - return join(env.tools.getHome(), ".genesis-tools", "question", "assets"); + return genesisToolsDir("question", "assets"); } const log = logger.child({ component: "question-transclude" }); diff --git a/src/scripts/lib/store.ts b/src/scripts/lib/store.ts index 0da0e9201f..4a9a2514c9 100644 --- a/src/scripts/lib/store.ts +++ b/src/scripts/lib/store.ts @@ -19,14 +19,12 @@ * aliases keep resolving inside the mapped files (per-package tsconfig). */ import { mkdir, stat } from "node:fs/promises"; -import { homedir } from "node:os"; import { join } from "node:path"; import { ui } from "@genesiscz/utils/cli/ui"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; - /** Absolute path to this GenesisTools checkout's `src/scripts/lib/`. `import.meta.dir` is already percent-decoded, unlike `new URL(...).pathname`. */ export const LIB_DIR = import.meta.dir; @@ -46,7 +44,7 @@ export async function pathExists(path: string): Promise { /** `~/.genesis-tools/scripts`, honoring the test override GENESIS_TOOLS_HOME. */ export function storeRoot(): string { - return join(env.tools.getHome() || homedir(), ".genesis-tools", "scripts"); + return genesisToolsDir("scripts"); } export function persistedDir(root = storeRoot()): string { diff --git a/src/shops/db/ShopsDatabase.ts b/src/shops/db/ShopsDatabase.ts index 9d117dd0c2..3b92af325b 100644 --- a/src/shops/db/ShopsDatabase.ts +++ b/src/shops/db/ShopsDatabase.ts @@ -1,5 +1,4 @@ import type { Database as BunDatabase } from "bun:sqlite"; -import { join } from "node:path"; import type { RawProduct } from "@app/shops/api/ShopApiClient.types"; import { SHOPS_MIGRATIONS } from "@app/shops/db/migrations"; import type { @@ -21,11 +20,10 @@ import { type Unit, } from "@app/shops/lib/normalize"; import { createKyselyClient, type DatabaseClient } from "@genesiscz/utils/database/client"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { type Insertable, type Kysely, type Selectable, sql } from "kysely"; - export interface StartCrawlRunInput { shopOrigin: string; strategy: string; @@ -67,7 +65,7 @@ export interface ListedProduct { imageUrl?: string; } -const DEFAULT_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "shops", "index.db"); +const DEFAULT_DB_PATH = genesisToolsDir("shops", "index.db"); export class ShopsDatabase { private readonly client: DatabaseClient; diff --git a/src/shops/lib/crypto.ts b/src/shops/lib/crypto.ts index d3cf11c8d6..d86210f8cf 100644 --- a/src/shops/lib/crypto.ts +++ b/src/shops/lib/crypto.ts @@ -1,10 +1,11 @@ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; import { existsSync, linkSync, readFileSync, unlinkSync } from "node:fs"; -import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + const ALGO = "aes-256-gcm"; const KEY_BYTES = 32; const IV_BYTES = 12; @@ -13,7 +14,7 @@ const TAG_BYTES = 16; let cachedKey: Buffer | null = null; function defaultKeyPath(): string { - return env.shops.getSecretKeyPath() ?? join(env.tools.getHome(), ".genesis-tools", "shops", ".secret-key"); + return env.shops.getSecretKeyPath() ?? genesisToolsDir("shops", ".secret-key"); } function readKey(path: string): Buffer { diff --git a/src/stash/lib/storage.ts b/src/stash/lib/storage.ts index 88cf651bb6..49914ee5fb 100644 --- a/src/stash/lib/storage.ts +++ b/src/stash/lib/storage.ts @@ -2,12 +2,13 @@ import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; export class StashStorage { private readonly base: string; constructor(base?: string) { const envRoot = env.getTrimmed("GENESIS_TOOLS_STASH_ROOT"); - this.base = base ?? envRoot ?? join(env.tools.getHome(), ".genesis-tools", "stash"); + this.base = base ?? envRoot ?? genesisToolsDir("stash"); } root(): string { diff --git a/src/task/lib/test-harness.ts b/src/task/lib/test-harness.ts index 711232bb32..079d72570f 100644 --- a/src/task/lib/test-harness.ts +++ b/src/task/lib/test-harness.ts @@ -3,6 +3,8 @@ import { fileURLToPath } from "node:url"; import { env } from "@genesiscz/utils/env"; import { filterLineRecords, readJsonlFile } from "@genesiscz/utils/log-session/jsonl-reader"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; + const REPO_ROOT = fileURLToPath(new URL("../../..", import.meta.url)); export interface RunTaskCaptureOptions { @@ -48,7 +50,7 @@ export async function runTaskCli(args: string[], opts: { homeDir: string }): Pro } export async function readTaskJsonl(session: string, homeDir: string) { - const path = join(homeDir, ".genesis-tools", "task", "sessions", `${session}.jsonl`); + const path = genesisToolsDirIn(homeDir, "task", "sessions", `${session}.jsonl`); const records = await readJsonlFile(path); return filterLineRecords(records); } diff --git a/src/task/tests/task-integration-env.ts b/src/task/tests/task-integration-env.ts index 36f9c3da25..0cc3442dc7 100644 --- a/src/task/tests/task-integration-env.ts +++ b/src/task/tests/task-integration-env.ts @@ -6,6 +6,8 @@ import { join, resolve } from "node:path"; import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; + const TASK_TOOL = resolve(import.meta.dir, "../../../tools"); export interface TaskRunResult { @@ -54,7 +56,7 @@ export function setupTaskIntegrationHome(): TaskIntegrationEnv { env: { ...childEnv, ...opts?.env }, }); }, - sessionsDir: () => join(homeDir, ".genesis-tools", "task", "sessions"), + sessionsDir: () => genesisToolsDirIn(homeDir, "task", "sessions"), clean, }; } diff --git a/src/telegram/lib/AssistantEngine.ts b/src/telegram/lib/AssistantEngine.ts index 541f66f979..2acb5c5528 100644 --- a/src/telegram/lib/AssistantEngine.ts +++ b/src/telegram/lib/AssistantEngine.ts @@ -1,6 +1,5 @@ -import { resolve } from "node:path"; import { AIChat } from "@app/ask/index.lib"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { z } from "zod"; import { parseDate } from "./DateParser"; import type { TelegramHistoryStore } from "./TelegramHistoryStore"; @@ -73,7 +72,7 @@ export class AssistantEngine { temperature: config.temperature ?? 0.7, session: { id: `telegram-assistant-${this.contact.userId}`, - dir: resolve(env.tools.getHome(), ".genesis-tools", "telegram", "ai-sessions"), + dir: genesisToolsDir("telegram", "ai-sessions"), autoSave: true, }, }); diff --git a/src/telegram/lib/AttachmentDownloader.ts b/src/telegram/lib/AttachmentDownloader.ts index 05a56c8611..8229302774 100644 --- a/src/telegram/lib/AttachmentDownloader.ts +++ b/src/telegram/lib/AttachmentDownloader.ts @@ -1,11 +1,11 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; import type { TelegramHistoryStore } from "./TelegramHistoryStore"; import type { TGClient } from "./TGClient"; -const ATTACHMENTS_BASE = resolve(env.tools.getHome(), ".genesis-tools/telegram/chats"); +const ATTACHMENTS_BASE = toolDataDir("telegram", "chats"); export class AttachmentDownloader { constructor( diff --git a/src/telegram/lib/TelegramHistoryStore.ts b/src/telegram/lib/TelegramHistoryStore.ts index 0f22179dbd..fbbad39f98 100644 --- a/src/telegram/lib/TelegramHistoryStore.ts +++ b/src/telegram/lib/TelegramHistoryStore.ts @@ -1,10 +1,10 @@ import { Database } from "bun:sqlite"; import { existsSync, mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { dirname } from "node:path"; import { logger } from "@genesiscz/utils/logger"; import { SearchEngine } from "@genesiscz/utils/search"; import { vectorSearch } from "@genesiscz/utils/search/drivers/sqlite-fts5/vector"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import type { SerializedMessage } from "./TelegramMessage"; import type { AttachmentRow, @@ -26,7 +26,7 @@ import type { UpsertMessageInput, } from "./types"; -const DB_PATH = join(env.tools.getHome(), ".genesis-tools", "telegram", "history.db"); +const DB_PATH = genesisToolsDir("telegram", "history.db"); /** MessageRow with index signature so it satisfies Record for SearchEngine */ type IndexableMessageRow = MessageRow & Record; diff --git a/src/telegram/lib/actions/ask.ts b/src/telegram/lib/actions/ask.ts index f4b022ebc1..a009561e2a 100644 --- a/src/telegram/lib/actions/ask.ts +++ b/src/telegram/lib/actions/ask.ts @@ -1,8 +1,7 @@ -import { resolve } from "node:path"; import { AIChat } from "@ask/AIChat"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { toolDataDir } from "@genesiscz/utils/storage/root"; import type { ActionHandler } from "../types"; /** Cache of AIChat instances per contact */ @@ -33,7 +32,7 @@ export const handleAsk: ActionHandler = async (message, contact, client, convers logLevel: "silent", session: { id: `telegram-${contact.userId}`, - dir: resolve(env.tools.getHome(), ".genesis-tools/telegram/ai-sessions"), + dir: toolDataDir("telegram", "ai-sessions"), autoSave: true, }, }); diff --git a/src/todo/lib/store.ts b/src/todo/lib/store.ts index b3a8482fbc..6ec12375f4 100644 --- a/src/todo/lib/store.ts +++ b/src/todo/lib/store.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { copyFileSync, existsSync, mkdirSync, readdirSync, rmSync } from "node:fs"; import { basename, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { withFileLock } from "@genesiscz/utils/storage/file-lock"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; import { nanoid } from "nanoid"; import { captureContext } from "./context"; @@ -24,7 +24,7 @@ function projectHash(projectRoot: string): string { } function defaultStorageRoot(): string { - return join(env.tools.getHome(), ".genesis-tools", "todo"); + return genesisToolsDir("todo"); } function applyFilters(todos: Todo[], filters: TodoFilters): Todo[] { diff --git a/src/ts/README.md b/src/ts/README.md index 7ee9ab210f..b3a47c60e2 100644 --- a/src/ts/README.md +++ b/src/ts/README.md @@ -174,11 +174,11 @@ GenesisTools knows nothing about the project. The caller passes the deployed bas | `sourcemaps find-commit --deployed --repo --range A..B --build-cmd --output` | The deployed commit, from one reference build (below) | | `sourcemaps check --deployed --build ` | How often `resolve` is right on THIS deployment, with its string literals as ground truth | | `sourcemaps carry --deployed --build --out ` | A map for every deployed chunk, carried from the build; `tools chrome-devtools intercept --map-dir ` shows them in DevTools | -| `sourcemaps cache list\|prune --project ` | What the cache holds; prune by age and remove folders of failed builds | +| `sourcemaps cache list\|prune --project ` (prune: or `--all`) | What the cache holds; prune by age, remove folders of failed builds and a build worktree unused for an hour, and print what it freed | **find-commit** (`--strategy tokens`, the default) builds one reference commit and ranks every first-parent commit of the range by how closely the strings and property names it would put in the bundle match the deployed ones; no build per commit. Commits that produce the same strings tie, and the oldest of the tie introduced the deployed code; one more build confirms it (`verified`). It also reports, as evidence only: commit shas the bundle names, the newest `Last-Modified` of the deployed files (commits after it cannot be the build), and inlined env values the build did not get. `--strategy rebuild` builds the commits that touched the differing files instead. -**Cache.** `~/.genesis-tools/ts/sourcemaps///` with an index in `/metadata.json` (kind, origin, size, last use). A deployment snapshot is keyed by its asset list, a build by commit + command + output folder, so a second run reuses both. Maps fetched by debug id are kept in `/debug-id-maps/`. `--out ` exposes an entry as a symlink. Every command that writes the cache first removes entries unused for 30 days, and folders a failed build left (`cache prune --max-age-days` to do it by hand). Builds run in one managed worktree per project (`/worktree`) unless `--worktree` names one; `--link .npmrc` symlinks untracked files the build needs. +**Cache.** `~/.genesis-tools/ts/sourcemaps///` with an index in `/metadata.json` (kind, origin, size, last use). A deployment snapshot is keyed by its asset list, a build by commit + command + output folder, so a second run reuses both. Maps fetched by debug id are kept in `/debug-id-maps/`. `--out ` exposes an entry as a symlink. Every command that writes the cache first removes entries unused for 30 days, and folders a failed build left (`cache prune --max-age-days` to do it by hand). Builds run in one managed worktree per project (`/worktree`) unless `--worktree` names one; `--link .npmrc` symlinks untracked files the build needs. The installed dependencies make that worktree gigabytes for a large app, so a command that built (`build`, `find-commit`, `rank`, `resolve`, `check`, `carry`) removes it with `git worktree remove` when it ends, also after a failure or Ctrl-C. `--keep-worktree` keeps it for a faster next build: on a small fixture (13 packages, 78 MB) a build took about 0.5 s with a fresh install and 0.2 s with the kept one. A worktree is never removed while a build runs in it, while it is locked with `git worktree lock`, while a process has its working directory inside it, or when it has tracked changes; a worktree named with `--worktree` is never removed. Profiling: `PROFILE=sourcemaps` times each phase. The full guide, with the fastest order, every strategy's cost and its measured hit rate, is `plugins/genesis-tools/skills/chrome-devtools/references/sourcemaps.md`. diff --git a/src/ts/commands/cli.test.ts b/src/ts/commands/cli.test.ts index 99708fa5e0..d5d7c67b46 100644 --- a/src/ts/commands/cli.test.ts +++ b/src/ts/commands/cli.test.ts @@ -150,3 +150,152 @@ describe("tools ts skeleton --rank", () => { expect(result.stderr).toMatch(/at least 1/); }); }); + +describe("sourcemap build interruption", () => { + for (const keepWorktree of [false, true]) { + test(`Ctrl-C stops an ${keepWorktree ? "explicit kept" : "interactively selected"} build`, () => { + const script = ` + import { mock } from "bun:test"; + import { SafeJSON } from "@genesiscz/utils/json"; + import { Command } from "commander"; + const cli = await import("@genesiscz/utils/cli"); + mock.module("@genesiscz/utils/cli", () => ({ ...cli, isInteractive: () => true })); + mock.module("@genesiscz/utils/prompts/p", () => ({ + select: async () => "command", + text: async ({ message }) => message.startsWith("Build command") ? "sleep 30" : "unused", + })); + const build = await import("@genesiscz/utils/ts/sourcemaps/build"); + let worktree; + mock.module("@genesiscz/utils/ts/sourcemaps/build", () => ({ + ...build, + buildAtCommit: async (options) => { + worktree = options.worktree; + queueMicrotask(() => process.emit("SIGINT")); + return new Promise(() => {}); + }, + releaseBuildWorktrees: async (options) => { + process.stdout.write(SafeJSON.stringify({ stop: options?.stop, worktree })); + return []; + }, + })); + const { registerSourcemapsCommands } = await import(${SafeJSON.stringify(join(import.meta.dir, "sourcemaps.ts"))}); + const program = new Command(); + registerSourcemapsCommands(program); + await program.parseAsync([ + "bun", "ts", "sourcemaps", "build", "--repo", "/fixture", "--commit", "HEAD", + "--output", "out", "--project", "interrupt-fixture", + ...${SafeJSON.stringify(keepWorktree ? ["--build-cmd", "sleep 30", "--keep-worktree"] : [])}, + ]); + `; + const result = Bun.spawnSync(["bun", "-e", script], { + env: process.env, + stdout: "pipe", + stderr: "pipe", + timeout: 3000, + }); + const stdout = new TextDecoder().decode(result.stdout); + const stderr = new TextDecoder().decode(result.stderr); + expect({ code: result.exitCode, stderr }).toMatchObject({ code: 130 }); + const release = SafeJSON.parse(stdout) as { stop: boolean; worktree?: string }; + expect(release.stop).toBe(true); + + if (keepWorktree) { + expect(release.worktree).toMatch(/interrupt-fixture[/\\]worktree$/); + } else { + expect(release.worktree).toBeUndefined(); + } + }); + } +}); + +describe("sourcemap cache prune --all", () => { + for (const json of [false, true]) { + test(`reports failed removals but not expected retention (${json ? "JSON" : "text"})`, () => { + const script = ` + import { mock } from "bun:test"; + import { Command } from "commander"; + const cache = await import("@genesiscz/utils/ts/sourcemaps/cache"); + mock.module("@genesiscz/utils/ts/sourcemaps/cache", () => ({ + ...cache, + cacheProjects: async () => ["broken", "busy", "last"], + SourcemapCache: class { + constructor(project) { this.project = project; } + async prune() { + return { removed: this.project === "last" ? [{ name: "old", reason: "unused", bytes: 10, diskBytes: 10 }] : [], kept: this.project === "last" ? [] : [{ + worktree: "/fixture/" + this.project, + outcome: this.project === "broken" ? "failed" : "busy", + bytes: 0, + detail: this.project === "broken" ? "git removal failed" : "a build is running", + }] }; + } + }, + })); + const { registerSourcemapsCommands } = await import(${SafeJSON.stringify(join(import.meta.dir, "sourcemaps.ts"))}); + const program = new Command(); + registerSourcemapsCommands(program); + await program.parseAsync(["bun", "ts", "sourcemaps", "cache", "prune", "--all", ...${SafeJSON.stringify(json ? ["--json"] : [])}]); + `; + const result = Bun.spawnSync(["bun", "-e", script], { + env: process.env, + stdout: "pipe", + stderr: "pipe", + timeout: 5000, + }); + const stdout = new TextDecoder().decode(result.stdout); + const stderr = new TextDecoder().decode(result.stderr); + expect(result.exitCode).toBe(1); + if (json) { + expect(SafeJSON.parse(stdout)).toMatchObject({ + projects: [{ project: "broken" }, { project: "busy" }, { project: "last" }], + failures: [{ project: "broken", error: "/fixture/broken: git removal failed" }], + }); + } else { + expect(stdout).toContain("last"); + expect(stderr).toContain("could not prune broken: /fixture/broken: git removal failed"); + expect(stderr).not.toContain("could not prune busy"); + } + }); + test(`reports successes and failures without aborting later projects (${json ? "JSON" : "text"})`, () => { + const script = ` + import { mock } from "bun:test"; + import { Command } from "commander"; + const cache = await import("@genesiscz/utils/ts/sourcemaps/cache"); + mock.module("@genesiscz/utils/ts/sourcemaps/cache", () => ({ + ...cache, + cacheProjects: async () => ["first", "broken", "last"], + SourcemapCache: class { + constructor(project) { this.project = project; } + async prune() { + if (this.project === "broken") throw new Error("fixture prune failed"); + return { removed: [{ name: "old", reason: "unused", bytes: 10, diskBytes: 10 }], kept: [] }; + } + }, + })); + const { registerSourcemapsCommands } = await import(${SafeJSON.stringify(join(import.meta.dir, "sourcemaps.ts"))}); + const program = new Command(); + registerSourcemapsCommands(program); + await program.parseAsync(["bun", "ts", "sourcemaps", "cache", "prune", "--all", ...${SafeJSON.stringify(json ? ["--json"] : [])}]); + `; + const result = Bun.spawnSync(["bun", "-e", script], { + env: process.env, + stdout: "pipe", + stderr: "pipe", + timeout: 5000, + }); + const stdout = new TextDecoder().decode(result.stdout); + const stderr = new TextDecoder().decode(result.stderr); + expect(result.exitCode).toBe(1); + if (json) { + expect(SafeJSON.parse(stdout)).toMatchObject({ + projects: [{ project: "first" }, { project: "last" }], + failures: [{ project: "broken", error: "fixture prune failed" }], + freedBytes: 20, + }); + } else { + expect(stdout).toContain("first"); + expect(stdout).toContain("last"); + expect(stderr).toContain("could not prune broken: fixture prune failed"); + } + }); + } +}); diff --git a/src/ts/commands/sourcemaps.ts b/src/ts/commands/sourcemaps.ts index 9da0561f45..68c652f504 100644 --- a/src/ts/commands/sourcemaps.ts +++ b/src/ts/commands/sourcemaps.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { basename, join, resolve } from "node:path"; import { isInteractive, suggestCommand } from "@genesiscz/utils/cli"; import { pickEnumFlag } from "@genesiscz/utils/cli/enum-flag"; +import { INTERRUPTED_EXIT_CODE, withInterrupt } from "@genesiscz/utils/cli/interrupt"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; import { env } from "@genesiscz/utils/env"; import { formatBytes } from "@genesiscz/utils/format"; @@ -10,8 +11,21 @@ import { logger } from "@genesiscz/utils/logger"; import { completeCaChain } from "@genesiscz/utils/net/ca-chain"; import * as p from "@genesiscz/utils/prompts/p"; import { createBoxTable, renderCliHeader, truncateDisplay } from "@genesiscz/utils/table"; -import { buildAtCommit, buildFilesDir, git } from "@genesiscz/utils/ts/sourcemaps/build"; -import { DEFAULT_MAX_AGE_DAYS, projectSlug, SourcemapCache } from "@genesiscz/utils/ts/sourcemaps/cache"; +import { + buildAtCommit, + buildFilesDir, + git, + managedWorktree, + releaseBuildWorktrees, + type WorktreeRemoval, +} from "@genesiscz/utils/ts/sourcemaps/build"; +import { + cacheProjects, + DEFAULT_MAX_AGE_DAYS, + type PruneResult, + projectSlug, + SourcemapCache, +} from "@genesiscz/utils/ts/sourcemaps/cache"; import { calibrate } from "@genesiscz/utils/ts/sourcemaps/calibrate"; import { carryMaps } from "@genesiscz/utils/ts/sourcemaps/carry"; import { @@ -86,6 +100,7 @@ interface BuildOpts extends CommonOpts { env?: string[]; deployedEnv?: boolean; worktree?: string; + keepWorktree?: boolean; buildCommit?: string; deployedCommit?: string; } @@ -307,6 +322,125 @@ function mapStatusLine(snap: { files: DeployedFile[] }): string { return [...counts.entries()].map(([status, count]) => `${count} ${status}`).join(" · "); } +/** + * Wraps every action that can build, including interactive selection. Ctrl-C stops the build before cleanup. + * `--keep-worktree` names the managed worktree explicitly, retaining its directory without bypassing cancellation. + */ +function releasesWorktree(action: (...args: A) => Promise): (...args: A) => Promise { + return async (...args: A) => { + const ending = await withInterrupt(async (signal) => { + const interrupted = new Promise<"interrupted">((resolve) => { + signal.addEventListener("abort", () => resolve("interrupted"), { once: true }); + }); + try { + return await Promise.race([action(...args).then(() => "done" as const), interrupted]); + } finally { + printReleased(await releaseBuildWorktrees({ stop: signal.aborted })); + } + }); + if (ending === "interrupted") { + process.exit(INTERRUPTED_EXIT_CODE); + } + }; +} + +/** One line per build worktree a command released: what removing it freed, or why it stayed. */ +function printReleased(removals: WorktreeRemoval[]): void { + for (const removal of removals) { + if (removal.outcome === "removed") { + out.log.info( + `removed the build worktree ${removal.worktree} (${sizeText(removal.bytes, removal.diskBytes ?? removal.bytes)}); --keep-worktree keeps it for a faster rebuild` + ); + } else if (removal.outcome !== "absent") { + const later = + removal.outcome === "locked" || removal.outcome === "changed" + ? "" + : `; ${CMD} cache prune removes it once it is unused for an hour`; + out.log.warn(`kept the build worktree ${removal.worktree}: ${removal.detail ?? removal.outcome}${later}`); + } + } +} + +/** What removing something freed, with its size on disk when clones share part of it (`du` counts those blocks). */ +function sizeText(bytes: number, diskBytes: number): string { + return diskBytes > bytes + ? `${formatBytes(bytes)} freed of ${formatBytes(diskBytes)} on disk; the rest is shared with clones, e.g. the package manager's cache` + : `${formatBytes(bytes)} freed`; +} + +/** `cache prune`: one project's cache, or every project's with --all; prints what it freed. */ +async function pruneCaches(opts: { + project?: string; + all?: boolean; + maxAgeDays: string; + json?: boolean; +}): Promise { + const maxAgeDays = Number(opts.maxAgeDays); + if (opts.maxAgeDays.trim() === "" || !Number.isFinite(maxAgeDays) || maxAgeDays < 0) { + fail( + `--max-age-days takes a number of days, 0 or more; got '${opts.maxAgeDays}'`, + `${CMD} cache prune --all --max-age-days ${DEFAULT_MAX_AGE_DAYS}` + ); + } + + const projects = opts.all + ? await cacheProjects() + : opts.project + ? [opts.project] + : fail("cache prune needs --project or --all", `${CMD} cache prune --all`); + const results: (PruneResult & { project: string })[] = []; + const failures: { project: string; error: string }[] = []; + for (const project of projects) { + try { + const result = await new SourcemapCache(project).prune({ maxAgeDays }); + results.push({ project, ...result }); + for (const kept of result.kept) { + if (kept.outcome === "failed") { + failures.push({ project, error: `${kept.worktree}: ${kept.detail ?? "worktree removal failed"}` }); + } + } + } catch (error) { + logger.warn({ error, project }, "could not prune sourcemaps cache"); + failures.push({ project, error: error instanceof Error ? error.message : String(error) }); + } + } + + if (failures.length > 0) { + process.exitCode = 1; + } + + const items = results.flatMap((result) => result.removed.map((item) => ({ project: result.project, ...item }))); + const freed = items.reduce((sum, item) => sum + item.bytes, 0); + const onDisk = items.reduce((sum, item) => sum + item.diskBytes, 0); + if (opts.json) { + out.result({ projects: results, failures, freedBytes: freed, diskBytes: onDisk }); + return; + } + + if (items.length > 0) { + const table = createBoxTable(["PROJECT", "REMOVED", "WHY", "ON DISK", "FREED"]); + for (const item of items) { + table.push([item.project, item.name, item.reason, formatBytes(item.diskBytes), formatBytes(item.bytes)]); + } + + out.println(table.toString()); + } + + out.println( + items.length > 0 + ? `${sizeText(freed, onDisk)}: ${items.length} item(s) across ${projects.length} project(s)` + : `nothing to prune in ${projects.length} project(s)` + ); + for (const result of results) { + for (const kept of result.kept) { + out.println(pc.yellow(`⚠ ${result.project}: kept ${kept.worktree}: ${kept.detail ?? kept.outcome}`)); + } + } + for (const failure of failures) { + out.log.error(`could not prune ${failure.project}: ${failure.error}`); + } +} + /** D5: a finished folder, or the caller's own build command at a commit. Asks in a TTY when neither is given. */ async function localBuilds( cache: SourcemapCache, @@ -362,7 +496,7 @@ async function localBuilds( outputDir: opts.output, links: opts.link, env: await buildEnv(opts, snap), - worktree: opts.worktree ? resolve(opts.worktree) : undefined, + worktree: opts.worktree ? resolve(opts.worktree) : opts.keepWorktree ? managedWorktree(cache) : undefined, onLine: (line) => spinner.message(line.slice(0, 100)), }); spinner.stop( @@ -454,7 +588,11 @@ function withBuildFlags(cmd: Command): Command { "--deployed-env", "also pass the env values the deployment inlined (what `env` prints); an explicit --env wins" ) - .option("--worktree ", "build in this worktree instead of the managed one under the cache") + .option("--worktree ", "build in this worktree instead of the managed one under the cache (never removed)") + .option( + "--keep-worktree", + "keep the managed build worktree and its installed dependencies for a faster next build (default: removed when the command ends; cache prune removes it after an hour unused)" + ) .option("--build-commit ", "the commit a --build folder was made from (enables line translation)") .option( "--deployed-commit ", @@ -509,42 +647,44 @@ Examples: ${CMD} resolve https://app.example.com/static/js/710.f9e49e56.chunk.js:95812 --deployed https://app.example.com/ --build ./web/build ${CMD} resolve : --deployed --repo . --commit v1.2.3 --build-cmd 'bun install && bun run build' --output web/build` ) - .action(async (positions: string[], opts: DeployedOpts & BuildOpts & DebugIdOpts) => { - const requests = positions.map((ref) => { - try { - return parsePositionRef(ref); - } catch (error) { - return fail(error instanceof Error ? error.message : String(error)); + .action( + releasesWorktree(async (positions: string[], opts: DeployedOpts & BuildOpts & DebugIdOpts) => { + const requests = positions.map((ref) => { + try { + return parsePositionRef(ref); + } catch (error) { + return fail(error instanceof Error ? error.message : String(error)); + } + }); + // A --deployed-dir snapshot still answers https:// positions: its files get URLs under this origin. + if (/^https?:/.test(requests[0].url)) { + opts.deployed ??= new URL(requests[0].url).origin; } - }); - // A --deployed-dir snapshot still answers https:// positions: its files get URLs under this origin. - if (/^https?:/.test(requests[0].url)) { - opts.deployed ??= new URL(requests[0].url).origin; - } - const cache = new SourcemapCache(projectOf(opts)); - const snap = await deployedSource(cache, opts); - const resolvers = await resolverChain(cache, snap, opts); - const results: ChainResult[] = []; - for (const request of requests) { - results.push(await resolvePosition(request, resolvers)); - } + const cache = new SourcemapCache(projectOf(opts)); + const snap = await deployedSource(cache, opts); + const resolvers = await resolverChain(cache, snap, opts); + const results: ChainResult[] = []; + for (const request of requests) { + results.push(await resolvePosition(request, resolvers)); + } - // A script that resolves a position must be able to tell "not found" from an answer by the exit code. - if (results.some((chain) => !chain.result)) { - process.exitCode = 1; - } + // A script that resolves a position must be able to tell "not found" from an answer by the exit code. + if (results.some((chain) => !chain.result)) { + process.exitCode = 1; + } - if (opts.json) { - out.result(results); - return; - } + if (opts.json) { + out.result(results); + return; + } - out.println(pc.dim(`published maps: ${mapStatusLine(snap)}`)); - for (const chain of results) { - printChain(chain); - } - }); + out.println(pc.dim(`published maps: ${mapStatusLine(snap)}`)); + for (const chain of results) { + printChain(chain); + } + }) + ); withDeployedFlags(group.command("fetch")) .description("snapshot a deployment's JS/CSS into the cache and preflight every file's published .map") @@ -580,47 +720,62 @@ far as these numbers say. Literal-heavy code is easier than average: read it as Example: ${CMD} check --deployed https://app.example.com/ --build ./web/build --files 40` ) - .action(async (opts: DeployedOpts & BuildOpts & DebugIdOpts & { perFile: string; files?: string }) => { - const cache = new SourcemapCache(projectOf(opts)); - const snap = await deployedSource(cache, opts); - const resolvers = await resolverChain(cache, snap, opts); - const result = await calibrate({ - deployed: snap, - resolvers, - perFile: Number(opts.perFile), - maxFiles: opts.files ? Number(opts.files) : undefined, - }); - if (opts.json) { - out.result(result); - return; - } + .action( + releasesWorktree( + async (opts: DeployedOpts & BuildOpts & DebugIdOpts & { perFile: string; files?: string }) => { + const cache = new SourcemapCache(projectOf(opts)); + const snap = await deployedSource(cache, opts); + const resolvers = await resolverChain(cache, snap, opts); + const result = await calibrate({ + deployed: snap, + resolvers, + perFile: Number(opts.perFile), + maxFiles: opts.files ? Number(opts.files) : undefined, + }); + if (opts.json) { + out.result(result); + return; + } - const pct = (part: number, whole: number) => (whole === 0 ? "—" : `${((100 * part) / whole).toFixed(1)}%`); - const table = createBoxTable(["CONFIDENCE", "SAMPLES", "NEAR (±3 LINES)", "ELSEWHERE", "NO SOURCE TEXT"]); - for (const [confidence, tier] of Object.entries(result.byConfidence)) { - table.push([ - confidence, - String(tier.samples), - pct(tier.near, tier.samples - tier.unverifiable), - String(tier.elsewhere), - String(tier.unverifiable), - ]); - } + const pct = (part: number, whole: number) => + whole === 0 ? "—" : `${((100 * part) / whole).toFixed(1)}%`; + const table = createBoxTable([ + "CONFIDENCE", + "SAMPLES", + "NEAR (±3 LINES)", + "ELSEWHERE", + "NO SOURCE TEXT", + ]); + for (const [confidence, tier] of Object.entries(result.byConfidence)) { + table.push([ + confidence, + String(tier.samples), + pct(tier.near, tier.samples - tier.unverifiable), + String(tier.elsewhere), + String(tier.unverifiable), + ]); + } - out.println(table.toString()); - out.println( - `${result.samples} literals in ${result.files} files, ${result.unresolved} unresolved (${pct(result.unresolved, result.samples)}), ${(result.ms / 1000).toFixed(1)} s` - ); - for (const reason of result.reasons.slice(0, 3)) { - out.println(pc.yellow(` unresolved ${reason.count}x: ${truncateDisplay(reason.example, 160)}`)); - } + out.println(table.toString()); + out.println( + `${result.samples} literals in ${result.files} files, ${result.unresolved} unresolved (${pct(result.unresolved, result.samples)}), ${(result.ms / 1000).toFixed(1)} s` + ); + for (const reason of result.reasons.slice(0, 3)) { + out.println( + pc.yellow(` unresolved ${reason.count}x: ${truncateDisplay(reason.example, 160)}`) + ); + } - for (const miss of result.misses.slice(0, 5)) { - out.println( - pc.dim(` miss: "${truncateDisplay(miss.literal, 40)}" -> ${miss.answer} (${miss.confidence})`) - ); - } - }); + for (const miss of result.misses.slice(0, 5)) { + out.println( + pc.dim( + ` miss: "${truncateDisplay(miss.literal, 40)}" -> ${miss.answer} (${miss.confidence})` + ) + ); + } + } + ) + ); withBuildFlags(withDeployedFlags(group.command("carry"))) .description("write a source map for every deployed chunk, carried over from a local build, for DevTools") @@ -640,42 +795,44 @@ Then let DevTools show the real sources while you debug the deployed page: Example: ${CMD} carry --deployed https://app.example.com/ --build ./web/build --out ./carried-maps` ) - .action(async (opts: DeployedOpts & BuildOpts & { out: string }) => { - const cache = new SourcemapCache(projectOf(opts)); - const snap = await deployedSource(cache, opts); - const builds = await localBuilds(cache, opts, snap); - const build = builds[0]; - if (!build) { - return fail("carry needs a local build: --build , or --repo/--commit/--build-cmd/--output"); - } + .action( + releasesWorktree(async (opts: DeployedOpts & BuildOpts & { out: string }) => { + const cache = new SourcemapCache(projectOf(opts)); + const snap = await deployedSource(cache, opts); + const builds = await localBuilds(cache, opts, snap); + const build = builds[0]; + if (!build) { + return fail("carry needs a local build: --build , or --repo/--commit/--build-cmd/--output"); + } - const outDir = resolve(opts.out); - await mkdir(outDir, { recursive: true }); - const result = await carryMaps({ deployed: snap, build, outDir }); - if (opts.json) { - out.result({ outDir, ...result }); - return; - } + const outDir = resolve(opts.out); + await mkdir(outDir, { recursive: true }); + const result = await carryMaps({ deployed: snap, build, outDir }); + if (opts.json) { + out.result({ outDir, ...result }); + return; + } - const segments = result.files.reduce((sum, file) => sum + file.segments, 0); - const carried = result.files.reduce((sum, file) => sum + file.carried, 0); - out.println( - `${result.files.length} maps written to ${outDir} (${((100 * carried) / Math.max(1, segments)).toFixed(1)}% of mappings carried), ${result.skipped.length} deployed file(s) without a local counterpart, ${(result.ms / 1000).toFixed(1)} s` - ); - for (const file of result.files.filter((entry) => entry.carried < entry.segments * 0.9).slice(0, 8)) { + const segments = result.files.reduce((sum, file) => sum + file.segments, 0); + const carried = result.files.reduce((sum, file) => sum + file.carried, 0); out.println( - pc.yellow( - ` ⚠ ${file.path}: only ${((100 * file.carried) / file.segments).toFixed(0)}% carried (the chunk differs from the build)` - ) + `${result.files.length} maps written to ${outDir} (${((100 * carried) / Math.max(1, segments)).toFixed(1)}% of mappings carried), ${result.skipped.length} deployed file(s) without a local counterpart, ${(result.ms / 1000).toFixed(1)} s` ); - } + for (const file of result.files.filter((entry) => entry.carried < entry.segments * 0.9).slice(0, 8)) { + out.println( + pc.yellow( + ` ⚠ ${file.path}: only ${((100 * file.carried) / file.segments).toFixed(0)}% carried (the chunk differs from the build)` + ) + ); + } - out.println( - pc.dim( - `DevTools: ${toolCommand("chrome-devtools", "intercept", "--match", "", "--map-dir", outDir)}` - ) - ); - }); + out.println( + pc.dim( + `DevTools: ${toolCommand("chrome-devtools", "intercept", "--match", "", "--map-dir", outDir)}` + ) + ); + }) + ); withDeployedFlags(group.command("env")) .description( @@ -728,59 +885,72 @@ Only env OBJECTS are recoverable: a value a define replaced one by one has lost .option("--project ", "cache folder name (default: the --repo folder name)") .option("--out ", "also expose the build output here (a symlink into the cache)") .option("--json", "machine-readable result") - .action(async (opts: BuildOpts & { out?: string }) => { - const cache = new SourcemapCache(projectOf(opts)); - const builds = await localBuilds(cache, { ...opts, build: [] }); - const built = builds[0]; - if (!built) { - fail("nothing to build", suggestCommand(CMD, { add: ["build", "--repo", ".", "--commit", "HEAD"] })); - } + .action( + releasesWorktree(async (opts: BuildOpts & { out?: string }) => { + const cache = new SourcemapCache(projectOf(opts)); + const builds = await localBuilds(cache, { ...opts, build: [] }); + const built = builds[0]; + if (!built) { + fail( + "nothing to build", + suggestCommand(CMD, { add: ["build", "--repo", ".", "--commit", "HEAD"] }) + ); + } - const id = basename(resolve(built.dir, "..")); - // The view is the build output itself (the folder resolve --build takes), not the entry with its log. - const link = opts.out ? await cache.linkOut(id, opts.out, "files") : null; - if (opts.json) { - out.result({ dir: built.dir, files: built.files.length, link }); - return; - } + const id = basename(resolve(built.dir, "..")); + // The view is the build output itself (the folder resolve --build takes), not the entry with its log. + const link = opts.out ? await cache.linkOut(id, opts.out, "files") : null; + if (opts.json) { + out.result({ dir: built.dir, files: built.files.length, link }); + return; + } - out.println( - `${built.label}: ${built.files.length} JS files in ${built.dir}${link ? `\nlink: ${link}` : ""}` - ); - }); + out.println( + `${built.label}: ${built.files.length} JS files in ${built.dir}${link ? `\nlink: ${link}` : ""}` + ); + }) + ); withBuildFlags(withDeployedFlags(group.command("rank"))) .description( "score how much of the deployed code each local build contains (which build matches the deployment)" ) - .action(async (opts: DeployedOpts & BuildOpts) => { - const cache = new SourcemapCache(projectOf(opts)); - const snap = await deployedSource(cache, opts); - const builds = await localBuilds(cache, opts, snap); - const scores = []; - for (const build of builds) { - scores.push(await scoreBuild({ deployed: snap, build })); - } + .action( + releasesWorktree(async (opts: DeployedOpts & BuildOpts) => { + const cache = new SourcemapCache(projectOf(opts)); + const snap = await deployedSource(cache, opts); + const builds = await localBuilds(cache, opts, snap); + const scores = []; + for (const build of builds) { + scores.push(await scoreBuild({ deployed: snap, build })); + } - scores.sort((a, b) => a.deployedOnly.length - b.deployedOnly.length || b.rate - a.rate); - if (opts.json) { - out.result(scores.map(({ chunks: _chunks, ...rest }) => rest)); - return; - } + scores.sort((a, b) => a.deployedOnly.length - b.deployedOnly.length || b.rate - a.rate); + if (opts.json) { + out.result(scores.map(({ chunks: _chunks, ...rest }) => rest)); + return; + } - const table = createBoxTable(["BUILD", "CODE FOUND", "MISSING CHUNKS", "EXTRA CHUNKS", "SAME DEBUG ID"]); - for (const score of scores) { - table.push([ - score.label, - `${(100 * score.rate).toFixed(3)}%`, - String(score.deployedOnly.length), - String(score.localOnly.length), - score.debugIds.compared === 0 ? "—" : `${score.debugIds.equal}/${score.debugIds.compared}`, + const table = createBoxTable([ + "BUILD", + "CODE FOUND", + "MISSING CHUNKS", + "EXTRA CHUNKS", + "SAME DEBUG ID", ]); - } + for (const score of scores) { + table.push([ + score.label, + `${(100 * score.rate).toFixed(3)}%`, + String(score.deployedOnly.length), + String(score.localOnly.length), + score.debugIds.compared === 0 ? "—" : `${score.debugIds.equal}/${score.debugIds.compared}`, + ]); + } - out.println(table.toString()); - }); + out.println(table.toString()); + }) + ); withBuildFlags(withDeployedFlags(group.command("find-commit"))) .description( @@ -811,173 +981,184 @@ Examples: --build-cmd 'bun run build' --output dist --strategy rebuild` ) .action( - async ( - opts: DeployedOpts & - BuildOpts & { - range: string; - base?: string; - strategy?: string | boolean; - verify?: boolean; - maxBuilds: string; - allParents?: boolean; + releasesWorktree( + async ( + opts: DeployedOpts & + BuildOpts & { + range: string; + base?: string; + strategy?: string | boolean; + verify?: boolean; + maxBuilds: string; + allParents?: boolean; + } + ) => { + if (!opts.repo || !opts.buildCmd || !opts.output) { + fail("find-commit needs --repo, --build-cmd and --output"); } - ) => { - if (!opts.repo || !opts.buildCmd || !opts.output) { - fail("find-commit needs --repo, --build-cmd and --output"); - } - const strategy = await pickEnumFlag({ - tool: CMD, - subcommand: ["find-commit"], - flag: "--strategy", - given: opts.strategy, - values: FIND_COMMIT_STRATEGIES, - fallback: "tokens", - accepts: (value): value is FindCommitStrategy => - (FIND_COMMIT_STRATEGIES as readonly string[]).includes(value), - }); - if (!strategy) { - return; - } + const strategy = await pickEnumFlag({ + tool: CMD, + subcommand: ["find-commit"], + flag: "--strategy", + given: opts.strategy, + values: FIND_COMMIT_STRATEGIES, + fallback: "tokens", + accepts: (value): value is FindCommitStrategy => + (FIND_COMMIT_STRATEGIES as readonly string[]).includes(value), + }); + if (!strategy) { + return; + } - const cache = new SourcemapCache(projectOf(opts)); - const deployed = await deployedSource(cache, opts); - const spinner = out.spinner(); - const started = performance.now(); - spinner.start("building the reference"); - const result = await findDeployedCommit({ - cache, - deployed, - repo: resolve(opts.repo), - range: opts.range, - base: opts.base, - strategy, - verify: opts.verify, - buildCmd: opts.buildCmd, - outputDir: opts.output, - links: opts.link, - env: await buildEnv(opts, deployed), - worktree: opts.worktree ? resolve(opts.worktree) : undefined, - firstParent: !opts.allParents, - maxBuilds: Number(opts.maxBuilds), - onEvent: (event) => { - if (event.type === "build-start") { - spinner.message(`building ${event.commit.slice(0, 12)}`); - } else if (event.type === "ranked") { - spinner.message(`ranked by ${event.tokens} tokens; confirming the best`); - } else if (event.type === "changed-files") { - spinner.message( - `${event.files.length} source files differ; asking git which commits touched them` - ); - } else if (event.type === "scored") { - spinner.message(`${event.commit.slice(0, 12)}: ${(100 * event.rate).toFixed(3)}%`); + const cache = new SourcemapCache(projectOf(opts)); + const deployed = await deployedSource(cache, opts); + const spinner = out.spinner(); + const started = performance.now(); + spinner.start("building the reference"); + const result = await findDeployedCommit({ + cache, + deployed, + repo: resolve(opts.repo), + range: opts.range, + base: opts.base, + strategy, + verify: opts.verify, + buildCmd: opts.buildCmd, + outputDir: opts.output, + links: opts.link, + env: await buildEnv(opts, deployed), + worktree: opts.worktree + ? resolve(opts.worktree) + : opts.keepWorktree + ? managedWorktree(cache) + : undefined, + firstParent: !opts.allParents, + maxBuilds: Number(opts.maxBuilds), + onEvent: (event) => { + if (event.type === "build-start") { + spinner.message(`building ${event.commit.slice(0, 12)}`); + } else if (event.type === "ranked") { + spinner.message(`ranked by ${event.tokens} tokens; confirming the best`); + } else if (event.type === "changed-files") { + spinner.message( + `${event.files.length} source files differ; asking git which commits touched them` + ); + } else if (event.type === "scored") { + spinner.message(`${event.commit.slice(0, 12)}: ${(100 * event.rate).toFixed(3)}%`); + } + }, + }); + spinner.stop(`done in ${Math.round((performance.now() - started) / 1000)} s`); + if (opts.json) { + out.result({ + ...result, + candidates: result.candidates.map(({ score: _score, ...rest }) => rest), + }); + return; + } + + renderCliHeader("Deployed commit", `${opts.range} · ${result.strategy}`); + if (result.ranking.length > 0) { + const ranking = createBoxTable(["COMMIT", "ERROR", "FILES", "SUBJECT"]); + for (const entry of result.ranking.slice(0, 10)) { + const id = entry.commit.slice(0, 12); + ranking.push([ + result.equivalent.includes(entry.commit) ? pc.green(id) : id, + String(entry.error), + `${entry.changedFiles}${entry.lockfileDiffers ? " + lock" : ""}`, + truncateDisplay(await subjectLine(opts.repo, entry.commit), 60), + ]); } - }, - }); - spinner.stop(`done in ${Math.round((performance.now() - started) / 1000)} s`); - if (opts.json) { - out.result({ ...result, candidates: result.candidates.map(({ score: _score, ...rest }) => rest) }); - return; - } - renderCliHeader("Deployed commit", `${opts.range} · ${result.strategy}`); - if (result.ranking.length > 0) { - const ranking = createBoxTable(["COMMIT", "ERROR", "FILES", "SUBJECT"]); - for (const entry of result.ranking.slice(0, 10)) { - const id = entry.commit.slice(0, 12); - ranking.push([ - result.equivalent.includes(entry.commit) ? pc.green(id) : id, - String(entry.error), - `${entry.changedFiles}${entry.lockfileDiffers ? " + lock" : ""}`, - truncateDisplay(await subjectLine(opts.repo, entry.commit), 60), - ]); + out.println(ranking.toString()); + out.println( + pc.dim( + `ERROR: how far the commit's predicted string and property-name counts are from the deployment (${result.rankingTokens} tokens). FILES: bundled source files that differ from the reference.` + ) + ); } - out.println(ranking.toString()); - out.println( - pc.dim( - `ERROR: how far the commit's predicted string and property-name counts are from the deployment (${result.rankingTokens} tokens). FILES: bundled source files that differ from the reference.` - ) - ); - } - - const builds = createBoxTable(["BUILD", "CODE FOUND", "MISSING CHUNKS", "SUBJECT"]); - for (const candidate of result.candidates) { - builds.push([ - candidate.commit === result.best?.commit - ? pc.green(candidate.commit.slice(0, 12)) - : candidate.commit.slice(0, 12), - `${(100 * candidate.rate).toFixed(3)}%`, - String(candidate.deployedOnly), - truncateDisplay(candidate.subject, 60), - ]); - } + const builds = createBoxTable(["BUILD", "CODE FOUND", "MISSING CHUNKS", "SUBJECT"]); + for (const candidate of result.candidates) { + builds.push([ + candidate.commit === result.best?.commit + ? pc.green(candidate.commit.slice(0, 12)) + : candidate.commit.slice(0, 12), + `${(100 * candidate.rate).toFixed(3)}%`, + String(candidate.deployedOnly), + truncateDisplay(candidate.subject, 60), + ]); + } - out.println(builds.toString()); - if (result.best) { - const span = - result.equivalent.length > 4 - ? `${result.equivalent.length} commits, ${result.equivalent.at(-1)?.slice(0, 12)} (oldest) .. ${result.equivalent[0]?.slice(0, 12)} (newest)` - : result.equivalent.map((commit) => commit.slice(0, 12)).join(", "); - out.println( - pc.green( - `best: ${result.best.commit.slice(0, 12)} ${truncateDisplay(result.best.subject, 60)} (${result.confidence}, ${result.buildsUsed} build(s))${result.equivalent.length > 1 ? `\nsame strings in the bundle: ${span}` : ""}` - ) - ); - } + out.println(builds.toString()); + if (result.best) { + const span = + result.equivalent.length > 4 + ? `${result.equivalent.length} commits, ${result.equivalent.at(-1)?.slice(0, 12)} (oldest) .. ${result.equivalent[0]?.slice(0, 12)} (newest)` + : result.equivalent.map((commit) => commit.slice(0, 12)).join(", "); + out.println( + pc.green( + `best: ${result.best.commit.slice(0, 12)} ${truncateDisplay(result.best.subject, 60)} (${result.confidence}, ${result.buildsUsed} build(s))${result.equivalent.length > 1 ? `\nsame strings in the bundle: ${span}` : ""}` + ) + ); + } - for (const note of result.notes) { - out.println(pc.yellow(`⚠ ${note}`)); + for (const note of result.notes) { + out.println(pc.yellow(`⚠ ${note}`)); + } } - } + ) ); group .command("cache") .description( - `list or prune the cache (~/.genesis-tools/ts/sourcemaps//); writes prune entries unused for ${DEFAULT_MAX_AGE_DAYS} days` + `list or prune the cache (~/.genesis-tools/ts/sourcemaps//); prune removes entries unused for ${DEFAULT_MAX_AGE_DAYS} days, folders failed builds left, and a build worktree unused for an hour. Every write prunes its project too` ) .argument("[action]", "list | prune", "list") - .requiredOption("--project ", "which project's cache") + .option("--project ", "which project's cache") + .option("--all", "prune every project's cache") .option("--max-age-days ", "prune entries unused for longer than this", String(DEFAULT_MAX_AGE_DAYS)) .option("--json", "machine-readable result") - .action(async (action: string, opts: { project: string; maxAgeDays: string; json?: boolean }) => { - const cache = new SourcemapCache(opts.project); - if (action === "prune") { - const maxAgeDays = Number(opts.maxAgeDays); - if (opts.maxAgeDays.trim() === "" || !Number.isFinite(maxAgeDays) || maxAgeDays < 0) { - fail( - `--max-age-days takes a number of days, 0 or more; got '${opts.maxAgeDays}'`, - `${CMD} cache prune --project ${opts.project} --max-age-days ${DEFAULT_MAX_AGE_DAYS}` - ); + .action( + async (action: string, opts: { project?: string; all?: boolean; maxAgeDays: string; json?: boolean }) => { + if (action === "prune") { + await pruneCaches(opts); + return; } - const removed = await cache.prune({ maxAgeDays }); - out.println(removed.length > 0 ? `removed ${removed.join(", ")}` : "nothing to prune"); - return; - } + if (action !== "list") { + fail(`unknown action '${action}'`, `${CMD} cache list --project `); + } - if (action !== "list") { - fail(`unknown action '${action}'`, `${CMD} cache list --project ${opts.project}`); - } + if (!opts.project) { + const known = await cacheProjects(); + return fail( + `cache list needs --project ${known.length > 0 ? `; cached projects: ${known.join(", ")}` : ""}`, + `${CMD} cache list --project ${known[0] ?? ""}` + ); + } - const entries = await cache.list(); - if (opts.json) { - out.result(entries); - return; - } + const cache = new SourcemapCache(opts.project); + const entries = await cache.list(); + if (opts.json) { + out.result(entries); + return; + } - const table = createBoxTable(["ID", "KIND", "LABEL", "SIZE", "LAST USED"]); - for (const entry of entries) { - table.push([ - entry.id, - entry.kind, - entry.label, - formatBytes(entry.bytes), - entry.lastUsedAt.slice(0, 16), - ]); - } + const table = createBoxTable(["ID", "KIND", "LABEL", "SIZE", "LAST USED"]); + for (const entry of entries) { + table.push([ + entry.id, + entry.kind, + entry.label, + formatBytes(entry.bytes), + entry.lastUsedAt.slice(0, 16), + ]); + } - out.println(`${cache.dir}\n${table.toString()}`); - }); + out.println(`${cache.dir}\n${table.toString()}`); + } + ); } diff --git a/src/utils/DashboardApp/pidFile.ts b/src/utils/DashboardApp/pidFile.ts index 51ee36646f..2bd015f157 100644 --- a/src/utils/DashboardApp/pidFile.ts +++ b/src/utils/DashboardApp/pidFile.ts @@ -14,7 +14,6 @@ import { existsSync, mkdirSync, statSync } from "node:fs"; import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { clearPidFile, inspectPidFile, @@ -24,8 +23,10 @@ import { } from "@genesiscz/utils/process/pidfile"; import type { PidIdentity } from "@genesiscz/utils/process-identity"; -const DASHBOARDS_DIR = join(env.tools.getHome(), ".genesis-tools", "dashboards"); -const LOGS_DIR = join(env.tools.getHome(), ".genesis-tools", "logs"); +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; + +const DASHBOARDS_DIR = genesisToolsDir("dashboards"); +const LOGS_DIR = genesisToolsDir("logs"); export function pidFilePath(key: string): string { return join(DASHBOARDS_DIR, `${key}.pid`); diff --git a/src/utils/agent-sessions/database.ts b/src/utils/agent-sessions/database.ts index 8be24b9251..4336acc92d 100644 --- a/src/utils/agent-sessions/database.ts +++ b/src/utils/agent-sessions/database.ts @@ -1,12 +1,13 @@ import { Database } from "bun:sqlite"; import { chmodSync, closeSync, existsSync, mkdirSync, openSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { dirname, resolve } from "node:path"; import { BaseDatabase } from "@genesiscz/utils/database/base"; import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; export function historyDatabasePath(): string { - return join(env.tools.getHome(), ".genesis-tools", "claude-history", "index.db"); + return genesisToolsDir("claude-history", "index.db"); } /** The caller owns this read-only handle. No migrations or directory creation occur. */ diff --git a/src/utils/agent-sessions/testing/baseline-oracle.ts b/src/utils/agent-sessions/testing/baseline-oracle.ts index 8fafdebb3e..6b5ea3a687 100644 --- a/src/utils/agent-sessions/testing/baseline-oracle.ts +++ b/src/utils/agent-sessions/testing/baseline-oracle.ts @@ -11,6 +11,7 @@ import { copyRepoTree } from "@genesiscz/utils/git/test-repo"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { withFileLock } from "@genesiscz/utils/storage/file-lock"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import type { HistoryFixtureWorld } from "./fixture-world"; export const BASELINE_REVISION = "010697b869a34af0e79363b5c74bfc4946f74b96"; @@ -803,7 +804,7 @@ export async function createBaselineOracle(options: { if (options.cacheHome !== undefined) { cacheHome = options.world.assertOwnedPath(options.cacheHome); await mkdir(cacheHome, { recursive: true }); - databasePath = join(cacheHome, ".genesis-tools", "claude-history", "index.db"); + databasePath = genesisToolsDirIn(cacheHome, "claude-history", "index.db"); } else if (options.cacheNamespace !== undefined) { if (!/^[a-z0-9][a-z0-9-]*$/i.test(options.cacheNamespace)) { throw new Error("Baseline cache namespace must contain only letters, digits, and hyphens"); @@ -812,7 +813,7 @@ export async function createBaselineOracle(options: { join(options.world.root, "baseline-instance-caches", options.cacheNamespace) ); await mkdir(cacheHome, { recursive: true }); - databasePath = join(cacheHome, ".genesis-tools", "claude-history", "index.db"); + databasePath = genesisToolsDirIn(cacheHome, "claude-history", "index.db"); } const environment = { ...options.world.environment, @@ -943,7 +944,7 @@ export async function createBaselineOracle(options: { environment: { ...environment, GENESIS_TOOLS_HOME: cacheHome, BASELINE_ONE_SHOT: "1" }, stdin: `${SafeJSON.stringify(request)}\n`, resultPrefix: BASELINE_RESULT_PREFIX, - databasePath: join(cacheHome, ".genesis-tools", "claude-history", "index.db"), + databasePath: genesisToolsDirIn(cacheHome, "claude-history", "index.db"), }; }, async probeFailure(message) { diff --git a/src/utils/agent-sessions/testing/fixture-world.ts b/src/utils/agent-sessions/testing/fixture-world.ts index 9f3a3cbb8a..6774667e5a 100644 --- a/src/utils/agent-sessions/testing/fixture-world.ts +++ b/src/utils/agent-sessions/testing/fixture-world.ts @@ -5,6 +5,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { env } from "@genesiscz/utils/env"; import { copyRepoTree } from "@genesiscz/utils/git/test-repo"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; export const FIXED_HISTORY_NOW = "2026-08-15T12:00:00.000Z"; export interface HistoryFixtureWorld { @@ -137,7 +138,7 @@ export async function createFixtureWorld( }; const legacyHome = join(root, "databases", "legacy-home"); const databases = { - legacy: join(legacyHome, ".genesis-tools", "claude-history", "index.db"), + legacy: genesisToolsDirIn(legacyHome, "claude-history", "index.db"), candidate: join(root, "databases", "candidate", "index.db"), }; const git = { root: join(root, "invented-repository"), head: "" }; diff --git a/src/utils/ai/local/runtimes/sherpa/paths.ts b/src/utils/ai/local/runtimes/sherpa/paths.ts index eb18f9ae91..ae91f012f6 100644 --- a/src/utils/ai/local/runtimes/sherpa/paths.ts +++ b/src/utils/ai/local/runtimes/sherpa/paths.ts @@ -1,6 +1,4 @@ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; - +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** * Where the sherpa-onnx diarization weights have always lived. It predates the * artifact store and sits under the transcribe tool rather than the ai one; @@ -14,4 +12,4 @@ import { env } from "@genesiscz/utils/env"; * `ArtifactStore.prune()` deletes under it — a sandboxed run must never point * that at the user's real weights. */ -export const DIARIZE_MODEL_DIR = join(env.tools.getHome(), ".genesis-tools", "transcribe", "models", "diarization"); +export const DIARIZE_MODEL_DIR = genesisToolsDir("transcribe", "models", "diarization"); diff --git a/src/utils/ai/openai/native-session-import.ts b/src/utils/ai/openai/native-session-import.ts index 9bda4dd8c5..b1c0840aa4 100644 --- a/src/utils/ai/openai/native-session-import.ts +++ b/src/utils/ai/openai/native-session-import.ts @@ -9,6 +9,7 @@ import type { } from "@genesiscz/utils/agent-sessions/types"; import { SafeJSON } from "@genesiscz/utils/json"; import { NETWORKED_LOCK_WAIT_MS, withFileLock } from "@genesiscz/utils/storage/file-lock"; +import { genesisToolsDirIn } from "@genesiscz/utils/storage/root"; import { z } from "zod"; const headerSchema = z.object({ @@ -100,7 +101,7 @@ export async function importNativeCodexSession( const sourceHome = await realpath(session.sourceHome ?? dirname(sourcePath)); const targetHome = await realpath(context.targetHome); const hash = digest(bytes); - const importsDir = join(targetHome, ".genesis-tools", "session-imports"); + const importsDir = genesisToolsDirIn(targetHome, "session-imports"); await mkdir(importsDir, { recursive: true, mode: 0o700 }); const recordPath = join(importsDir, `${digest(`${sourceHome}\0${session.sessionId}`)}.json`); const result = (sessionId: string, copied: boolean): NativeSessionImportResult => ({ diff --git a/src/utils/ai/proxy/AiProxyClient.ts b/src/utils/ai/proxy/AiProxyClient.ts index d26c15fdc2..0e405d9260 100644 --- a/src/utils/ai/proxy/AiProxyClient.ts +++ b/src/utils/ai/proxy/AiProxyClient.ts @@ -44,12 +44,11 @@ * those four. Fold it the moment `coreChat` grows them. */ import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { repairJson } from "@genesiscz/utils/json/repair"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; export interface AiProxyClientOptions { baseUrl?: string; apiKey?: string; @@ -164,7 +163,7 @@ const DEFAULT_PORT = 8317; * time read the user's REAL proxy key out of every one of them. */ function configPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "ai-proxy", "config.json"); + return genesisToolsDir("ai-proxy", "config.json"); } interface LocalProxyConfig { diff --git a/src/utils/ai/transcripts/turn-index.ts b/src/utils/ai/transcripts/turn-index.ts index 46b7da42a0..e7c25d6848 100644 --- a/src/utils/ai/transcripts/turn-index.ts +++ b/src/utils/ai/transcripts/turn-index.ts @@ -26,9 +26,9 @@ import { import { join } from "node:path"; import { agentProgressToSubagent } from "@genesiscz/utils/claude/session.utils"; import type { ConversationMessage, ProgressMessage } from "@genesiscz/utils/claude/types"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { claudeMessagesToTurns } from "./claude"; import type { ResolvedTranscript } from "./resolve"; import { @@ -43,7 +43,6 @@ import { terminatedOf, totalsOf, } from "./types"; - export const INDEX_MIN_BYTES = 16 * 1024 * 1024; // 4: a task result and a message typed mid-turn became turns (prompt parts), which moves the offsets. // 5: usage counts once per assistant message id (`messageUsage`), as the full parser does. @@ -100,7 +99,7 @@ export interface TurnIndexOptions { } export function turnIndexDir(): string { - return join(env.tools.getHome(), ".genesis-tools", "cache", "transcript-index"); + return genesisToolsDir("cache", "transcript-index"); } function indexFileFor(dir: string, path: string): string { diff --git a/src/utils/claude/keychain.ts b/src/utils/claude/keychain.ts index f712db98df..be9612ea3a 100644 --- a/src/utils/claude/keychain.ts +++ b/src/utils/claude/keychain.ts @@ -7,6 +7,7 @@ import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { isJsonObject } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { isTestProcess } from "@genesiscz/utils/test-process"; import { CLAUDE_CODE_CLIENT_ID, fetchOAuthProfile } from "./auth"; @@ -364,7 +365,7 @@ export async function syncKeychainToConfig(aiConfig: AIConfig): Promise { - const dir = join(env.tools.getHome(), ".genesis-tools", "claude", "keychain-backups"); + const dir = genesisToolsDir("claude", "keychain-backups"); const path = join(dir, `keychain-${new Date().toISOString().replace(/[:.]/g, "-")}.json`); await Bun.write(path, SafeJSON.stringify(payload, null, 2)); chmodSync(path, 0o600); diff --git a/src/utils/claude/subscription-auth.ts b/src/utils/claude/subscription-auth.ts index 1ad8efdf6c..0287d85762 100644 --- a/src/utils/claude/subscription-auth.ts +++ b/src/utils/claude/subscription-auth.ts @@ -1,16 +1,14 @@ import { appendFileSync, chmodSync, existsSync, readFileSync, statSync } from "node:fs"; -import { join } from "node:path"; import { retry } from "@genesiscz/utils/async"; import { toolCommand } from "@genesiscz/utils/cli/tool-command"; -import { env } from "@genesiscz/utils/env"; import { parseJSON, SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { NETWORKED_LOCK_WAIT_MS, withFileLock } from "@genesiscz/utils/storage/file-lock"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/storage"; import type { OAuthTokens } from "./auth"; import { claudeOAuth } from "./auth"; import { longLivedTokenUsable } from "./token-verify"; - export interface SubscriptionAccount { name: string; label?: string; @@ -86,7 +84,7 @@ function isTransientRefreshError(error: unknown): boolean { const INVALID_GRANT_COOLDOWN_MS = 10 * 60 * 1000; function invalidGrantPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "ai", "invalid-grant.json"); + return genesisToolsDir("ai", "invalid-grant.json"); } function readInvalidGrants(): Record { @@ -286,7 +284,7 @@ export function pruneJournal(path: string): void { } function journalPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "ai", "token-journal.jsonl"); + return genesisToolsDir("ai", "token-journal.jsonl"); } /** diff --git a/src/utils/claude/worker-paths.ts b/src/utils/claude/worker-paths.ts index 1b75e09cb2..18a2d50171 100644 --- a/src/utils/claude/worker-paths.ts +++ b/src/utils/claude/worker-paths.ts @@ -1,7 +1,5 @@ -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; - +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** Where `tools claude worker` keeps `.meta.json` and `.turn.jsonl`. */ export function workersDir(): string { - return join(env.tools.getHome(), ".genesis-tools", "claude", "workers"); + return genesisToolsDir("claude", "workers"); } diff --git a/src/utils/cli/bun-link.ts b/src/utils/cli/bun-link.ts index 853282c07d..debe503c9f 100644 --- a/src/utils/cli/bun-link.ts +++ b/src/utils/cli/bun-link.ts @@ -1,8 +1,8 @@ import { existsSync, linkSync, mkdirSync, statSync, unlinkSync } from "node:fs"; import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** * Resolve a bun executable named after the tool it will run, so the process * shows up in Activity Monitor / ps / pgrep as "gt-" instead of "bun". @@ -21,7 +21,7 @@ export function namedBunExecPath(name: string, bunPath: string = process.execPat } const linkName = `gt-${name.replace(/[^\w.-]+/g, "-")}`; - const binDir = join(env.tools.getHome(), ".genesis-tools", "bin"); + const binDir = genesisToolsDir("bin"); const linkPath = join(binDir, linkName); try { diff --git a/src/utils/cli/worktree-install.ts b/src/utils/cli/worktree-install.ts new file mode 100644 index 0000000000..b991f57ba4 --- /dev/null +++ b/src/utils/cli/worktree-install.ts @@ -0,0 +1,27 @@ +import { statSync } from "node:fs"; +import { join } from "node:path"; + +/** Linked checkouts do not auto-install; unreadable markers fail closed unless explicitly opted in. */ +export function skipWorktreeInstall({ + workspaceRoot, + autoInstall, + stat = statSync, +}: { + workspaceRoot: string; + autoInstall?: string; + stat?: (path: string) => { isFile(): boolean }; +}): boolean { + if (autoInstall === "1") { + return false; + } + + try { + return stat(join(workspaceRoot, ".git")).isFile(); + } catch (error) { + if (error instanceof Error && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR")) { + return false; + } + + throw error; + } +} diff --git a/src/utils/codex/worker-paths.ts b/src/utils/codex/worker-paths.ts index 0cef32cada..55b119631c 100644 --- a/src/utils/codex/worker-paths.ts +++ b/src/utils/codex/worker-paths.ts @@ -1,9 +1,8 @@ import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { safeNamedPath } from "@genesiscz/utils/worker/safe-path"; - export function codexRoot(): string { - return join(env.tools.getHome(), ".genesis-tools", "codex"); + return genesisToolsDir("codex"); } export function sessionsDir(): string { diff --git a/src/utils/debugging-master/llm-log.ts b/src/utils/debugging-master/llm-log.ts index 4d23094e37..2dd4589de4 100644 --- a/src/utils/debugging-master/llm-log.ts +++ b/src/utils/debugging-master/llm-log.ts @@ -14,6 +14,18 @@ * dbg.session('my-feature'); * dbg.info('request received', { url: req.url }); * dbg.dump('response', data); + * dbg.timerStart('db-query'); + * // ... do work ... + * dbg.timerEnd('db-query'); + * dbg.snapshot('state', { user, cart, flags }); + * + * Every entry captures the full call stack by default (callers up the chain). + * To opt out per-call: dbg.info('msg', undefined, { stack: false }); + * To opt out globally (one-time): dbg.configure({ captureStackByDefault: false }); + * + * Every method accepts a final `opts` argument with `{ h?, stack? }`: + * - `h` — hypothesis tag to filter by in the dashboard + * - `stack` — false to skip stack capture for this call; string to override */ // ─── Config ────────────────────────────────────────────────────────────────── diff --git a/src/utils/docker/qdrant.ts b/src/utils/docker/qdrant.ts index faaa156d6a..bb943a5d34 100644 --- a/src/utils/docker/qdrant.ts +++ b/src/utils/docker/qdrant.ts @@ -1,6 +1,6 @@ import { mkdirSync } from "node:fs"; -import { join } from "node:path"; import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { DockerContainer, type DockerProgressCallback } from "./docker-container"; const QDRANT_IMAGE = "qdrant/qdrant:v1.17.0"; @@ -21,7 +21,7 @@ function parsePort(envValue: string | undefined, defaultPort: number): number { return parsed; } -const QDRANT_STORAGE = join(env.tools.getHome(), ".genesis-tools", "indexes", "qdrant"); +const QDRANT_STORAGE = genesisToolsDir("indexes", "qdrant"); let instance: DockerContainer | null = null; diff --git a/src/utils/grok/worker-paths.ts b/src/utils/grok/worker-paths.ts index bca8f1305b..8836b6bc39 100644 --- a/src/utils/grok/worker-paths.ts +++ b/src/utils/grok/worker-paths.ts @@ -1,9 +1,8 @@ import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { safeNamedPath } from "@genesiscz/utils/worker/safe-path"; - export function grokRoot(): string { - return join(env.tools.getHome(), ".genesis-tools", "grok"); + return genesisToolsDir("grok"); } export function sessionsDir(): string { diff --git a/src/utils/macos/genesis-app.ts b/src/utils/macos/genesis-app.ts index 56dffb7f23..5688935a89 100644 --- a/src/utils/macos/genesis-app.ts +++ b/src/utils/macos/genesis-app.ts @@ -4,6 +4,7 @@ import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; import { inspectPidFile } from "@genesiscz/utils/process/pidfile"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; /** * GenesisTools.app is a tiny signed launcher (src/macos/GenesisTools) that becomes the macOS TCC * "responsible process" for everything `tools` runs. Calendar, Reminders, Contacts, Full Disk @@ -22,7 +23,7 @@ export const GENESIS_MARKDOWN_BUNDLE_ID = "dev.foltyn.genesis.markdown"; /** Build manifest and prompt stamps; the bundle itself lives in ~/Applications. */ export function genesisAppDir(): string { - return join(env.tools.getHome(), ".genesis-tools", "app"); + return genesisToolsDir("app"); } export function genesisAppBundlePath(): string { diff --git a/src/utils/obsidian/config.ts b/src/utils/obsidian/config.ts index 7ab5763bee..8fbbde8e55 100644 --- a/src/utils/obsidian/config.ts +++ b/src/utils/obsidian/config.ts @@ -1,11 +1,9 @@ import { existsSync, readFileSync, renameSync } from "node:fs"; -import { join } from "node:path"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { resolveActiveVault } from "./discovery"; - export interface ObsidianConfig { vaultRoot: string | null; } @@ -18,12 +16,12 @@ export interface ObsidianConfig { * vault kept their paths somewhere else entirely. One file, one key per consumer. */ export function pluginConfigPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "plugins", "config.json"); + return genesisToolsDir("plugins", "config.json"); } /** Where the vault root lived before the move. Migrated once, on first read. */ export function obsidianConfigPath(): string { - return join(env.tools.getHome(), ".genesis-tools", "obsidian", "config.json"); + return genesisToolsDir("obsidian", "config.json"); } type PluginConfig = Record; diff --git a/src/utils/package.json b/src/utils/package.json index 4f0ab0f797..2b08cc4235 100644 --- a/src/utils/package.json +++ b/src/utils/package.json @@ -466,6 +466,11 @@ "types": "./storage/atomic-file.ts", "default": "./storage/atomic-file.ts" }, + "./storage/root": { + "bun": "./storage/root.ts", + "types": "./storage/root.ts", + "default": "./storage/root.ts" + }, "./string": { "bun": "./string.ts", "types": "./string.ts", diff --git a/src/utils/profile/profile.ts b/src/utils/profile/profile.ts index 816ecdc84f..7fdf7f8ced 100644 --- a/src/utils/profile/profile.ts +++ b/src/utils/profile/profile.ts @@ -18,13 +18,13 @@ // p.summary(); import { appendFileSync, mkdirSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { dirname } from "node:path"; import { formatLocalDate } from "@genesiscz/utils/date"; import { env } from "@genesiscz/utils/env"; import { getProfilingConfig, type ProfilingDetail } from "@genesiscz/utils/GenesisTools"; import { assertTestSafePath } from "@genesiscz/utils/storage/real-home-guard"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { currentTraceId } from "@genesiscz/utils/trace"; - export interface ProfilerGate { on: boolean; scopes: string[] | null; @@ -200,7 +200,7 @@ export const FLUSH_INTERVAL_MS = 250; function defaultLogPath(): string { const day = formatLocalDate(new Date()); - return join(env.tools.getHome(), ".genesis-tools", "logs", `${day}-profiling.log`); + return genesisToolsDir("logs", `${day}-profiling.log`); } /** Set once a file write fails, so a broken sink cannot keep throwing. */ diff --git a/src/utils/services/ensure.ts b/src/utils/services/ensure.ts index d7392ba435..396c3f4cd5 100644 --- a/src/utils/services/ensure.ts +++ b/src/utils/services/ensure.ts @@ -1,11 +1,9 @@ import { connect } from "node:net"; -import { join } from "node:path"; import { commandWords, spawnToolDetached } from "@genesiscz/utils/cli"; import { formatLocalDate } from "@genesiscz/utils/date"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { registryEntryForPort } from "@genesiscz/utils/ui/dashboards"; - export interface EnsureTarget { name: string; launch: string | null; @@ -100,7 +98,7 @@ export async function ensureRegisteredPort(port: number): Promise sleep: (ms) => Bun.sleep(ms), now: () => Date.now(), // A started tool writes the shared day log; the detached process has no terminal of its own. - logPath: () => join(env.tools.getHome(), ".genesis-tools", "logs", `${formatLocalDate(new Date())}.log`), + logPath: () => genesisToolsDir("logs", `${formatLocalDate(new Date())}.log`), }); } diff --git a/src/utils/storage/root.ts b/src/utils/storage/root.ts index 2e7265b996..5643a6dcb6 100644 --- a/src/utils/storage/root.ts +++ b/src/utils/storage/root.ts @@ -11,6 +11,16 @@ export function genesisToolsDir(...segments: string[]): string { return join(env.tools.getHome(), ".genesis-tools", ...segments); } +/** + * `/.genesis-tools/` under an explicit home: a sandbox, a fixture, a benchmark arm. Code + * that means the user's own tree calls `genesisToolsDir` instead, so the GENESIS_TOOLS_HOME override and + * the test sandbox always apply. Every `.genesis-tools` path is built by this file (lint rule + * `no-hand-joined-genesis-tools`). + */ +export function genesisToolsDirIn(home: string, ...segments: string[]): string { + return join(home, ".genesis-tools", ...segments); +} + /** `~/.genesis-tools//`: the directory `new Storage(tool).getBaseDir()` returns. */ export function toolDataDir(tool: string, ...segments: string[]): string { return genesisToolsDir(tool, ...segments); diff --git a/src/utils/ts/sourcemaps/build.ts b/src/utils/ts/sourcemaps/build.ts index 672cf7819e..1d1b687f86 100644 --- a/src/utils/ts/sourcemaps/build.ts +++ b/src/utils/ts/sourcemaps/build.ts @@ -1,10 +1,15 @@ import { createHash } from "node:crypto"; import { createWriteStream } from "node:fs"; -import { cp, lstat, mkdir, readFile, rename, rm, stat, symlink } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { cp, lstat, mkdir, readFile, rename, rm, stat, symlink, utimes } from "node:fs/promises"; +import { dirname, join, sep } from "node:path"; +import { canonicalDir } from "@genesiscz/utils/fs/canonical"; +import { measureTree } from "@genesiscz/utils/fs/disk-usage"; +import { parseWorktreeListZ, WORKTREE_LIST_ARGS, type WorktreeEntry } from "@genesiscz/utils/git/porcelain"; import { logger } from "@genesiscz/utils/logger"; +import { batchCwd, listPsRows } from "@genesiscz/utils/process/ps"; import { profiler } from "@genesiscz/utils/profile"; -import { withFileLock } from "@genesiscz/utils/storage"; +import { LockTimeoutError, withFileLock } from "@genesiscz/utils/storage"; +import type { Subprocess } from "bun"; import type { CacheEntry, SourcemapCache } from "./cache.ts"; const prof = profiler.scope("sourcemaps"); @@ -13,6 +18,34 @@ const BUILD_LOCK_TIMEOUT_MS = 30 * 60_000; const { log } = logger.scoped("ts:sourcemaps:build"); +/** + * What this process's builds hold: the managed worktrees they ran in, the build commands still running, and the + * builds that hold a worktree lock. A command calls `releaseBuildWorktrees` when it ends, so a worktree with its + * installed dependencies (gigabytes for a large app) does not outlive the command. + */ +const held = { + worktrees: new Map(), + processes: new Set(), + cancelled: new WeakSet(), + /** Each settles once its build has ended and released the worktree lock. */ + builds: new Set>(), + stopping: false, +}; + +/** Why `removeBuildWorktree` removed a worktree or left it in place. */ +export type WorktreeOutcome = "removed" | "absent" | "recent" | "busy" | "locked" | "in-use" | "changed" | "failed"; + +export interface WorktreeRemoval { + worktree: string; + outcome: WorktreeOutcome; + /** Bytes freed: on APFS the bytes no clone shares (what the disk gets back), elsewhere the allocated size. */ + bytes: number; + /** Removed only: its size on disk as `du` counts it, blocks shared with clones (a package manager's cache) included. */ + diskBytes?: number; + /** Why it was kept: the lock reason, the processes inside it, its tracked changes, the git error. */ + detail?: string; +} + export interface BuildAtCommitOptions { cache: SourcemapCache; /** The caller's checkout; the build runs in a separate worktree of it, never in the checkout itself. */ @@ -101,6 +134,16 @@ async function exists(path: string): Promise { ); } +/** The worktree a project's builds share when the caller names none. */ +export function managedWorktree(cache: SourcemapCache): string { + return join(cache.dir, "worktree"); +} + +/** One build at a time holds this lock on `worktree`, across processes; removing the worktree takes it too. */ +function worktreeLockPath(cache: SourcemapCache, worktree: string): string { + return join(cache.dir, `.worktree-${createHash("sha1").update(worktree).digest("hex").slice(0, 8)}.lock`); +} + async function prepareWorktree(options: { repo: string; worktree: string; commit: string }): Promise { if (!(await exists(options.worktree))) { await mkdir(dirname(options.worktree), { recursive: true }); @@ -119,6 +162,25 @@ async function prepareWorktree(options: { repo: string; worktree: string; commit await git(options.worktree, ["checkout", "--detach", options.commit]); } +async function stopBuild(proc: Subprocess): Promise { + held.cancelled.add(proc); + const signalGroup = (signal: NodeJS.Signals): void => { + try { + // pid-verified: proc is this process's own Bun.spawn child, started detached, so -pid is the group it leads + process.kill(-proc.pid, signal); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ESRCH")) { + log.warn({ error, pid: proc.pid, signal }, "could not signal build process group"); + proc.kill(signal); + } + } + }; + signalGroup("SIGTERM"); + // The shell may exit first while a child still holds the output pipes open. + await Bun.sleep(250); + signalGroup("SIGKILL"); +} + /** The build itself; the caller holds the worktree lock. */ async function buildLocked(input: { options: BuildAtCommitOptions; @@ -129,6 +191,9 @@ async function buildLocked(input: { }): Promise { const { options, worktree, commit, id, dir } = input; await prepareWorktree({ repo: options.repo, worktree, commit }); + // `cache prune` removes a worktree idle for an hour; reusing one changes no folder entry, so mark it used. + const now = new Date(); + await utimes(worktree, now, now); for (const link of options.links ?? []) { const target = join(worktree, link); if (!(await exists(target))) { @@ -139,17 +204,28 @@ async function buildLocked(input: { await mkdir(dir, { recursive: true }); const logPath = join(dir, ".build.log"); - const logFile = createWriteStream(logPath); const started = performance.now(); + if (held.stopping || options.signal?.aborted) { + throw new Error(`build of ${commit.slice(0, 12)} not started: the command is stopping its builds`); + } + + const logFile = createWriteStream(logPath); log.info({ commit, worktree, buildCmd: options.buildCmd }, "building"); const proc = Bun.spawn(["sh", "-c", options.buildCmd], { cwd: worktree, + detached: true, env: { ...process.env, ...options.env }, stdout: "pipe", stderr: "pipe", }); - const abort = (): void => proc.kill(); + held.processes.add(proc); + const abort = (): void => { + void stopBuild(proc); + }; options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) { + abort(); + } const pump = async (stream: ReadableStream): Promise => { const decoder = new TextDecoder(); let pending = ""; @@ -166,12 +242,20 @@ async function buildLocked(input: { }; await Promise.all([pump(proc.stdout), pump(proc.stderr)]); const code = await proc.exited; + held.processes.delete(proc); options.signal?.removeEventListener("abort", abort); logFile.end(); if (code !== 0) { throw new Error(`build command exited ${code} at ${commit.slice(0, 12)}; log: ${logPath}`); } + const assertNotCancelled = (): void => { + if (held.cancelled.has(proc) || options.signal?.aborted || held.stopping) { + throw new Error(`build cancelled at ${commit.slice(0, 12)}; log: ${logPath}`); + } + }; + assertNotCancelled(); + const output = join(worktree, options.outputDir); if (!(await stat(output).catch(() => null))?.isDirectory()) { throw new Error(`build finished but ${output} is not a folder; check --output`); @@ -183,15 +267,29 @@ async function buildLocked(input: { const staging = `${files}.staging`; await rm(staging, { recursive: true, force: true }); await cp(output, staging, { recursive: true }); + assertNotCancelled(); + // A refresh must not leave old metadata pointing at newly published but cancelled output. + const previous = await options.cache.invalidate(id); + assertNotCancelled(); await rm(files, { recursive: true, force: true }); await rename(staging, files); - const entry = await options.cache.commit({ - id, - kind: "build", - label: `${commit.slice(0, 12)} ${options.outputDir}`, - origin: { repo: options.repo, commit, buildCmd: options.buildCmd, outputDir: options.outputDir }, - }); - return { entry, dir, commit, reused: false, durationMs: Math.round(performance.now() - started) }; + assertNotCancelled(); + try { + const entry = await options.cache.commit({ + id, + kind: "build", + label: `${commit.slice(0, 12)} ${options.outputDir}`, + origin: { repo: options.repo, commit, buildCmd: options.buildCmd, outputDir: options.outputDir }, + assertValid: assertNotCancelled, + previous, + }); + assertNotCancelled(); + return { entry, dir, commit, reused: false, durationMs: Math.round(performance.now() - started) }; + } catch (error) { + // commit also prunes asynchronously; cancellation there must retract an entry already written. + await options.cache.invalidate(id); + throw error; + } } /** @@ -211,30 +309,188 @@ export async function buildAtCommit(options: BuildAtCommitOptions): Promise void) | undefined; + try { + // One build at a time per worktree, across processes: a checkout in the middle of another build would + // cache one commit's output under the other commit's id. + return await withFileLock( + worktreeLockPath(options.cache, worktree), + async () => { + // Whoever held the lock may have just built this very recipe. + const built = options.refresh ? null : await options.cache.get(id); + if (built) { + await options.cache.touch(id); + return { entry: built, dir, commit, reused: true, durationMs: 0 }; + } + + // A worktree the caller named is theirs; only the managed one is removed when the command ends. + if (!options.worktree) { + held.worktrees.set(worktree, options.cache); + } + + const { promise, resolve } = Promise.withResolvers(); + held.builds.add(promise); + ended = () => { + held.builds.delete(promise); + resolve(); + }; + return buildLocked({ options, worktree, commit, id, dir }); + }, + BUILD_LOCK_TIMEOUT_MS + ); + } finally { + // Only now is the lock released, so a release that waits for this build can take it. + ended?.(); + } + } finally { + stopProfile(); + } +} + +/** + * Removes the managed worktrees this process built in, each through `removeBuildWorktree`, so a command leaves no + * installed dependencies behind. It waits until every build has released its worktree. With `stop` (the command was + * interrupted) it first ends the running build commands and refuses to start new ones until it returns. + */ +export async function releaseBuildWorktrees(options: { stop?: boolean } = {}): Promise { + held.stopping = options.stop === true; + try { + if (options.stop) { + await Promise.all([...held.processes].map(stopBuild)); + } + + await Promise.all(held.builds); + const removals: WorktreeRemoval[] = []; + for (const [worktree, cache] of held.worktrees) { + removals.push(await removeBuildWorktree({ cache, worktree })); + } + + held.worktrees.clear(); + return removals; + } finally { + held.stopping = false; + } +} + +/** + * Removes a build worktree with `git worktree remove`, never by deleting files itself. It is kept while it may still + * be wanted: a build runs in it (the build lock is held), it is locked with `git worktree lock`, a process has its + * working directory inside it, or it has tracked changes. With `idleMs` it is also kept when used more recently. + */ +export async function removeBuildWorktree(options: { + cache: SourcemapCache; + worktree: string; + idleMs?: number; +}): Promise { + const { worktree } = options; + const folder = await stat(worktree).catch(() => null); + if (!folder?.isDirectory()) { + return { worktree, outcome: "absent", bytes: 0 }; + } + + if (options.idleMs !== undefined && Date.now() - folder.mtimeMs < options.idleMs) { + return { worktree, outcome: "recent", bytes: 0 }; + } + + try { + // No waiting: a held lock means a build is running in it right now. return await withFileLock( - join(options.cache.dir, lockName), + worktreeLockPath(options.cache, worktree), async () => { - // Whoever held the lock may have just built this very recipe. - const built = options.refresh ? null : await options.cache.get(id); - if (built) { - await options.cache.touch(id); - return { entry: built, dir, commit, reused: true, durationMs: 0 }; + if (options.idleMs !== undefined) { + const current = await stat(worktree); + if (Date.now() - current.mtimeMs < options.idleMs) { + return { worktree, outcome: "recent", bytes: 0 }; + } } - return buildLocked({ options, worktree, commit, id, dir }); + return removeUnusedWorktree(worktree); }, - BUILD_LOCK_TIMEOUT_MS + 0 ); - } finally { - stopProfile(); + } catch (error) { + if (error instanceof LockTimeoutError) { + return { worktree, outcome: "busy", bytes: 0, detail: "a build is running in it" }; + } + + // A command's own error must not be replaced by its cleanup's: report the worktree as kept instead. + log.warn({ err: error, worktree }, "could not check the build worktree; left in place"); + return { + worktree, + outcome: "failed", + bytes: 0, + detail: error instanceof Error ? error.message : String(error), + }; } } +/** The checks and the removal that need the build lock; see `removeBuildWorktree`. */ +async function removeUnusedWorktree(worktree: string): Promise { + const kept = (outcome: WorktreeOutcome, detail: string): WorktreeRemoval => { + log.info({ worktree, outcome, detail }, "kept build worktree"); + return { worktree, outcome, bytes: 0, detail }; + }; + const real = canonicalDir(worktree); + let commonDir: string; + let listed: WorktreeEntry | undefined; + try { + commonDir = await git(worktree, ["rev-parse", "--path-format=absolute", "--git-common-dir"]); + listed = parseWorktreeListZ(await git(commonDir, [...WORKTREE_LIST_ARGS])).find( + (entry) => canonicalDir(entry.path) === real + ); + } catch (error) { + return kept("failed", `git cannot read it: ${error instanceof Error ? error.message : String(error)}`); + } + + if (!listed) { + return kept("failed", "git does not list it as a worktree"); + } + + if (listed.locked !== null) { + return kept("locked", `locked with git worktree lock: ${listed.locked}`); + } + + const rows = await listPsRows(); + if (rows.length === 0) { + return kept("in-use", "the process list is unreadable, so a process may work in it"); + } + + const cwds = batchCwd(rows.map((row) => row.pid)); + // The kernel reports a working directory already resolved; canonicalizing 1,300 of them cost 0.6 to 5 s here. + const inside = rows.filter((row) => { + const cwd = cwds.get(row.pid); + return cwd !== undefined && (cwd === real || cwd.startsWith(real + sep)); + }); + if (inside.length > 0) { + return kept( + "in-use", + `a process works in it: ${inside + .slice(0, 3) + .map((row) => `${row.pid} ${row.command.slice(0, 60)}`) + .join(", ")}` + ); + } + + const changes = await git(worktree, ["status", "--porcelain", "--untracked-files=no"]); + if (changes) { + return kept("changed", `it has tracked changes:\n${changes}`); + } + + const usage = measureTree(worktree, { exact: false }); + try { + // --force lets untracked build output and --link symlinks go with it; one --force still refuses a locked tree. + await git(commonDir, ["worktree", "remove", "--force", worktree]); + } catch (error) { + return kept("failed", error instanceof Error ? error.message : String(error)); + } + + const bytes = usage.private ?? usage.allocated; + log.info({ worktree, bytes, diskBytes: usage.allocated, files: usage.fileCount }, "removed build worktree"); + return { worktree, outcome: "removed", bytes, diskBytes: usage.allocated }; +} + /** The folder of a build entry that holds the build output. */ export function buildFilesDir(cache: SourcemapCache, id: string): string { return join(cache.entryDir(id), "files"); diff --git a/src/utils/ts/sourcemaps/cache.ts b/src/utils/ts/sourcemaps/cache.ts index a13e614e1d..9b55ddbac8 100644 --- a/src/utils/ts/sourcemaps/cache.ts +++ b/src/utils/ts/sourcemaps/cache.ts @@ -6,6 +6,7 @@ import { profiler } from "@genesiscz/utils/profile"; import { withFileLock } from "@genesiscz/utils/storage"; import { atomicWriteFileSync } from "@genesiscz/utils/storage/atomic-file"; import { toolDataDir } from "@genesiscz/utils/storage/root"; +import { managedWorktree, removeBuildWorktree, type WorktreeRemoval } from "./build.ts"; const { log } = logger.scoped("ts:sourcemaps:cache"); @@ -15,6 +16,8 @@ export const DEFAULT_MAX_AGE_DAYS = 30; const UNFINISHED_GRACE_MS = 60 * 60 * 1000; /** How long a metadata change waits for another process's change (a prune that removes large folders). */ const METADATA_LOCK_TIMEOUT_MS = 60_000; +/** Prune removes the managed build worktree once it has not been used this long (a command removes its own at its end). */ +export const WORKTREE_IDLE_MS = 60 * 60 * 1000; export type CacheEntryKind = "deployed" | "build"; @@ -47,11 +50,43 @@ export interface CacheMetadata { entries: Record; } +export interface PrunedItem { + /** The entry id, or `worktree` for the build worktree: a folder name in the project's cache. */ + name: string; + reason: "unused" | "unfinished" | "idle worktree"; + /** Bytes freed; for the worktree, without the blocks it shared with clones (see `WorktreeRemoval`). */ + bytes: number; + /** Size on disk as `du` counts it. */ + diskBytes: number; +} + +export interface PruneResult { + removed: PrunedItem[]; + /** The build worktree when it was idle past the hour and prune still left it in place, with the reason. */ + kept: WorktreeRemoval[]; +} + /** `~/.genesis-tools/ts/sourcemaps`. */ export function sourcemapsRoot(): string { return toolDataDir("ts", "sourcemaps"); } +/** The projects that have a cache folder under `root`, sorted. */ +export async function cacheProjects(root: string = sourcemapsRoot()): Promise { + const entries = await readdir(root, { withFileTypes: true }).catch((error: unknown) => { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + log.debug({ err: error, root }, "no sourcemaps cache yet"); + return []; + } + + throw error; + }); + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort(); +} + /** A folder-safe project name: lower-case letters, digits, dot, dash and underscore. */ export function projectSlug(name: string): string { const slug = name @@ -129,6 +164,17 @@ export class SourcemapCache { return (await this.read()).entries[id] ?? null; } + /** Stops readers reusing an entry while its files are replaced; unfinished files remain prunable. */ + async invalidate(id: string): Promise { + return this.transaction(async () => { + const metadata = await this.read(); + const previous = metadata.entries[id]; + delete metadata.entries[id]; + await this.write(metadata); + return previous; + }); + } + async list(): Promise { return Object.values((await this.read()).entries).sort((a, b) => b.lastUsedAt.localeCompare(a.lastUsedAt)); } @@ -143,12 +189,15 @@ export class SourcemapCache { label: string; origin: CacheEntryOrigin; maxAgeDays?: number; + assertValid?: () => void; + /** The same recipe's metadata invalidated before replacing its files. */ + previous?: CacheEntry; }): Promise { const size = await folderSize(this.entryDir(options.id)); const entry = await this.transaction(async () => { const metadata = await this.read(); const now = new Date().toISOString(); - const previous = metadata.entries[options.id]; + const previous = metadata.entries[options.id] ?? options.previous; const recorded: CacheEntry = { id: options.id, kind: options.kind, @@ -161,6 +210,8 @@ export class SourcemapCache { links: previous?.links ?? [], }; metadata.entries[options.id] = recorded; + // No await separates this guard from the synchronous atomic metadata write. + options.assertValid?.(); await this.write(metadata); return recorded; }); @@ -233,23 +284,44 @@ export class SourcemapCache { return resolve(dirname(link), await readlink(link)); } - /** Removes entries (folder, metadata row, `--out` symlinks) unused for `maxAgeDays`. Returns their ids. */ - prune(options: { maxAgeDays?: number; keep?: string[] } = {}): Promise { + /** + * Removes entries (folder, metadata row, `--out` symlinks) unused for `maxAgeDays`, folders that failed builds + * left, and the managed build worktree once it has been idle for `WORKTREE_IDLE_MS`. + */ + async prune(options: { maxAgeDays?: number; keep?: string[] } = {}): Promise { const maxAgeDays = options.maxAgeDays ?? DEFAULT_MAX_AGE_DAYS; // NaN compares false against every age, so an unparsed value would remove every entry. if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0) { - return Promise.reject(new RangeError(`maxAgeDays must be a number of days, 0 or more; got ${maxAgeDays}`)); + throw new RangeError(`maxAgeDays must be a number of days, 0 or more; got ${maxAgeDays}`); } - return profiler - .scope("sourcemaps") - .measureAsync("cache.prune", () => this.transaction(() => this.pruneNow(options))); + return profiler.scope("sourcemaps").measureAsync("cache.prune", async () => { + const removed = await this.transaction(() => this.pruneNow(options)); + // Outside the metadata lock: removing a worktree with its dependencies takes seconds. + const worktree = await removeBuildWorktree({ + cache: this, + worktree: managedWorktree(this), + idleMs: WORKTREE_IDLE_MS, + }); + if (worktree.outcome === "removed") { + removed.push({ + name: relative(this.dir, worktree.worktree), + reason: "idle worktree", + bytes: worktree.bytes, + diskBytes: worktree.diskBytes ?? worktree.bytes, + }); + } + + const quiet = + worktree.outcome === "removed" || worktree.outcome === "absent" || worktree.outcome === "recent"; + return { removed, kept: quiet ? [] : [worktree] }; + }); } - private async pruneNow(options: { maxAgeDays?: number; keep?: string[] }): Promise { + private async pruneNow(options: { maxAgeDays?: number; keep?: string[] }): Promise { const maxAgeMs = (options.maxAgeDays ?? DEFAULT_MAX_AGE_DAYS) * 86_400_000; const metadata = await this.read(); - const removed: string[] = []; + const removed: PrunedItem[] = []; for (const entry of Object.values(metadata.entries)) { if (options.keep?.includes(entry.id) || Date.now() - Date.parse(entry.lastUsedAt) < maxAgeMs) { continue; @@ -264,13 +336,13 @@ export class SourcemapCache { await rm(this.entryDir(entry.id), { recursive: true, force: true }); delete metadata.entries[entry.id]; - removed.push(entry.id); + removed.push({ name: entry.id, reason: "unused", bytes: entry.bytes, diskBytes: entry.bytes }); } // A build or fetch that failed or was interrupted leaves its folder without a metadata entry. Such a // folder is never reused (the next run starts it over), so it only holds disk. The grace period keeps // a build that is running right now. - const unfinished: string[] = []; + const unfinished: PrunedItem[] = []; for (const name of await readdir(this.dir).catch(() => [] as string[])) { if (!/^(build|deployed)-/.test(name) || metadata.entries[name] || options.keep?.includes(name)) { continue; @@ -278,8 +350,9 @@ export class SourcemapCache { const folder = await stat(this.entryDir(name)).catch(() => null); if (folder?.isDirectory() && Date.now() - folder.mtimeMs > UNFINISHED_GRACE_MS) { + const size = await folderSize(this.entryDir(name)); await rm(this.entryDir(name), { recursive: true, force: true }); - unfinished.push(name); + unfinished.push({ name, reason: "unfinished", bytes: size.bytes, diskBytes: size.bytes }); } } @@ -288,6 +361,6 @@ export class SourcemapCache { await this.write(metadata); } - return [...removed, ...unfinished.map((name) => `${name} (unfinished)`)]; + return [...removed, ...unfinished]; } } diff --git a/src/utils/ts/sourcemaps/sourcemaps.test.ts b/src/utils/ts/sourcemaps/sourcemaps.test.ts index 57275e6043..0c38a19977 100644 --- a/src/utils/ts/sourcemaps/sourcemaps.test.ts +++ b/src/utils/ts/sourcemaps/sourcemaps.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { existsSync, lstatSync, @@ -11,15 +11,17 @@ import { utimesSync, writeFileSync, } from "node:fs"; +import * as fsPromises from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { deflateRawSync } from "node:zlib"; import { env } from "@genesiscz/utils/env"; import { unzip } from "@genesiscz/utils/fs/unzip"; import { SafeJSON } from "@genesiscz/utils/json"; +import { isProcessAlive } from "@genesiscz/utils/process-alive"; import { alignOffset, normalizeIdentifiers } from "./align.ts"; -import { buildAtCommit } from "./build.ts"; -import { SourcemapCache } from "./cache.ts"; +import { buildAtCommit, managedWorktree, releaseBuildWorktrees, removeBuildWorktree } from "./build.ts"; +import { cacheProjects, type PruneResult, SourcemapCache } from "./cache.ts"; import { calibrate, sampleLiterals } from "./calibrate.ts"; import { carryMap, type EqualRun, encodeMappings, equalRuns } from "./carry.ts"; import { DirectoryMapSource, debugIdOf, mapsInArtifactBundle, SentryArtifactSource } from "./debug-id.ts"; @@ -220,7 +222,54 @@ describe("CLI arguments", () => { }); }); +/** A git repo with one commit of `a.txt`; returns a runner for git in it that throws on failure. */ +function fixtureRepo(dir: string): (...args: string[]) => string { + mkdirSync(dir, { recursive: true }); + const git = (...args: string[]): string => { + const result = Bun.spawnSync(["git", "-C", dir, ...args], { env: process.env }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString()}`); + } + + return result.stdout.toString().trim(); + }; + git("init", "-q"); + writeFileSync(join(dir, "a.txt"), "a"); + git("add", "a.txt"); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "-qm", "one"); + return git; +} + +const prunedNames = (result: PruneResult): string[] => result.removed.map((item) => item.name); + describe("SourcemapCache", () => { + test("cache discovery treats only a missing root as empty", async () => { + const root = mkdtempSync(join(tmpdir(), "gt-cache-discovery-")); + try { + expect(await cacheProjects(join(root, "missing"))).toEqual([]); + const file = join(root, "file"); + writeFileSync(file, "not a directory"); + await expect(cacheProjects(file)).rejects.toMatchObject({ code: "ENOTDIR" }); + mkdirSync(join(root, "zebra")); + mkdirSync(join(root, "alpha")); + expect(await cacheProjects(root)).toEqual(["alpha", "zebra"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("cache discovery propagates permission and IO failures unchanged", async () => { + const readDirectory = spyOn(fsPromises, "readdir"); + try { + for (const code of ["EACCES", "EIO"]) { + const error = Object.assign(new Error(`cannot inspect cache: ${code}`), { code }); + readDirectory.mockRejectedValueOnce(error); + await expect(cacheProjects("unreadable-cache-fixture")).rejects.toBe(error); + } + } finally { + readDirectory.mockRestore(); + } + }); const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) { @@ -270,7 +319,7 @@ describe("SourcemapCache", () => { const metadata = await cache.read(); metadata.entries.old.lastUsedAt = new Date(Date.now() - 31 * 86_400_000).toISOString(); writeFileSync(cache.metadataPath, SafeJSON.stringify(metadata)); - expect(await cache.prune()).toEqual(["old"]); + expect(prunedNames(await cache.prune())).toEqual(["old"]); expect(existsSync(link)).toBe(false); }); @@ -289,7 +338,7 @@ describe("SourcemapCache", () => { const metadata = await cache.read(); metadata.entries["build-1"].lastUsedAt = new Date(Date.now() - 31 * 86_400_000).toISOString(); writeFileSync(cache.metadataPath, SafeJSON.stringify(metadata)); - expect(await cache.prune()).toEqual(["build-1"]); + expect(prunedNames(await cache.prune())).toEqual(["build-1"]); expect(lstatSync(link).isSymbolicLink()).toBe(true); }); @@ -305,7 +354,7 @@ describe("SourcemapCache", () => { expect(Object.keys((await cache.read()).entries)).toEqual(["kept"]); expect(existsSync(cache.entryDir("kept"))).toBe(true); - expect(await cache.prune({ maxAgeDays: 0 })).toEqual(["kept"]); + expect(prunedNames(await cache.prune({ maxAgeDays: 0 }))).toEqual(["kept"]); }); test("concurrent commits never drop each other's entry", async () => { @@ -340,37 +389,342 @@ describe("SourcemapCache", () => { const metadata = await cache.read(); metadata.entries.old.lastUsedAt = new Date(Date.now() - 31 * 86_400_000).toISOString(); writeFileSync(cache.metadataPath, SafeJSON.stringify(metadata)); - expect(await cache.prune()).toEqual(["old"]); + expect(prunedNames(await cache.prune())).toEqual(["old"]); expect(lstatSync(viaCache).isSymbolicLink()).toBe(true); expect(lstatSync(byHand).isSymbolicLink()).toBe(true); }); - test("prune removes a build folder a failed run left behind, but not one that may still be building", async () => { + test("prune removes a failed build's folder and the build worktree once each is idle, not a build running now", async () => { const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-cache-")); roots.push(root); - const cache = new SourcemapCache("app", root); - for (const name of ["build-failed-1", "build-running-2", "worktree"]) { + const git = fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + for (const name of ["build-failed-1", "build-running-2"]) { mkdirSync(cache.entryDir(name), { recursive: true }); } + writeFileSync(join(cache.entryDir("build-failed-1"), ".build.log"), "12345"); + const worktree = managedWorktree(cache); + git("worktree", "add", "-q", "--detach", worktree, "HEAD"); const twoHoursAgo = new Date(Date.now() - 2 * 3_600_000); utimesSync(cache.entryDir("build-failed-1"), twoHoursAgo, twoHoursAgo); - utimesSync(cache.entryDir("worktree"), twoHoursAgo, twoHoursAgo); - expect(await cache.prune()).toEqual(["build-failed-1 (unfinished)"]); - expect(existsSync(cache.entryDir("build-failed-1"))).toBe(false); + + // The worktree was used in the last hour, so a rebuild may still want its installed dependencies. + expect(await cache.prune()).toEqual({ + removed: [{ name: "build-failed-1", reason: "unfinished", bytes: 5, diskBytes: 5 }], + kept: [], + }); + expect(existsSync(worktree)).toBe(true); + + utimesSync(worktree, twoHoursAgo, twoHoursAgo); + const pruned = await cache.prune(); + expect(pruned.removed.map((item) => [item.name, item.reason])).toEqual([["worktree", "idle worktree"]]); + expect(existsSync(worktree)).toBe(false); + expect(git("worktree", "list", "--porcelain").match(/^worktree /gm)).toHaveLength(1); expect(existsSync(cache.entryDir("build-running-2"))).toBe(true); - expect(existsSync(cache.entryDir("worktree"))).toBe(true); + }); + + test("prune rechecks idle age after taking the worktree lock", async () => { + const root = mkdtempSync(join(tmpdir(), "gt-idle-race-")); + roots.push(root); + const git = fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + const worktree = managedWorktree(cache); + git("worktree", "add", "-q", "--detach", worktree, "HEAD"); + const old = new Date(Date.now() - 2 * 3_600_000); + utimesSync(worktree, old, old); + const stat = fsPromises.stat; + let refreshed = false; + const statAndRefresh = async (...args: Parameters) => { + const result = await stat(...args); + if (String(args[0]) === worktree && !refreshed) { + refreshed = true; + const now = new Date(); + utimesSync(worktree, now, now); + } + return result; + }; + // Forwarding every argument and result preserves stat's numeric and bigint overloads. + const spy = spyOn(fsPromises, "stat").mockImplementation(statAndRefresh as typeof stat); + try { + expect(await removeBuildWorktree({ cache, worktree, idleMs: 3_600_000 })).toEqual({ + worktree, + outcome: "recent", + bytes: 0, + }); + expect(refreshed).toBe(true); + expect(existsSync(worktree)).toBe(true); + } finally { + spy.mockRestore(); + } + }); + + test("prune keeps an idle build worktree that is locked, has tracked changes or has a process inside", async () => { + const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-cache-")); + roots.push(root); + const git = fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + const worktree = managedWorktree(cache); + git("worktree", "add", "-q", "--detach", worktree, "HEAD"); + const twoHoursAgo = new Date(Date.now() - 2 * 3_600_000); + const pruneIdle = async () => { + utimesSync(worktree, twoHoursAgo, twoHoursAgo); + const result = await cache.prune(); + return [...prunedNames(result), ...result.kept.map((kept) => kept.outcome)]; + }; + + git("worktree", "lock", "--reason", "agent session", worktree); + expect((await cache.prune()).kept).toEqual([]); + expect(await pruneIdle()).toEqual(["locked"]); + git("worktree", "unlock", worktree); + + writeFileSync(join(worktree, "a.txt"), "edited"); + expect(await pruneIdle()).toEqual(["changed"]); + writeFileSync(join(worktree, "a.txt"), "a"); + + const inside = Bun.spawn(["sleep", "30"], { cwd: worktree, env: process.env }); + try { + expect(await pruneIdle()).toEqual(["in-use"]); + } finally { + inside.kill(); + await inside.exited; + } + + expect(existsSync(worktree)).toBe(true); + // Nothing holds it any more: the same worktree goes. + expect(await pruneIdle()).toEqual(["worktree"]); + expect(existsSync(worktree)).toBe(false); }); }); describe("buildAtCommit", () => { const roots: string[] = []; - afterEach(() => { + afterEach(async () => { + await releaseBuildWorktrees(); for (const root of roots.splice(0)) { rmSync(root, { recursive: true, force: true }); } }); + test("the command's release removes the managed worktree it built in, never a worktree the caller named", async () => { + const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-build-")); + roots.push(root); + const git = fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + // Untracked, not ignored: the install a real build leaves, which only `git worktree remove --force` takes. + const buildCmd = "mkdir -p out node_modules/dep && echo dep > node_modules/dep/index.js && cp a.txt out/a.js"; + const build = (worktree?: string) => + buildAtCommit({ + cache, + repo: join(root, "repo"), + commit: "HEAD", + buildCmd, + outputDir: "out", + worktree, + refresh: worktree !== undefined, + }); + await build(); + const named = join(root, "named"); + expect((await build(named)).reused).toBe(false); + + const released = await releaseBuildWorktrees(); + expect(released.map((removal) => [removal.worktree, removal.outcome])).toEqual([ + [managedWorktree(cache), "removed"], + ]); + expect(existsSync(managedWorktree(cache))).toBe(false); + expect(existsSync(join(named, "node_modules", "dep", "index.js"))).toBe(true); + expect(git("worktree", "list", "--porcelain").match(/^worktree /gm)).toHaveLength(2); + // The output stays in the cache. + expect((await build()).reused).toBe(true); + expect(await releaseBuildWorktrees()).toEqual([]); + }); + + test("a stopping release ends the running build, waits for it, then removes its worktree", async () => { + const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-build-")); + roots.push(root); + fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + const { promise: started, resolve: onStarted } = Promise.withResolvers(); + let childPid = 0; + const building = buildAtCommit({ + cache, + repo: join(root, "repo"), + commit: "HEAD", + buildCmd: `sh -c 'trap "" TERM; echo child:$$; exec sleep 30'; echo finished`, + outputDir: "out", + onLine: (line) => { + if (line.startsWith("child:")) { + childPid = Number(line.slice("child:".length)); + onStarted(); + } + }, + }); + const failed = building.then( + () => null, + (error: unknown) => error + ); + await started; + expect((await removeBuildWorktree({ cache, worktree: managedWorktree(cache) })).outcome).toBe("busy"); + + const released = await releaseBuildWorktrees({ stop: true }); + expect(childPid).toBeGreaterThan(0); + expect(isProcessAlive(childPid)).toBe(false); + expect(String(await failed)).toContain("build command exited"); + expect(released.map((removal) => removal.outcome)).toEqual(["removed"]); + expect(existsSync(managedWorktree(cache))).toBe(false); + }); + + for (const cancellation of ["release", "signal"] as const) { + test(`a zero-exit ${cancellation} cancellation never completes or reuses partial output`, async () => { + const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-cancel-")); + roots.push(root); + fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + const script = join(root, "build.ts"); + const retry = join(root, "retry"); + writeFileSync( + script, + ` + import { existsSync, mkdirSync, writeFileSync } from "node:fs"; + process.on("SIGTERM", () => process.exit(0)); + mkdirSync("out", { recursive: true }); + const retry = existsSync(${SafeJSON.stringify(retry)}); + writeFileSync("out/a.js", retry ? "complete" : "partial"); + if (!retry) { + console.log("ready"); + await Bun.sleep(30_000); + } + ` + ); + const controller = new AbortController(); + const { promise: ready, resolve: onReady } = Promise.withResolvers(); + const options = { + cache, + repo: join(root, "repo"), + commit: "HEAD", + buildCmd: `exec "${process.execPath}" "${script}"`, + outputDir: "out", + // A named worktree isolates cancellation from process-inspection permissions used during removal. + worktree: join(root, "worktree"), + onLine: (line: string) => { + if (line === "ready") { + onReady(); + } + }, + }; + const building = buildAtCommit({ ...options, signal: controller.signal }); + const failed = building.catch((error: unknown) => error); + await ready; + + if (cancellation === "release") { + await releaseBuildWorktrees({ stop: true }); + } else { + controller.abort(); + } + + expect(String(await failed)).toContain("build cancelled"); + expect(await cache.list()).toEqual([]); + writeFileSync(retry, "retry"); + const result = await buildAtCommit(options); + expect(result.reused).toBe(false); + expect(readFileSync(join(result.dir, "files", "a.js"), "utf8")).toBe("complete"); + expect((await buildAtCommit(options)).reused).toBe(true); + }); + } + + for (const stage of ["copy", "remove", "rename", "commit", "after commit"] as const) { + for (const cancellation of ["signal", "release"] as const) { + test(`a refresh cancelled by ${cancellation} during ${stage} preserves old output or invalidates reuse`, async () => { + const root = mkdtempSync(join(tmpdir(), "gt-refresh-cancel-")); + roots.push(root); + fixtureRepo(join(root, "repo")); + const cache = new SourcemapCache("app", join(root, "cache")); + const input = join(root, "input"); + writeFileSync(input, "old complete output"); + const options = { + cache, + repo: join(root, "repo"), + commit: "HEAD", + outputDir: "out", + worktree: join(root, "worktree"), + buildCmd: `mkdir -p out && cp '${input}' out/a.js`, + }; + const first = await buildAtCommit(options); + writeFileSync(input, "replacement output"); + const files = join(first.dir, "files"); + const controller = new AbortController(); + let stopped: Promise | undefined; + let cancelled = false; + const cancel = () => { + cancelled = true; + if (cancellation === "signal") { + controller.abort(); + } else { + stopped = releaseBuildWorktrees({ stop: true }); + } + }; + let restore: () => void; + if (stage === "copy") { + const copy = fsPromises.cp; + const spy = spyOn(fsPromises, "cp").mockImplementation(async (...args) => { + await copy(...args); + if (String(args[1]) === `${files}.staging`) { + cancel(); + } + }); + restore = () => spy.mockRestore(); + } else if (stage === "remove") { + const remove = fsPromises.rm; + const spy = spyOn(fsPromises, "rm").mockImplementation(async (...args) => { + await remove(...args); + if (String(args[0]) === files) { + cancel(); + } + }); + restore = () => spy.mockRestore(); + } else if (stage === "rename") { + const rename = fsPromises.rename; + const spy = spyOn(fsPromises, "rename").mockImplementation(async (...args) => { + await rename(...args); + if (String(args[1]) === files) { + cancel(); + } + }); + restore = () => spy.mockRestore(); + } else { + const commit = cache.commit.bind(cache); + const spy = spyOn(cache, "commit").mockImplementation(async (args) => { + if (stage === "commit") { + cancel(); + return commit(args); + } + const result = await commit(args); + cancel(); + return result; + }); + restore = () => spy.mockRestore(); + } + try { + await expect( + buildAtCommit({ ...options, refresh: true, signal: controller.signal }) + ).rejects.toThrow("build cancelled"); + await stopped; + expect(cancelled).toBe(true); + } finally { + restore(); + } + const reusable = await cache.get(first.entry.id); + if (reusable) { + expect(readFileSync(join(files, "a.js"), "utf8")).toBe("old complete output"); + expect((await buildAtCommit(options)).reused).toBe(true); + } else { + expect((await buildAtCommit(options)).reused).toBe(false); + expect(readFileSync(join(files, "a.js"), "utf8")).toBe("replacement output"); + } + }); + } + } + test("a refresh replaces the cached output, and a different env is a different build", async () => { const root = mkdtempSync(join(tmpdir(), "gt-sourcemaps-build-")); roots.push(root); @@ -391,9 +745,13 @@ describe("buildAtCommit", () => { const first = await build("a"); expect(readdirSync(join(first.dir, "files"))).toEqual(["chunk-1.js"]); + const linked = join(root, "linked-output"); + await cache.linkOut(first.entry.id, linked, "files"); const refreshed = await build("a", true); expect(refreshed.dir).toBe(first.dir); expect(readdirSync(join(refreshed.dir, "files"))).toEqual(["chunk-2.js"]); + expect(refreshed.entry.createdAt).toBe(first.entry.createdAt); + expect(refreshed.entry.links).toEqual([linked]); const otherEnv = await build("b"); expect(otherEnv.reused).toBe(false); diff --git a/src/youtube/lib/cache.ts b/src/youtube/lib/cache.ts index 20d339e464..c86f81b487 100644 --- a/src/youtube/lib/cache.ts +++ b/src/youtube/lib/cache.ts @@ -4,12 +4,12 @@ import { dirname, join } from "node:path"; import type { CacheLayout } from "@app/youtube/lib/cache.types"; import type { ChannelHandle } from "@app/youtube/lib/channel.types"; import type { VideoId } from "@app/youtube/lib/video.types"; -import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; // Root via `env.tools.getHome()` (falls back to `homedir()`) so the test // sandbox's GENESIS_TOOLS_HOME is honoured and no test can prune real media. -export const DEFAULT_CACHE_DIR = join(env.tools.getHome(), ".genesis-tools", "youtube", "cache"); +export const DEFAULT_CACHE_DIR = genesisToolsDir("youtube", "cache"); export function videoDir(layout: CacheLayout, handle: ChannelHandle, id: VideoId): string { const safeHandle = handle.replace(/^@/, "").replace(/[^a-zA-Z0-9_-]/g, "_"); diff --git a/src/youtube/lib/config.ts b/src/youtube/lib/config.ts index 4823af54e2..ebfa6b3297 100644 --- a/src/youtube/lib/config.ts +++ b/src/youtube/lib/config.ts @@ -3,11 +3,11 @@ import { dirname, join } from "node:path"; import { isDeepStrictEqual } from "node:util"; import type { YoutubeConfigInit, YoutubeConfigPatch } from "@app/youtube/lib/config.api.types"; import type { YoutubeConfigShape } from "@app/youtube/lib/config.types"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { withFileLock } from "@genesiscz/utils/storage"; import { atomicWriteFileSync, isJsonObject } from "@genesiscz/utils/storage/atomic-file"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; import { WEB_SERVICES } from "@genesiscz/utils/ui/dashboards"; const API_PORT = WEB_SERVICES["youtube-server"].port; @@ -36,7 +36,7 @@ function withoutLegacyPort(patch: YoutubeConfigPatch): YoutubeConfigPatch { // `env.tools.getHome()` falls back to `homedir()`, so production is unchanged. // It reads GENESIS_TOOLS_HOME, which the test preload points at a tmp dir — // with a bare `homedir()` here the sandbox cannot protect the real server.json. -export const DEFAULT_BASE_DIR = join(env.tools.getHome(), ".genesis-tools", "youtube"); +export const DEFAULT_BASE_DIR = genesisToolsDir("youtube"); export const CONFIG_FILENAME = "server.json"; export const DEFAULT_YOUTUBE_CONFIG: YoutubeConfigShape = { diff --git a/src/youtube/lib/db.ts b/src/youtube/lib/db.ts index f3de85c33b..8835e8cdc9 100644 --- a/src/youtube/lib/db.ts +++ b/src/youtube/lib/db.ts @@ -1,4 +1,3 @@ -import { join } from "node:path"; import { deleteIfExists } from "@app/youtube/lib/cache"; import type { Channel, ChannelHandle } from "@app/youtube/lib/channel.types"; import type { FetchedComment, VideoComment } from "@app/youtube/lib/comments.types"; @@ -83,15 +82,15 @@ import type { VideoReport, } from "@app/youtube/lib/video.types"; import { BaseDatabase, SQL_NOW_UTC } from "@genesiscz/utils/database"; -import { env } from "@genesiscz/utils/env"; import { SafeJSON } from "@genesiscz/utils/json"; import { logger } from "@genesiscz/utils/logger"; import { withFileLock } from "@genesiscz/utils/storage"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; // Root via `env.tools.getHome()` (falls back to `homedir()`) so the test // sandbox's GENESIS_TOOLS_HOME is honoured — a bare `homedir()` here would let // any test constructing `new YoutubeDatabase()` write the user's real library. -export const DEFAULT_DB_PATH = join(env.tools.getHome(), ".genesis-tools", "youtube", "youtube.db"); +export const DEFAULT_DB_PATH = genesisToolsDir("youtube", "youtube.db"); const SCHEMA_VERSION = 2; diff --git a/src/youtube/lib/server/port-file.ts b/src/youtube/lib/server/port-file.ts index 6ba162f018..938ab14480 100644 --- a/src/youtube/lib/server/port-file.ts +++ b/src/youtube/lib/server/port-file.ts @@ -1,11 +1,11 @@ import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; -import { env } from "@genesiscz/utils/env"; +import { genesisToolsDir } from "@genesiscz/utils/storage/root"; // Root via `env.tools.getHome()` (falls back to `homedir()`) so the test // sandbox's GENESIS_TOOLS_HOME is honoured — a test writing the real port file // would point every CLI at a port the running launchd server does not own. -export const SERVER_BASE_DIR = join(env.tools.getHome(), ".genesis-tools", "youtube-server"); +export const SERVER_BASE_DIR = genesisToolsDir("youtube-server"); export const PORT_FILE = join(SERVER_BASE_DIR, "port"); export interface PortFileOptions { diff --git a/tools b/tools index b4c1449a31..4eeb990651 100755 --- a/tools +++ b/tools @@ -6,10 +6,12 @@ import { basename, join, resolve } from "node:path"; import { namedBunExecPath } from "@genesiscz/utils/cli/bun-link"; import { takeDetachedMarker } from "@genesiscz/utils/cli/detached"; import { planToolExec, terminateOrphanedChild } from "@genesiscz/utils/cli/tool-exec"; +import { skipWorktreeInstall } from "@genesiscz/utils/cli/worktree-install"; import { env } from "@genesiscz/utils/env"; import { logger } from "@genesiscz/utils/logger"; import { refreshInstallRecord } from "@genesiscz/utils/install-record"; import { genesisAppLauncher } from "@genesiscz/utils/macos/genesis-app"; +import { shellQuote } from "@genesiscz/utils/shell/quote"; import { diagnose } from "./scripts/test-deps"; // --- Configuration --- @@ -291,6 +293,17 @@ async function runWithReinstallGuard(args: string[], workspaceRoot: string, exec return exitCode; } + // A linked worktree (its .git is a file) never installs by itself: a fresh install writes new native binaries + // (tsgo, biome, node-gyp builds), and on 2026-10-14 each one waited minutes for syspolicyd's first-run check + // while several review workers shared the machine. Print the repair instead; TOOLS_AUTOINSTALL=1 opts in. + if (skipWorktreeInstall({ workspaceRoot, autoInstall: env.get("TOOLS_AUTOINSTALL") })) { + process.stderr.write( + "\n\x1b[33m[tools]\x1b[0m Likely node_modules mismatch in this worktree. Not installing automatically here;\n" + + `\x1b[33m[tools]\x1b[0m repair it with: cd ${shellQuote(workspaceRoot)} && bun install (or rerun with TOOLS_AUTOINSTALL=1)\n` + ); + return exitCode; + } + process.stderr.write( "\n\x1b[33m[tools]\x1b[0m Detected a likely node_modules mismatch (duplicate React / missing module).\n" );