From 28e0668648a33bd447b166f972510d67cac21a84 Mon Sep 17 00:00:00 2001 From: John McChesney TenEyck Jr <59268465+jmcte@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:21:27 +0100 Subject: [PATCH 1/3] Resolve SQLite snapshot symlinks Signed-off-by: John McChesney TenEyck Jr <59268465+jmcte@users.noreply.github.com> --- .../ICloudCLICore/SQLiteSnapshotQuery.swift | 7 ++++--- .../SQLiteSnapshotQueryTests.swift | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift index 966cf92..2e80726 100644 --- a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift +++ b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift @@ -89,7 +89,8 @@ public struct SQLiteSnapshotQueryEngine: Sendable { } func withSnapshot(_ operation: (URL, URL) throws -> Result) throws -> Result { - guard FileManager.default.fileExists(atPath: source.path) else { + let resolvedSource = source.resolvingSymlinksInPath() + guard FileManager.default.fileExists(atPath: resolvedSource.path) else { throw LocalInventoryError.missingStore(reportedStore) } @@ -107,9 +108,9 @@ public struct SQLiteSnapshotQueryEngine: Sendable { let snapshot = directory.appendingPathComponent("snapshot.sqlite") do { - try copy(source, to: snapshot) + try copy(resolvedSource, to: snapshot) for suffix in ["-wal", "-shm"] { - let companion = URL(fileURLWithPath: source.path + suffix) + let companion = URL(fileURLWithPath: resolvedSource.path + suffix) guard FileManager.default.fileExists(atPath: companion.path) else { continue } try copy(companion, to: URL(fileURLWithPath: snapshot.path + suffix)) } diff --git a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift index a91284e..94c04fb 100644 --- a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift +++ b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift @@ -32,6 +32,25 @@ private struct SnapshotValueRow: Decodable, Equatable { let value: String } #expect(rows == [SnapshotValueRow(value: "wal-row")]) } +@Test func snapshotQueryFollowsSymlinkedStoreAndCopiesWALCompanions() throws { + let root = try temporarySQLiteSnapshotDirectory(named: "symlink-wal") + defer { try? FileManager.default.removeItem(at: root) } + let liveStore = root.appendingPathComponent("live/source.db") + try FileManager.default.createDirectory(at: liveStore.deletingLastPathComponent(), withIntermediateDirectories: true) + let writer = try openWALFixture(database: liveStore) + defer { writer.stop() } + let linkedStore = root.appendingPathComponent("linked-source.db") + try FileManager.default.createSymbolicLink(atPath: linkedStore.path, withDestinationPath: liveStore.path) + + let engine = SQLiteSnapshotQueryEngine(source: linkedStore) + try engine.withSnapshot { snapshot, workspace in + #expect((try? FileManager.default.destinationOfSymbolicLink(atPath: snapshot.path)) == nil) + #expect(FileManager.default.fileExists(atPath: snapshot.path + "-wal")) + let rows: [SnapshotValueRow] = try engine.querySnapshot(snapshot, workspace: workspace, sql: "SELECT value FROM values_table ORDER BY value;") + #expect(rows == [SnapshotValueRow(value: "wal-row")]) + } +} + @Test func snapshotQueryMapsSchemaDriftAndBusyFailures() { let schema = sqliteError(from: Data("Error: no such table: missing".utf8), store: "/private/source.db") #expect(schema == .unsupportedSchema(store: "/private/source.db", detail: "Error: no such table: missing")) From 1d506499d6202938db83b9b7804e2315850a3d9b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 23 Jul 2026 01:06:55 +0100 Subject: [PATCH 2/3] Create coherent SQLite snapshots # Conflicts: # docs/sqlite-snapshots.md --- .../ICloudCLICore/SQLiteSnapshotQuery.swift | 52 ++++++++++++++----- .../SQLiteSnapshotQueryTests.swift | 4 +- docs/sqlite-snapshots.md | 4 +- 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift index 2e80726..b21e1f9 100644 --- a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift +++ b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift @@ -108,26 +108,54 @@ public struct SQLiteSnapshotQueryEngine: Sendable { let snapshot = directory.appendingPathComponent("snapshot.sqlite") do { - try copy(resolvedSource, to: snapshot) - for suffix in ["-wal", "-shm"] { - let companion = URL(fileURLWithPath: resolvedSource.path + suffix) - guard FileManager.default.fileExists(atPath: companion.path) else { continue } - try copy(companion, to: URL(fileURLWithPath: snapshot.path + suffix)) - } + try createSQLiteSnapshot(from: resolvedSource, to: snapshot) + } catch let error as LocalInventoryError { + throw error } catch { - if isPermissionError(error) { throw LocalInventoryError.permissionDenied(reportedStore) } throw LocalInventoryError.sqliteFailure("Unable to create SQLite snapshot") } return try operation(snapshot, directory) } - private func copy(_ source: URL, to destination: URL) throws { - try FileManager.default.copyItem(at: source, to: destination) + private func createSQLiteSnapshot(from source: URL, to destination: URL) throws { + let process = Process() + let errors = Pipe() + process.executableURL = URL(fileURLWithPath: "/usr/bin/sqlite3") + process.arguments = [ + "-cmd", ".timeout \(busyTimeoutMilliseconds)", + source.path, + "VACUUM INTO '\(sqliteLiteral(destination.path))';", + ] + process.standardOutput = FileHandle.nullDevice + process.standardError = errors + + let completed = DispatchSemaphore(value: 0) + process.terminationHandler = { _ in completed.signal() } + do { + try process.run() + } catch { + throw LocalInventoryError.sqliteFailure(error.localizedDescription) + } + guard completed.wait(timeout: .now() + timeout) == .success else { + if process.isRunning { process.terminate() } + if completed.wait(timeout: .now() + 1) != .success, process.isRunning { + Darwin.kill(process.processIdentifier, SIGKILL) + _ = completed.wait(timeout: .now() + 1) + } + throw LocalInventoryError.queryTimeout(reportedStore) + } + + let errorData = errors.fileHandleForReading.readDataToEndOfFile() + guard process.terminationStatus == 0 else { + if String(decoding: errorData, as: UTF8.self).lowercased().contains("unable to open database") { + throw LocalInventoryError.permissionDenied(reportedStore) + } + throw sqliteError(from: errorData, store: reportedStore) + } try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: destination.path) } - private func isPermissionError(_ error: Error) -> Bool { - let cocoa = error as NSError - return cocoa.domain == NSCocoaErrorDomain && [NSFileReadNoPermissionError, NSFileWriteNoPermissionError].contains(cocoa.code) + private func sqliteLiteral(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") } } diff --git a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift index 94c04fb..25d11a3 100644 --- a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift +++ b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift @@ -32,7 +32,7 @@ private struct SnapshotValueRow: Decodable, Equatable { let value: String } #expect(rows == [SnapshotValueRow(value: "wal-row")]) } -@Test func snapshotQueryFollowsSymlinkedStoreAndCopiesWALCompanions() throws { +@Test func snapshotQueryFollowsSymlinkedStoreWithCoherentWALSnapshot() throws { let root = try temporarySQLiteSnapshotDirectory(named: "symlink-wal") defer { try? FileManager.default.removeItem(at: root) } let liveStore = root.appendingPathComponent("live/source.db") @@ -45,7 +45,7 @@ private struct SnapshotValueRow: Decodable, Equatable { let value: String } let engine = SQLiteSnapshotQueryEngine(source: linkedStore) try engine.withSnapshot { snapshot, workspace in #expect((try? FileManager.default.destinationOfSymbolicLink(atPath: snapshot.path)) == nil) - #expect(FileManager.default.fileExists(atPath: snapshot.path + "-wal")) + #expect(!FileManager.default.fileExists(atPath: snapshot.path + "-wal")) let rows: [SnapshotValueRow] = try engine.querySnapshot(snapshot, workspace: workspace, sql: "SELECT value FROM values_table ORDER BY value;") #expect(rows == [SnapshotValueRow(value: "wal-row")]) } diff --git a/docs/sqlite-snapshots.md b/docs/sqlite-snapshots.md index 7eb947a..5a87dc8 100644 --- a/docs/sqlite-snapshots.md +++ b/docs/sqlite-snapshots.md @@ -2,9 +2,9 @@ Apple applications often keep committed SQLite state across a database file and its write-ahead log. Querying the live path directly can produce inconsistent reads, wait indefinitely on a busy store, or interact with an Apple-owned database in ways the CLI does not intend. -`SQLiteSnapshotQueryEngine` creates a mode-`0700` temporary directory, copies the database and any present `-wal` and `-shm` companions as mode-`0600` files, and queries only that private copy. The `sqlite3` process runs with `-readonly`, `PRAGMA query_only=ON`, a one-second busy timeout, and a ten-second process timeout. The snapshot, query output, and error output are deleted before the call returns or throws. Copied contents are never logged. +`SQLiteSnapshotQueryEngine` creates a mode-`0700` temporary directory and asks SQLite to create a consistent private snapshot with `VACUUM INTO`. SQLite coordinates the main database and WAL under one read view, so a live writer cannot leave a mixed-generation main/WAL copy. The resulting snapshot is mode-`0600` and queried with `-readonly`, `PRAGMA query_only=ON`, a bounded busy timeout, and a bounded process timeout. Production callers use `SQLiteSnapshotQueryEngine.production`, which enforces minimums of 500ms busy timeout and 5s process timeout. The snapshot, query output, and error output are deleted before the call returns or throws. Snapshot contents are never logged. -Errors retain the original store path and distinguish missing stores, permission denial, unsupported schemas, locked or busy stores, and timeouts. The implementation does not checkpoint, lock, vacuum, or write to the Apple-owned source. +Errors retain the original store path and distinguish missing stores, permission denial, unsupported schemas, locked or busy stores, and timeouts. `VACUUM INTO` reads the Apple-owned source without checkpointing, mutating, or creating source-side files. ## Migration status From 1589c65691eb5414fe5daf61d95960f29db0ba89 Mon Sep 17 00:00:00 2001 From: Hermes Date: Fri, 24 Jul 2026 08:48:54 +0100 Subject: [PATCH 3/3] Enforce production SQLite snapshot timeouts Add and adopt the documented production snapshot factory so live SQLite inventory reads enforce 5-second process and 500-millisecond busy-timeout floors. Cover clamped and higher caller values. --- Sources/ICloudCLICore/LocalInventories.swift | 2 +- .../ICloudCLICore/SQLiteSnapshotQuery.swift | 16 ++++++++++++++++ .../SQLiteSnapshotQueryTests.swift | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Sources/ICloudCLICore/LocalInventories.swift b/Sources/ICloudCLICore/LocalInventories.swift index 8b00497..c66e0c4 100644 --- a/Sources/ICloudCLICore/LocalInventories.swift +++ b/Sources/ICloudCLICore/LocalInventories.swift @@ -696,7 +696,7 @@ public struct LocalSQLiteInventoryReader: Sendable { } private func withSnapshot(_ operation: (LocalSQLiteInventoryReader) throws -> Result) throws -> Result { - try SQLiteSnapshotQueryEngine(source: database).withSnapshot { snapshot, workspace in + try SQLiteSnapshotQueryEngine.production(source: database).withSnapshot { snapshot, workspace in try operation(LocalSQLiteInventoryReader(database: snapshot, snapshotsLiveStores: false, reportedStore: reportedStore, snapshotWorkspace: workspace)) } } diff --git a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift index b21e1f9..1bf3b3f 100644 --- a/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift +++ b/Sources/ICloudCLICore/SQLiteSnapshotQuery.swift @@ -22,6 +22,22 @@ public struct SQLiteSnapshotQueryEngine: Sendable { self.reportedStore = reportedStore ?? source.path } + public static func production( + source: URL, + timeout: TimeInterval = 10, + busyTimeoutMilliseconds: Int = 1_000, + temporaryRoot: URL = FileManager.default.temporaryDirectory, + reportedStore: String? = nil + ) -> Self { + Self( + source: source, + timeout: max(5, timeout), + busyTimeoutMilliseconds: max(500, busyTimeoutMilliseconds), + temporaryRoot: temporaryRoot, + reportedStore: reportedStore + ) + } + public func query(_ sql: String) throws -> [T] { try withSnapshot { snapshot, directory in try querySnapshot(snapshot, workspace: directory, sql: sql) diff --git a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift index 25d11a3..66e529a 100644 --- a/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift +++ b/Tests/ICloudCLICoreTests/SQLiteSnapshotQueryTests.swift @@ -4,6 +4,25 @@ import Testing private struct SnapshotValueRow: Decodable, Equatable { let value: String } +@Test func productionSnapshotEngineEnforcesTimeoutFloors() { + let source = URL(fileURLWithPath: "/private/source.db") + let constrained = SQLiteSnapshotQueryEngine.production( + source: source, + timeout: 0.001, + busyTimeoutMilliseconds: 0 + ) + let relaxed = SQLiteSnapshotQueryEngine.production( + source: source, + timeout: 7, + busyTimeoutMilliseconds: 800 + ) + + #expect(constrained.timeout == 5) + #expect(constrained.busyTimeoutMilliseconds == 500) + #expect(relaxed.timeout == 7) + #expect(relaxed.busyTimeoutMilliseconds == 800) +} + @Test func snapshotQueryReadsDatabaseWithoutCompanionFilesAndCleansUp() throws { let root = try temporarySQLiteSnapshotDirectory(named: "no-companions") defer { try? FileManager.default.removeItem(at: root) }