diff --git a/README.md b/README.md index 1908e56..3fb3057 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,19 @@ brew services start sand The service restarts sand if it exits, runs it as an interactive process so launchd doesn't throttle its CPU and I/O (or the VMs it starts), and writes output to `~/Library/Logs/sand.log`. Stop it with `brew services stop sand`. +To let active jobs finish before stopping the service: + +```bash +sand drain --config ~/sand.yml +brew services stop sand +``` + +`sand drain` waits for runner cleanup, then reports completion. Idle GitHub runners are deregistered before stopping their VMs; busy ephemeral runners finish their current job. Script provisioners finish the current script and post-run hook. Sand stops creating replacement VMs and stays alive but idle so Homebrew does not restart it. Use `brew services restart sand` to resume work. + +Use the same configuration path as the running service. Sand keeps small `.sand-lock`, `.sand-drain`, and `.sand-drained` files beside the config, so its directory must be writable. These files are removed when sand exits normally or handles SIGINT/SIGTERM, including `brew services stop sand`. They remain while sand is drained and idle. After a crash or SIGKILL, leftover files are ignored by the next instance and removed when it exits. Only one sand process can use a configuration at a time. The command fails if no service is running or it stops before completion. Interrupting the drain command does not cancel the drain request. + +If GitHub cannot confirm that a runner is idle, sand leaves it running and logs the issue; draining can wait indefinitely for a job or API recovery. A job assigned concurrently with the drain request may still finish on that ephemeral runner. Normal Ctrl+C, SIGTERM, and `brew services stop sand` remain immediate stops, so wait for `sand drain` to complete first. + ## Logs sand logs to macOS default logging system using `os_log`. To see the log diff --git a/Sources/sand/Drain.swift b/Sources/sand/Drain.swift new file mode 100644 index 0000000..9b21269 --- /dev/null +++ b/Sources/sand/Drain.swift @@ -0,0 +1,17 @@ +import ArgumentParser +import Foundation + +@available(macOS 15.0, *) +struct Drain: AsyncParsableCommand { + static let configuration = CommandConfiguration(abstract: "Wait for active jobs to finish and leave sand idle.") + + @Option(name: .shortAndLong, help: "Configuration used by the running sand service.") + var config: String = Config.defaultPath + + func run() async throws { + let (paths, instance) = try ServiceControl.requestDrain(configPath: config) + print("Drain requested. Waiting for active jobs and VM cleanup...") + try await ServiceControl.waitUntilDrained(paths: paths, instance: instance) + print("Drain complete. sand is idle; it is safe to stop or restart the service.") + } +} diff --git a/Sources/sand/GitHub/GitHubService.swift b/Sources/sand/GitHub/GitHubService.swift index 75388df..568fbd3 100644 --- a/Sources/sand/GitHub/GitHubService.swift +++ b/Sources/sand/GitHub/GitHubService.swift @@ -28,6 +28,7 @@ struct GitHubService: Sendable { struct Runner: Decodable { let id: Int let name: String + let busy: Bool? } let runners: [Runner] @@ -48,16 +49,29 @@ struct GitHubService: Sendable { } func deleteRunner(named name: String) async throws -> Bool { + try await removeRunner(named: name, onlyIfIdle: false) == .removed + } + + enum RemovalResult: Equatable { case removed, missing, busy } + + func removeRunner(named name: String, onlyIfIdle: Bool) async throws -> RemovalResult { let installationId = try await installationID() let accessToken = try await installationAccessToken(installationId: installationId) let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-._~")) let encodedName = name.addingPercentEncoding(withAllowedCharacters: allowed) ?? name let response: RunnersResponse = try await request(path: "\(runnersPath())?name=\(encodedName)", method: "GET", token: accessToken) guard let runner = response.runners.first(where: { $0.name == name }) else { - return false + return .missing + } + // Missing busy information is not evidence that it is safe to stop a runner. + if onlyIfIdle, runner.busy != false { return .busy } + do { + _ = try await send(path: "\(runnersPath())/\(runner.id)", method: "DELETE", token: accessToken) + } catch GitHubServiceError.httpError(let status, _) where onlyIfIdle && status == 422 { + // GitHub can assign a job between the lookup and deletion. Let it finish. + return .busy } - _ = try await send(path: "\(runnersPath())/\(runner.id)", method: "DELETE", token: accessToken) - return true + return .removed } diff --git a/Sources/sand/Runner.swift b/Sources/sand/Runner.swift index 1eee67d..4addabe 100644 --- a/Sources/sand/Runner.swift +++ b/Sources/sand/Runner.swift @@ -6,6 +6,7 @@ struct Runner: Sendable { let provisioner: GitHubProvisioner let runnerVersionResolver: GitHubRunnerVersionResolver let runnerCache: RunnerCache + let serviceControl: ServiceControl? let config: Config.RunnerConfig let shutdownCoordinator: VMShutdownCoordinator let control: RunnerControl @@ -36,13 +37,15 @@ struct Runner: Sendable { vmName: String, logLabel: String, logLevel: LogLevel, - logSink: LogFileSink? + logSink: LogFileSink?, + serviceControl: ServiceControl? = nil ) { self.tart = tart self.github = github self.provisioner = provisioner self.runnerVersionResolver = runnerVersionResolver self.runnerCache = runnerCache + self.serviceControl = serviceControl self.config = config self.shutdownCoordinator = shutdownCoordinator self.control = control @@ -52,34 +55,30 @@ struct Runner: Sendable { } func run() async throws { - if let stopAfter = config.stopAfter { - guard stopAfter > 0 else { - return - } - for _ in 0..= maxRetries { let statusLabel = lastStatus.map(statusLabel) ?? "unknown" let statusErrorLabel = lastStatusError ?? "none" @@ -606,10 +619,13 @@ struct Runner: Sendable { _ commands: [String], ssh: SSHClient, healthCheckState: HealthCheckState, - secrets: [String] = [] + secrets: [String] = [], + githubRunnerName: String? = nil ) async -> ProvisionerSequenceOutcome { for command in commands { - let outcome = await runProvisionerCommand(command, ssh: ssh, healthCheckState: healthCheckState, secrets: secrets) + if drainRequested { return .completed } + let outcome = await runProvisionerCommand(command, ssh: ssh, healthCheckState: healthCheckState, secrets: secrets, + githubRunnerName: isRunnerCommand(command) ? githubRunnerName : nil) switch outcome { case .completed: continue @@ -626,7 +642,8 @@ struct Runner: Sendable { _ command: String, ssh: SSHClient, healthCheckState: HealthCheckState, - secrets: [String] + secrets: [String], + githubRunnerName: String? ) async -> ProvisionerOutcome { let displayCommand = Runner.redact(command, secrets: secrets) logScript(displayCommand) @@ -639,7 +656,7 @@ struct Runner: Sendable { let handle = try ssh.start(command: command) await control.setProvisioningHandle(handle) logger.debug("\(labeledCommand) started; awaiting completion or healthCheck failure") - let outcome = await awaitProvisionerCommand(handle: handle, healthCheckState: healthCheckState) + let outcome = await awaitProvisionerCommand(handle: handle, healthCheckState: healthCheckState, githubRunnerName: githubRunnerName) switch outcome { case let .completed(result): let stdoutLabel = commandLabel.isEmpty ? "stdout" : "stdout (\(commandLabel))" @@ -680,7 +697,8 @@ struct Runner: Sendable { private func awaitProvisionerCommand( handle: ProcessHandle, - healthCheckState: HealthCheckState + healthCheckState: HealthCheckState, + githubRunnerName: String? ) async -> ProvisionerOutcome { await withTaskGroup(of: ProvisionerOutcome?.self) { group in group.addTask { @@ -701,6 +719,34 @@ struct Runner: Sendable { return .failed(error) } } + if let githubRunnerName, let github, let serviceControl { + group.addTask { + while !Task.isCancelled { + if serviceControl.drainRequested { + do { + switch try await github.removeRunner(named: githubRunnerName, onlyIfIdle: true) { + case .removed: + logger.info("drain: removed idle runner \(githubRunnerName)") + await handle.terminate() + return nil + case .busy: + logger.info("drain: waiting for \(githubRunnerName) to finish its job") + return nil + case .missing: + // Absence from the API alone is not proof the guest has finished. + break + } + } catch { + if Task.isCancelled { return nil } + logger.warning("drain: cannot confirm runner is idle; leaving it running: \(error)") + } + } + do { try await Task.sleep(for: .seconds(serviceControl.drainRequested ? 5 : 1)) } + catch { return nil } + } + return nil + } + } while let outcome = await group.next() { if let outcome { logger.debug("provisioner outcome received: \(provisionerOutcomeLabel(outcome))") @@ -766,6 +812,7 @@ struct Runner: Sendable { } private func retrySSHIfNeeded(error: Error, stage: String, attempt: inout Int) async -> Bool { + guard !drainRequested else { return false } guard shouldRetrySSH(error), attempt < sshRetryDelays.count else { return false } diff --git a/Sources/sand/Sand.swift b/Sources/sand/Sand.swift index 839691d..ed690af 100644 --- a/Sources/sand/Sand.swift +++ b/Sources/sand/Sand.swift @@ -6,7 +6,7 @@ import Foundation @available(macOS 15.0, *) struct Sand: AsyncParsableCommand { static let configuration = CommandConfiguration( - subcommands: [Run.self, Destroy.self, Doctor.self, Validate.self] + subcommands: [Run.self, Drain.self, Destroy.self, Doctor.self, Validate.self] ) } @@ -29,7 +29,8 @@ struct Run: AsyncParsableCommand { if !missing.isEmpty { throw ValidationError("Missing required dependencies in PATH: \(missing.joined(separator: ", ")). Install them and re-run.") } - let config = try Config.load(path: config) + let configPath = config + let config = try Config.load(path: configPath) let validator = ConfigValidator() let issues = validator.validate(config) let errors = issues.filter { $0.severity == .error } @@ -59,6 +60,8 @@ struct Run: AsyncParsableCommand { return } + let serviceControl = try ServiceControl(configPath: configPath) + defer { serviceControl.cleanup() } let provisioner = GitHubProvisioner() let runnerVersionResolver = GitHubRunnerVersionResolver() let runnerCache = RunnerCache() @@ -89,7 +92,8 @@ struct Run: AsyncParsableCommand { vmName: runnerName, logLabel: logLabel, logLevel: level, - logSink: logSink + logSink: logSink, + serviceControl: serviceControl ) runners.append(runner) } @@ -112,6 +116,7 @@ struct Run: AsyncParsableCommand { } } group.wait() + serviceControl.cleanup() } defer { _ = signalHandler @@ -124,6 +129,12 @@ struct Run: AsyncParsableCommand { } try await group.waitForAll() } + if serviceControl.drainRequested { + try serviceControl.markDrained() + logger.info("drain complete; sand is idle until stopped or restarted") + // Stay alive so launchd's KeepAlive does not create new runners. + while true { try await Task.sleep(for: .seconds(3600)) } + } } private func githubService(for provisioner: Config.Provisioner?) throws -> GitHubService? { diff --git a/Sources/sand/ServiceControl.swift b/Sources/sand/ServiceControl.swift new file mode 100644 index 0000000..69739d0 --- /dev/null +++ b/Sources/sand/ServiceControl.swift @@ -0,0 +1,141 @@ +import Darwin +import Foundation +import Synchronization + +/// A config-specific lock and request file avoid PID reuse and signalling the wrong process. +final class ServiceControl: Sendable { + struct Paths: Sendable { + let lock: String + let request: String + let complete: String + + init(configPath: String) { + let config = URL(fileURLWithPath: Config.expandPath(configPath)).standardizedFileURL.resolvingSymlinksInPath().path + lock = config + ".sand-lock" + request = config + ".sand-drain" + complete = config + ".sand-drained" + } + } + + enum ControlError: Error, CustomStringConvertible { + case alreadyRunning, notRunning, serviceStopped, system(String) + + var description: String { + switch self { + case .alreadyRunning: "sand is already running with this configuration." + case .notRunning: "No running sand service found for this configuration." + case .serviceStopped: "sand stopped or restarted before drain completion." + case let .system(message): message + } + } + } + + let paths: Paths + let instance: String + private let descriptor: Int32 + private let stopped = Mutex(false) + + init(configPath: String) throws { + paths = Paths(configPath: configPath) + instance = UUID().uuidString + var fd: Int32 + while true { + fd = open(paths.lock, O_RDWR | O_CREAT | O_NOFOLLOW | O_CLOEXEC, 0o600) + guard fd >= 0 else { throw ControlError.system(String(cString: strerror(errno))) } + guard flock(fd, LOCK_EX | LOCK_NB) == 0 else { + close(fd) + throw ControlError.alreadyRunning + } + // Shutdown may have unlinked this inode after we opened it. + var opened = stat() + var current = stat() + if fstat(fd, &opened) == 0, lstat(paths.lock, ¤t) == 0, + opened.st_dev == current.st_dev, opened.st_ino == current.st_ino { break } + close(fd) + } + let bytes = Array(instance.utf8) + guard ftruncate(fd, 0) == 0, + bytes.withUnsafeBytes({ Darwin.write(fd, $0.baseAddress, $0.count) }) == bytes.count else { + let message = String(cString: strerror(errno)) + unlink(paths.lock) + close(fd) + throw ControlError.system(message) + } + do { + // Clients only open this existing file, so a late request cannot recreate it after shutdown. + try Self.write("", to: paths.request) + } catch { + unlink(paths.lock) + close(fd) + throw error + } + descriptor = fd + } + + deinit { cleanup() } + + func cleanup() { + stopped.withLock { stopped in + guard !stopped else { return } + stopped = true + unlink(paths.request) + unlink(paths.complete) + // Remove the lock last, while we still own it. + unlink(paths.lock) + close(descriptor) + } + } + + var drainRequested: Bool { + (try? String(contentsOfFile: paths.request, encoding: .utf8)) == instance + } + + func markDrained() throws { + try stopped.withLock { stopped in + guard !stopped else { throw ControlError.serviceStopped } + try Self.write(instance, to: paths.complete) + } + } + + static func requestDrain(configPath: String) throws -> (Paths, String) { + let paths = Paths(configPath: configPath) + guard isRunning(paths) else { throw ControlError.notRunning } + let instance = try String(contentsOfFile: paths.lock, encoding: .utf8) + guard !instance.isEmpty else { throw ControlError.notRunning } + let request = open(paths.request, O_WRONLY | O_NOFOLLOW | O_CLOEXEC) + guard request >= 0 else { throw ControlError.serviceStopped } + defer { close(request) } + guard isRunning(paths), + (try? String(contentsOfFile: paths.lock, encoding: .utf8)) == instance else { + throw ControlError.serviceStopped + } + let bytes = Array(instance.utf8) + guard bytes.withUnsafeBytes({ pwrite(request, $0.baseAddress, $0.count, 0) }) == bytes.count else { + throw ControlError.system(String(cString: strerror(errno))) + } + return (paths, instance) + } + + static func waitUntilDrained(paths: Paths, instance: String) async throws { + while true { + guard isRunning(paths), + (try? String(contentsOfFile: paths.lock, encoding: .utf8)) == instance else { + throw ControlError.serviceStopped + } + if (try? String(contentsOfFile: paths.complete, encoding: .utf8)) == instance { return } + try await Task.sleep(for: .seconds(1)) + } + } + + private static func isRunning(_ paths: Paths) -> Bool { + let descriptor = open(paths.lock, O_RDWR | O_NOFOLLOW | O_CLOEXEC) + guard descriptor >= 0 else { return false } + defer { close(descriptor) } + if flock(descriptor, LOCK_EX | LOCK_NB) == 0 { return false } + return errno == EWOULDBLOCK + } + + private static func write(_ value: String, to path: String) throws { + try value.write(toFile: path, atomically: true, encoding: .utf8) + } +} diff --git a/Tests/sandTests/DrainTests.swift b/Tests/sandTests/DrainTests.swift new file mode 100644 index 0000000..c18b98c --- /dev/null +++ b/Tests/sandTests/DrainTests.swift @@ -0,0 +1,186 @@ +import Foundation +import Testing +import Synchronization +@testable import sand + +private func drainConfigPath() throws -> String { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: false) + return directory.appendingPathComponent("sand.yml").path +} + +@Test func drainControlRequestsCompletionAndRejectsDuplicateService() async throws { + let path = try drainConfigPath() + defer { try? FileManager.default.removeItem(atPath: (path as NSString).deletingLastPathComponent) } + let control = try ServiceControl(configPath: path) + #expect(!control.drainRequested) + #expect(throws: ServiceControl.ControlError.self) { try ServiceControl(configPath: path) } + let (paths, instance) = try ServiceControl.requestDrain(configPath: path) + #expect(control.drainRequested) + let (_, again) = try ServiceControl.requestDrain(configPath: path) + #expect(again == instance) + try control.markDrained() + try await ServiceControl.waitUntilDrained(paths: paths, instance: instance) +} + +@Test func drainControlDoesNotUseStaleRequestsOrCompletion() async throws { + let path = try drainConfigPath() + defer { try? FileManager.default.removeItem(atPath: (path as NSString).deletingLastPathComponent) } + var first: ServiceControl? = try ServiceControl(configPath: path) + let (paths, oldInstance) = try ServiceControl.requestDrain(configPath: path) + try first?.markDrained() + first = nil + for file in [paths.lock, paths.request, paths.complete] { + #expect(!FileManager.default.fileExists(atPath: file)) + // Simulate files left behind by an uncatchable termination. + try oldInstance.write(toFile: file, atomically: true, encoding: .utf8) + } + #expect(throws: ServiceControl.ControlError.self) { try ServiceControl.requestDrain(configPath: path) } + let second = try ServiceControl(configPath: path) + #expect(!second.drainRequested) + await #expect(throws: ServiceControl.ControlError.self) { + try await ServiceControl.waitUntilDrained(paths: paths, instance: oldInstance) + } + #expect(second.instance != oldInstance) + second.cleanup() + for file in [paths.lock, paths.request, paths.complete] { + #expect(!FileManager.default.fileExists(atPath: file)) + } + #expect(throws: ServiceControl.ControlError.self) { try second.markDrained() } + let third = try ServiceControl(configPath: path) + second.cleanup() // Repeated cleanup must not remove the replacement instance's files. + #expect(throws: ServiceControl.ControlError.self) { try ServiceControl(configPath: path) } + #expect(FileManager.default.fileExists(atPath: third.paths.lock)) +} + +@Test func drainControlResolvesConfigSymlinks() throws { + let path = try drainConfigPath() + defer { try? FileManager.default.removeItem(atPath: (path as NSString).deletingLastPathComponent) } + try "runners: []".write(toFile: path, atomically: true, encoding: .utf8) + let alias = path + ".alias" + try FileManager.default.createSymbolicLink(atPath: alias, withDestinationPath: path) + let control = try ServiceControl(configPath: path) + _ = try ServiceControl.requestDrain(configPath: alias) + #expect(control.drainRequested) +} + +@Test(arguments: [true, false]) +func drainGitHubOnlyRemovesIdleRunners(busy: Bool) async throws { + let session = MockSession() + session.responses["/orgs/org/installation"] = (Data(#"{"id":1}"#.utf8), 200) + session.responses["/app/installations/1/access_tokens"] = (Data(#"{"token":"access"}"#.utf8), 200) + session.responses["/orgs/org/actions/runners"] = (Data("{\"runners\":[{\"id\":7,\"name\":\"runner\",\"busy\":\(busy)}]}".utf8), 200) + session.responses["/orgs/org/actions/runners/7"] = (Data(), 204) + let service = GitHubService(auth: MockAuth(), session: session, organization: "org", repository: nil) + let result = try await service.removeRunner(named: "runner", onlyIfIdle: true) + #expect(result == (busy ? .busy : .removed)) + #expect(session.requests.contains { $0.httpMethod == "DELETE" } == !busy) +} + +@Test(arguments: [422, 403, 500]) +func drainGitHubHandlesAssignmentRaceAndAPIFailure(status: Int) async throws { + let session = MockSession() + session.responses["/repos/org/repo/installation"] = (Data(#"{"id":1}"#.utf8), 200) + session.responses["/app/installations/1/access_tokens"] = (Data(#"{"token":"access"}"#.utf8), 200) + session.responses["/repos/org/repo/actions/runners"] = (Data(#"{"runners":[{"id":7,"name":"runner","busy":false}]}"#.utf8), 200) + session.responses["/repos/org/repo/actions/runners/7"] = (Data("error".utf8), status) + let service = GitHubService(auth: MockAuth(), session: session, organization: "org", repository: "repo") + if status == 422 { + #expect(try await service.removeRunner(named: "runner", onlyIfIdle: true) == .busy) + } else { + await #expect(throws: GitHubServiceError.self) { try await service.removeRunner(named: "runner", onlyIfIdle: true) } + } +} + +@Test(arguments: [#"{"runners":[]}"#, #"{"runners":[{"id":7,"name":"runner"}]}"#]) +func drainGitHubDoesNotDeleteUnknownState(payload: String) async throws { + let session = MockSession() + session.responses["/orgs/org/installation"] = (Data(#"{"id":1}"#.utf8), 200) + session.responses["/app/installations/1/access_tokens"] = (Data(#"{"token":"access"}"#.utf8), 200) + session.responses["/orgs/org/actions/runners"] = (Data(payload.utf8), 200) + let service = GitHubService(auth: MockAuth(), session: session, organization: "org", repository: nil) + #expect(try await service.removeRunner(named: "runner", onlyIfIdle: true) != .removed) + #expect(!session.requests.contains { $0.httpMethod == "DELETE" }) +} + +// Real subprocesses exercise cancellation and job completion; VM/SSH transport is simulated. +private final class DrainTestProcessRunner: ProcessRunning, Sendable { + struct State { + var exists = false + var running = false + var clones = 0 + var vm: ProcessHandle? + var calls: [[String]] = [] + } + let state = Synchronization.Mutex(State()) + let system = SystemProcessRunner() + + func run(executable: String, arguments: [String], wait: Bool) async throws -> ProcessResult? { + state.withLock { $0.calls.append(arguments) } + var stdout = "" + if executable == "tart" { + switch arguments.first { + case "list": + stdout = state.withLock { $0.exists ? "[{\"Name\":\"drain-test\",\"Running\":\($0.running)}]" : "[]" } + case "clone": + state.withLock { $0.exists = true; $0.clones += 1 } + case "ip": stdout = "127.0.0.1" + case "stop": + let vm = state.withLock { $0.running = false; return $0.vm } + await vm?.terminate() + case "delete": state.withLock { $0.exists = false } + default: break + } + } + return ProcessResult(stdout: stdout, stderr: "", exitCode: 0) + } + + func start(executable: String, arguments: [String]) throws -> ProcessHandle { + if executable == "tart" { + let vm = try system.start(executable: "/bin/sleep", arguments: ["30"]) + state.withLock { $0.vm = vm; $0.running = true } + return vm + } + return try system.start(executable: "/bin/sh", arguments: ["-c", arguments.last!]) + } +} + +@Test func drainLetsActiveScriptFinishWithoutCreatingAnotherVM() async throws { + let path = try drainConfigPath() + let directory = (path as NSString).deletingLastPathComponent + defer { try? FileManager.default.removeItem(atPath: directory) } + let control = try ServiceControl(configPath: path) + let process = DrainTestProcessRunner() + let tart = makeTart(process) + let started = directory + "/started" + let release = directory + "/release" + let finished = directory + "/finished" + let quote = GitHubProvisioner.shellQuote + let script = "touch \(quote(started)); for i in {1..100}; do test -f \(quote(release)) && break; sleep 0.05; done; test -f \(quote(release)) && touch \(quote(finished))" + let config = Config.RunnerConfig(name: "drain-test", vm: Config.VM( + source: .init(type: .local, image: nil, name: "source"), hardware: nil, + mounts: [], cache: nil, run: .default, diskSizeGb: nil, ssh: .standard), + provisioner: .init(type: .script, script: .init(run: script), github: nil), + preRun: nil, postRun: "echo post-run", stopAfter: nil, healthCheck: nil) + let logger = Logger(label: "drain.test", minimumLevel: .critical) + let runner = Runner(tart: tart, github: nil, provisioner: GitHubProvisioner(), + runnerVersionResolver: GitHubRunnerVersionResolver(), runnerCache: RunnerCache(), config: config, + shutdownCoordinator: VMShutdownCoordinator(destroyer: VMDestroyer(tart: tart, logger: logger), logger: logger), + control: RunnerControl(), vmName: "drain-test", logLabel: "drain-test", logLevel: .critical, + logSink: nil, serviceControl: control) + let task = Task { try await runner.run() } + defer { task.cancel() } + for _ in 0..<200 { + if FileManager.default.fileExists(atPath: started) { break } + try await Task.sleep(for: .milliseconds(10)) + } + try #require(FileManager.default.fileExists(atPath: started)) + _ = try ServiceControl.requestDrain(configPath: path) + #expect(process.state.withLock { $0.running }) + #expect(!FileManager.default.fileExists(atPath: finished)) + try Data().write(to: URL(fileURLWithPath: release)) + try await task.value + #expect(FileManager.default.fileExists(atPath: finished)) + #expect(process.state.withLock { $0.clones == 1 && !$0.exists }) + #expect(process.state.withLock { $0.calls.contains { $0.last?.contains("echo post-run") == true } }) +}