stmitt · Sep 29, 2026
diff --git a/README.md b/README.md
@@ -55,6 +55,19 @@ brew services start sand
 
 The service restarts sand if it exits, runs it as an interactive process so launchd doesn't throttle its CPU and I/O (or the VMs it starts), and writes output to `~/Library/Logs/sand.log`. Stop it with `brew services stop sand`.
 
+To let active jobs finish before stopping the service:
+
+```bash
+sand drain --config ~/sand.yml
+brew services stop sand
+```
+
+`sand drain` waits for runner cleanup, then reports completion. Idle GitHub runners are deregistered before stopping their VMs; busy ephemeral runners finish their current job. Script provisioners finish the current script and post-run hook. Sand stops creating replacement VMs and stays alive but idle so Homebrew does not restart it. Use `brew services restart sand` to resume work.
+
+Use the same configuration path as the running service. Sand keeps small `.sand-lock`, `.sand-drain`, and `.sand-drained` files beside the config, so its directory must be writable. These files are removed when sand exits normally or handles SIGINT/SIGTERM, including `brew services stop sand`. They remain while sand is drained and idle. After a crash or SIGKILL, leftover files are ignored by the next instance and removed when it exits. Only one sand process can use a configuration at a time. The command fails if no service is running or it stops before completion. Interrupting the drain command does not cancel the drain request.
+
+If GitHub cannot confirm that a runner is idle, sand leaves it running and logs the issue; draining can wait indefinitely for a job or API recovery. A job assigned concurrently with the drain request may still finish on that ephemeral runner. Normal Ctrl+C, SIGTERM, and `brew services stop sand` remain immediate stops, so wait for `sand drain` to complete first.
+
 ## Logs
 
 sand logs to macOS default logging system using `os_log`. To see the log

diff --git a/Sources/sand/Drain.swift b/Sources/sand/Drain.swift
@@ -0,0 +1,17 @@
+import ArgumentParser
+import Foundation
+
+@available(macOS 15.0, *)
+struct Drain: AsyncParsableCommand {
+    static let configuration = CommandConfiguration(abstract: "Wait for active jobs to finish and leave sand idle.")
+
+    @Option(name: .shortAndLong, help: "Configuration used by the running sand service.")
+    var config: String = Config.defaultPath
+
+    func run() async throws {
+        let (paths, instance) = try ServiceControl.requestDrain(configPath: config)
+        print("Drain requested. Waiting for active jobs and VM cleanup...")
+        try await ServiceControl.waitUntilDrained(paths: paths, instance: instance)
+        print("Drain complete. sand is idle; it is safe to stop or restart the service.")
+    }
+}
diff --git a/Sources/sand/GitHub/GitHubService.swift b/Sources/sand/GitHub/GitHubService.swift
@@ -28,6 +28,7 @@ struct GitHubService: Sendable {
         struct Runner: Decodable {
             let id: Int
             let name: String
+            let busy: Bool?
         }
 
         let runners: [Runner]
@@ -48,16 +49,29 @@ struct GitHubService: Sendable {
     }
 
     func deleteRunner(named name: String) async throws -> Bool {
+        try await removeRunner(named: name, onlyIfIdle: false) == .removed
+    }
+
+    enum RemovalResult: Equatable { case removed, missing, busy }
+
+    func removeRunner(named name: String, onlyIfIdle: Bool) async throws -> RemovalResult {
         let installationId = try await installationID()
         let accessToken = try await installationAccessToken(installationId: installationId)
         let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-._~"))
         let encodedName = name.addingPercentEncoding(withAllowedCharacters: allowed) ?? name
         let response: RunnersResponse = try await request(path: "\(runnersPath())?name=\(encodedName)", method: "GET", token: accessToken)
         guard let runner = response.runners.first(where: { $0.name == name }) else {
-            return false
+            return .missing
+        }
+        // Missing busy information is not evidence that it is safe to stop a runner.
+        if onlyIfIdle, runner.busy != false { return .busy }
+        do {
+            _ = try await send(path: "\(runnersPath())/\(runner.id)", method: "DELETE", token: accessToken)
+        } catch GitHubServiceError.httpError(let status, _) where onlyIfIdle && status == 422 {
+            // GitHub can assign a job between the lookup and deletion. Let it finish.
+            return .busy
         }
-        _ = try await send(path: "\(runnersPath())/\(runner.id)", method: "DELETE", token: accessToken)
-        return true
+        return .removed
     }
 
 

diff --git a/Sources/sand/Runner.swift b/Sources/sand/Runner.swift
@@ -6,6 +6,7 @@ struct Runner: Sendable {
     let provisioner: GitHubProvisioner
     let runnerVersionResolver: GitHubRunnerVersionResolver
     let runnerCache: RunnerCache
+    let serviceControl: ServiceControl?
     let config: Config.RunnerConfig
     let shutdownCoordinator: VMShutdownCoordinator
     let control: RunnerControl
@@ -36,13 +37,15 @@ struct Runner: Sendable {
         vmName: String,
         logLabel: String,
         logLevel: LogLevel,
-        logSink: LogFileSink?
+        logSink: LogFileSink?,
+        serviceControl: ServiceControl? = nil
     ) {
         self.tart = tart
         self.github = github
         self.provisioner = provisioner
         self.runnerVersionResolver = runnerVersionResolver
         self.runnerCache = runnerCache
+        self.serviceControl = serviceControl
         self.config = config
         self.shutdownCoordinator = shutdownCoordinator
         self.control = control
@@ -52,34 +55,30 @@ struct Runner: Sendable {
     }
 
     func run() async throws {
-        if let stopAfter = config.stopAfter {
-            guard stopAfter > 0 else {
-                return
-            }
-            for _ in 0..<stopAfter {
-                do {
-                    try await runOnce()
-                } catch {
-                    logger.error("runOnce failed (vm=\(vmName)): \(String(describing: error))")
-                    throw error
-                }
-            }
-            return
-        }
-        while true {
+        var completed = 0
+        while !drainRequested && (config.stopAfter.map { completed < $0 } ?? true) {
+            try Task.checkCancellation()
             do {
                 try await runOnce()
             } catch {
                 logger.error("runOnce failed (vm=\(vmName)): \(String(describing: error))")
+                if drainRequested {
+                    await shutdownCoordinator.cleanup(reason: "draining")
+                    return
+                }
                 throw error
             }
+            completed += 1
         }
     }
 
+    private var drainRequested: Bool { serviceControl?.drainRequested == true }
+
     private func runOnce() async throws {
         let stopAfterLabel = config.stopAfter.map(String.init) ?? "nil"
         logger.debug("runOnce start (vm=\(vmName), stopAfter=\(stopAfterLabel))")
         await applyRestartBackoffIfNeeded()
+        guard !drainRequested else { return }
         let name = vmName
         let vm = config.vm
         let provisionerConfig = config.provisioner
@@ -95,6 +94,7 @@ struct Runner: Sendable {
         } else {
             logger.info("local source \(source); skipping registry pull")
         }
+        guard !drainRequested else { return }
         do {
             if try await tart.isRunning(name: name) {
                 logger.info("VM \(name) already running, stopping before boot")
@@ -111,6 +111,10 @@ struct Runner: Sendable {
             throw error
         }
         await shutdownCoordinator.activate(name: name)
+        if drainRequested {
+            await shutdownCoordinator.cleanup(reason: "draining before boot")
+            return
+        }
         do {
             try await applyVMConfigIfNeeded(name: name, vm: vm)
         } catch {
@@ -158,6 +162,10 @@ struct Runner: Sendable {
             await shutdownCoordinator.cleanup(reason: "ssh not ready")
             return
         }
+        if drainRequested {
+            await shutdownCoordinator.cleanup(reason: "draining before pre-run")
+            return
+        }
         if let preRun = config.preRun {
             logger.info("run preRun")
             logScript(preRun)
@@ -177,6 +185,10 @@ struct Runner: Sendable {
                 throw error
             }
         }
+        if drainRequested {
+            await shutdownCoordinator.cleanup(reason: "draining before provisioning")
+            return
+        }
         let healthCheckState = HealthCheckState()
         logger.debug("healthCheck task preparing (vm=\(name))")
         let healthCheckTask = startHealthCheck(
@@ -241,7 +253,7 @@ struct Runner: Sendable {
                     runnerName: runnerName,
                     runnerToken: token
                 )
-                let outcome = await runProvisionerCommands(commands, ssh: ssh, healthCheckState: healthCheckState, secrets: [token])
+                let outcome = await runProvisionerCommands(commands, ssh: ssh, healthCheckState: healthCheckState, secrets: [token], githubRunnerName: runnerName)
                 switch outcome {
                 case .completed:
                     logger.warning("github provisioner completed; runner exited, restarting VM")
@@ -372,6 +384,7 @@ struct Runner: Sendable {
         var lastSSHError: String?
         logger.debug("waitForSSH start (vm=\(vmName), maxRetries=\(maxRetries.map(String.init) ?? "nil"))")
         while true {
+            if drainRequested { return false }
             if let maxRetries, attempt >= maxRetries {
                 let statusLabel = lastStatus.map(statusLabel) ?? "unknown"
                 let statusErrorLabel = lastStatusError ?? "none"
@@ -606,10 +619,13 @@ struct Runner: Sendable {
         _ commands: [String],
         ssh: SSHClient,
         healthCheckState: HealthCheckState,
-        secrets: [String] = []
+        secrets: [String] = [],
+        githubRunnerName: String? = nil
     ) async -> ProvisionerSequenceOutcome {
         for command in commands {
-            let outcome = await runProvisionerCommand(command, ssh: ssh, healthCheckState: healthCheckState, secrets: secrets)
+            if drainRequested { return .completed }
+            let outcome = await runProvisionerCommand(command, ssh: ssh, healthCheckState: healthCheckState, secrets: secrets,
+                                                      githubRunnerName: isRunnerCommand(command) ? githubRunnerName : nil)
             switch outcome {
             case .completed:
                 continue
@@ -626,7 +642,8 @@ struct Runner: Sendable {
         _ command: String,
         ssh: SSHClient,
         healthCheckState: HealthCheckState,
-        secrets: [String]
+        secrets: [String],
+        githubRunnerName: String?
     ) async -> ProvisionerOutcome {
         let displayCommand = Runner.redact(command, secrets: secrets)
         logScript(displayCommand)
@@ -639,7 +656,7 @@ struct Runner: Sendable {
                 let handle = try ssh.start(command: command)
                 await control.setProvisioningHandle(handle)
                 logger.debug("\(labeledCommand) started; awaiting completion or healthCheck failure")
-                let outcome = await awaitProvisionerCommand(handle: handle, healthCheckState: healthCheckState)
+                let outcome = await awaitProvisionerCommand(handle: handle, healthCheckState: healthCheckState, githubRunnerName: githubRunnerName)
                 switch outcome {
                 case let .completed(result):
                     let stdoutLabel = commandLabel.isEmpty ? "stdout" : "stdout (\(commandLabel))"
@@ -680,7 +697,8 @@ struct Runner: Sendable {
 
     private func awaitProvisionerCommand(
         handle: ProcessHandle,
-        healthCheckState: HealthCheckState
+        healthCheckState: HealthCheckState,
+        githubRunnerName: String?
     ) async -> ProvisionerOutcome {
         await withTaskGroup(of: ProvisionerOutcome?.self) { group in
             group.addTask {
@@ -701,6 +719,34 @@ struct Runner: Sendable {
                     return .failed(error)
                 }
             }
+            if let githubRunnerName, let github, let serviceControl {
+                group.addTask {
+                    while !Task.isCancelled {
+                        if serviceControl.drainRequested {
+                            do {
+                                switch try await github.removeRunner(named: githubRunnerName, onlyIfIdle: true) {
+                                case .removed:
+                                    logger.info("drain: removed idle runner \(githubRunnerName)")
+                                    await handle.terminate()
+                                    return nil
+                                case .busy:
+                                    logger.info("drain: waiting for \(githubRunnerName) to finish its job")
+                                    return nil
+                                case .missing:
+                                    // Absence from the API alone is not proof the guest has finished.
+                                    break
+                                }
+                            } catch {
+                                if Task.isCancelled { return nil }
+                                logger.warning("drain: cannot confirm runner is idle; leaving it running: \(error)")
+                            }
+                        }
+                        do { try await Task.sleep(for: .seconds(serviceControl.drainRequested ? 5 : 1)) }
+                        catch { return nil }
+                    }
+                    return nil
+                }
+            }
             while let outcome = await group.next() {
                 if let outcome {
                     logger.debug("provisioner outcome received: \(provisionerOutcomeLabel(outcome))")
@@ -766,6 +812,7 @@ struct Runner: Sendable {
     }
 
     private func retrySSHIfNeeded(error: Error, stage: String, attempt: inout Int) async -> Bool {
+        guard !drainRequested else { return false }
         guard shouldRetrySSH(error), attempt < sshRetryDelays.count else {
             return false
         }

diff --git a/Sources/sand/Sand.swift b/Sources/sand/Sand.swift
@@ -6,7 +6,7 @@ import Foundation
 @available(macOS 15.0, *)
 struct Sand: AsyncParsableCommand {
     static let configuration = CommandConfiguration(
-        subcommands: [Run.self, Destroy.self, Doctor.self, Validate.self]
+        subcommands: [Run.self, Drain.self, Destroy.self, Doctor.self, Validate.self]
     )
 }
 
@@ -29,7 +29,8 @@ struct Run: AsyncParsableCommand {
         if !missing.isEmpty {
             throw ValidationError("Missing required dependencies in PATH: \(missing.joined(separator: ", ")). Install them and re-run.")
         }
-        let config = try Config.load(path: config)
+        let configPath = config
+        let config = try Config.load(path: configPath)
         let validator = ConfigValidator()
         let issues = validator.validate(config)
         let errors = issues.filter { $0.severity == .error }
@@ -59,6 +60,8 @@ struct Run: AsyncParsableCommand {
             return
         }
 
+        let serviceControl = try ServiceControl(configPath: configPath)
+        defer { serviceControl.cleanup() }
         let provisioner = GitHubProvisioner()
         let runnerVersionResolver = GitHubRunnerVersionResolver()
         let runnerCache = RunnerCache()
@@ -89,7 +92,8 @@ struct Run: AsyncParsableCommand {
                 vmName: runnerName,
                 logLabel: logLabel,
                 logLevel: level,
-                logSink: logSink
+                logSink: logSink,
+                serviceControl: serviceControl
             )
             runners.append(runner)
         }
@@ -112,6 +116,7 @@ struct Run: AsyncParsableCommand {
                 }
             }
             group.wait()
+            serviceControl.cleanup()
         }
         defer {
             _ = signalHandler
@@ -124,6 +129,12 @@ struct Run: AsyncParsableCommand {
             }
             try await group.waitForAll()
         }
+        if serviceControl.drainRequested {
+            try serviceControl.markDrained()
+            logger.info("drain complete; sand is idle until stopped or restarted")
+            // Stay alive so launchd's KeepAlive does not create new runners.
+            while true { try await Task.sleep(for: .seconds(3600)) }
+        }
     }
 
     private func githubService(for provisioner: Config.Provisioner?) throws -> GitHubService? {