Linux sandbox finish: CI x86_64+aarch64, mutation rows, aarch64 traces, docs/sandbox.md

.github/workflows/ci.yml

@@ -30,14 +30,21 @@ env:
 jobs:
   checks:
     if: github.event_name != 'schedule'
-    # Compilation and portable tests need no Seatbelt. Worker tests, including
-    # their codesign and privacy-disclaim probes, must stay on a macOS host.
+    # The worker confines itself on macOS (Seatbelt) and on Linux (seccomp and
+    # Landlock), so every leg runs the worker packages. Their macOS-only probes
+    # (codesign, privacy disclaim) are compiled only on macOS. The Linux
+    # seccomp filter has a separate syscall table per architecture, so Linux
+    # runs natively on both x86_64 and aarch64. Analysis, script checks and
+    # catalogue validation run once, on the x86_64 Linux leg; the aarch64 leg
+    # replays no mutation rows, so it installs no mutation runner.
     strategy:
       fail-fast: false
       matrix:
         include:
           - platform: linux
             os: ubuntu-latest
+          - platform: linux-arm64
+            os: ubuntu-24.04-arm
           - platform: macos
             os: macos-latest
     runs-on: ${{ matrix.os }}
@@ -64,12 +71,13 @@ jobs:
             ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('Cargo.lock') }}-${{ steps.rust.outputs.version }}-checks-
             ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('Cargo.lock') }}-${{ steps.rust.outputs.version }}-
       - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        if: matrix.platform != 'linux-arm64'
         id: mutate-cache
         with:
           path: ~/.cargo/bin/ckdev-mutate
           key: ${{ runner.os }}-${{ runner.arch }}-ckdev-mutate-${{ hashFiles('Cargo.lock') }}-${{ steps.rust.outputs.version }}-${{ env.CK_MUTATE_REV }}
       - name: Install pinned mutation runner
-        if: steps.mutate-cache.outputs.cache-hit != 'true'
+        if: matrix.platform != 'linux-arm64' && steps.mutate-cache.outputs.cache-hit != 'true'
         run: cargo install --locked --git https://github.com/cortexkit/commons --rev "$CK_MUTATE_REV" cortexkit-mutate
       - run: cargo fmt --all --check
         if: matrix.platform == 'linux'
@@ -78,19 +86,25 @@ jobs:
       - run: cargo clippy --workspace --all-targets --features basal-module/rig-kill-hook -- -D warnings
         if: matrix.platform == 'linux'
       - name: Test portable crates
-        if: matrix.platform == 'linux'
+        if: runner.os == 'Linux'
         run: cargo test --locked -p basal-proto -p basal-core -p basal-host
       # The module feature propagates to basal-host, so portable tests need both
       # modes as well. Enable it on the selected host crate without selecting the
-      # worker-using module. Worker packages run both modes only on macOS.
+      # worker-using module. Worker packages run both modes on every leg.
       - name: Test portable crates with the rig kill hook
-        if: matrix.platform == 'linux'
+        if: runner.os == 'Linux'
         run: cargo test --locked -p basal-proto -p basal-core -p basal-host --features basal-host/rig-kill-hook
+      # A procfs mounted with hidepid hides the confined worker from its parent,
+      # and the Linux startup tests then accept a weaker self-reported status.
+      # GitHub's runners show every process, so require the exact parent-side
+      # /proc/<pid>/status readback here; the variable has no effect on macOS.
       - name: Test confined worker packages
-        if: matrix.platform == 'macos'
+        env:
+          BASAL_REQUIRE_VISIBLE_PROC: ${{ runner.os == 'Linux' && '1' || '' }}
         run: cargo test --locked -p basal-worker -p basal-module -p basal-testkit -p basal-rig
       - name: Test confined worker packages with the rig kill hook
-        if: matrix.platform == 'macos'
+        env:
+          BASAL_REQUIRE_VISIBLE_PROC: ${{ runner.os == 'Linux' && '1' || '' }}
         run: cargo test --locked -p basal-worker -p basal-module -p basal-testkit -p basal-rig --features basal-module/rig-kill-hook
       - name: Check shell scripts
         if: matrix.platform == 'linux'
@@ -130,6 +144,35 @@ jobs:
       # so no worker starts. Rows marked for the other OS are skipped, so each
       # host validates the rows it will replay.
       - run: ckdev-mutate check
+        if: matrix.platform != 'linux-arm64'
+
+  # Native aarch64 evidence for the Linux seccomp allowlist. The filter has a
+  # separate aarch64 syscall table, and only a native trace shows which calls
+  # glibc and the release worker actually make there. The script fails when no
+  # confined worker was traced or a panic/abort probe misbehaves; its reduced
+  # traces are uploaded for review against
+  # crates/basal-worker/sandbox/linux/syscall-admission.md.
+  aarch64-syscall-trace:
+    if: github.event_name != 'schedule'
+    runs-on: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Install stable Rust and strace
+        run: |
+          rustup toolchain install stable --profile minimal
+          rustup default stable
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends strace
+      - name: Trace the native release worker
+        run: |
+          rm -rf crates/basal-worker/sandbox/linux/traces/aarch64
+          env -u BASAL_WORKER_BIN bash crates/basal-worker/sandbox/linux/trace-native.sh
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: syscall-trace-aarch64
+          path: crates/basal-worker/sandbox/linux/traces/aarch64
+          if-no-files-found: error
 
   mutations-pr:
     if: github.event_name == 'pull_request'

README.md

@@ -11,7 +11,7 @@ basal runs as a module supervised by the subc daemon, with the binary `ck-basal`
 | Crate | What |
 |---|---|
 | `basal-proto` | The versioned IPC protocol between the parent and its worker processes. |
-| `basal-worker` | `ck-basal-worker`: the confined QuickJS engine. It applies its own Seatbelt sandbox before reading anything, so it runs on macOS only. |
+| `basal-worker` | `ck-basal-worker`: the confined QuickJS engine. It confines itself before reading anything: Seatbelt on macOS, seccomp and Landlock on Linux. See [docs/sandbox.md](docs/sandbox.md). |
 | `basal-core` | Runtime state: the SQLite store, admission, the run state machine, the journal and mailbox, manifests and authorization, the scheduler and the activation driver. |
 | `basal-host` | The `Host` boundary that flow calls go through (module ops, facts, model calls, sinks), the adapters to the fleet's modules, and a deterministic mock. |
 | `basal-module` | `ck-basal`: the supervised module, with its subc manifest, the worker pool, the flow ops, dry runs and consent cards. |
@@ -20,7 +20,7 @@ basal runs as a module supervised by the subc daemon, with the binary `ck-basal`
 
 ## Building and testing
 
-basal needs Rust 1.88 or newer (edition 2024) on macOS. Dependencies are published on crates.io or pinned to an immutable Git revision.
+basal needs Rust 1.88 or newer (edition 2024) on macOS or Linux (x86_64 or aarch64). Dependencies are published on crates.io or pinned to an immutable Git revision.
 
 ```sh
 cargo build --workspace
@@ -77,6 +77,7 @@ ckdev-mutate prove --id worker-closes-descriptors-proof \
 - [docs/script.md](docs/script.md): how a flow script runs, and two deliberate limits of the script runtime.
 - [docs/ops.md](docs/ops.md): the operations agents reach through prefrontal-core's relay, with their request and reply shapes.
 - [docs/deploy.md](docs/deploy.md): staging, signing and placing the two binaries, and rolling them back.
+- [docs/sandbox.md](docs/sandbox.md): how the worker is confined on each operating system, what it can still do, and the differences between systems.
 
 ## License
 

crates/basal-worker/tests/sandbox_page.rs

@@ -0,0 +1,104 @@
+//! The public sandbox page, docs/sandbox.md, must keep stating the limits of
+//! the worker's confinement on each operating system. Operators and flow
+//! authors read that page to decide what a flow could do if it subverted the
+//! engine, so an edit that drops one of these statements fails here.
+
+const PAGE: &str = include_str!("../../../docs/sandbox.md");
+
+/// The text under a `## ` heading, up to the next `## ` heading, with line
+/// wrapping collapsed so a phrase can be found across a line break.
+fn section(heading: &str) -> String {
+    let start = PAGE
+        .lines()
+        .position(|line| line == format!("## {heading}"))
+        .unwrap_or_else(|| panic!("docs/sandbox.md has no `## {heading}` section"));
+    let body: Vec<&str> = PAGE
+        .lines()
+        .skip(start + 1)
+        .take_while(|line| !line.starts_with("## "))
+        .collect();
+    normalize(&body.join("\n"))
+}
+
+fn normalize(text: &str) -> String {
+    text.split_whitespace().collect::<Vec<_>>().join(" ")
+}
+
+/// The paragraph (blank-line separated) of `text` that contains `anchor`.
+fn paragraph<'a>(text: &'a str, anchor: &str) -> &'a str {
+    text.split("\n\n")
+        .find(|p| normalize(p).contains(anchor))
+        .unwrap_or_else(|| panic!("no paragraph containing {anchor:?}"))
+}
+
+fn assert_mentions(text: &str, phrase: &str) {
+    assert!(
+        text.contains(phrase),
+        "docs/sandbox.md must say {phrase:?}; section text: {text}"
+    );
+}
+
+#[test]
+fn linux_section_names_seccomp_as_the_boundary() {
+    assert_mentions(&section("Linux"), "seccomp is the Linux boundary");
+}
+
+#[test]
+fn linux_section_lists_every_landlock_gap_in_one_paragraph() {
+    let raw = PAGE
+        .split("\n## ")
+        .find(|s| s.starts_with("Linux\n"))
+        .expect("Linux section");
+    let gap = normalize(paragraph(raw, "Landlock alone"));
+    for item in [
+        "stat",
+        "readlink",
+        "getxattr",
+        "UDP",
+        "netlink",
+        "path-based Unix sockets",
+    ] {
+        assert_mentions(&gap, item);
+    }
+}
+
+#[test]
+fn macos_section_says_thread_creation_is_not_blocked() {
+    assert_mentions(&section("macOS"), "Thread creation is not blocked on macOS");
+}
+
+#[test]
+fn macos_section_describes_only_what_the_worker_applies_today() {
+    let macos = section("macOS");
+    for layer in [
+        "worker.sb",
+        "deny-by-default",
+        "hardened runtime",
+        "disclaimed",
+    ] {
+        assert_mentions(&macos, layer);
+    }
+}
+
+#[test]
+fn pooled_worker_cpu_bound_is_the_wall_deadline_plus_the_idle_retire_limit() {
+    let differences = section("Differences between the systems");
+    assert_mentions(&differences, "wall-clock deadline");
+    assert_mentions(&differences, "idle-retire limit");
+}
+
+#[test]
+fn a_caught_memory_or_stack_failure_does_not_retire_the_worker() {
+    assert_mentions(
+        &section("Differences between the systems"),
+        "A caught memory or stack failure does not retire the worker",
+    );
+}
+
+#[test]
+fn windows_section_says_confinement_is_not_provided_yet() {
+    assert_mentions(
+        &section("Windows"),
+        "Windows confinement is not provided yet",
+    );
+}

docs/sandbox.md

@@ -0,0 +1,148 @@
+# The worker sandbox
+
+basal runs every flow script in `ck-basal-worker`, a separate process. The
+script is approved by the operator, but it is still untrusted input to a C
+JavaScript engine (QuickJS), so basal assumes the engine can be subverted and
+confines the worker so that a subverted worker can do little more than talk to
+its parent over stdin and stdout. Every effect a flow has on the world is a
+host call that the parent checks against the approved manifest and journals.
+The worker holds no durable state.
+
+The worker confines itself before it reads its first message from the parent.
+The parent then accepts a worker only if its first message, `Welcome`, reports
+the confinement this operating system requires. A worker that reports anything
+else is killed before it sees any flow code.
+
+This page describes, for each operating system, which layers apply, what a
+confined worker can still do, and how the systems differ.
+
+## Linux
+
+### Layers
+
+In order, before the first read:
+
+1. **Startup checks.** The worker refuses to start unless `/proc` is a genuine
+   procfs, it has exactly one thread, stdin is the read end and stdout and
+   stderr are write ends of anonymous pipes, the `READ_IMPLIES_EXEC`
+   personality is off, and no memory mapping is shared, writable and
+   executable at once, or an executable stack.
+2. **Descriptor close.** Every descriptor above stdio is closed, so nothing the
+   parent might have leaked (a store, a socket) reaches the engine.
+3. **`no_new_privs` and non-dumpable.** The worker can never gain privileges
+   through exec, and other processes, its parent included, cannot ptrace it or
+   read its memory through `/proc`.
+4. **Landlock.** An empty ruleset that restricts every filesystem, TCP and
+   scope right of the kernel's Landlock ABI. It is required by default; the
+   module runs workers without it only when the operator configures the
+   optional policy, and even then only when the kernel lacks Landlock.
+5. **Descriptors are closed again** after Landlock, so the ruleset's own
+   descriptor cannot survive.
+6. **seccomp.** A fixed allowlist of system calls, with argument checks,
+   installed on all threads. Any other call, or an admitted call with a refused
+   argument, kills the whole process with SIGSYS. The handler cannot catch it.
+
+**seccomp is the Linux boundary.** Landlock is a second layer, and no system
+call is admitted because Landlock would stop it. Landlock alone leaves gaps:
+it does not stop stat, readlink or getxattr on files outside any rule, UDP
+traffic, netlink sockets, or connecting to path-based Unix sockets. The
+seccomp allowlist kills all of those at their entry system call.
+
+The module accepts a Linux worker only if its `Welcome` reports seccomp
+installed and, under the default policy, a Landlock ruleset at the highest ABI
+both the kernel and the worker support. It counts workers killed by SIGSYS in
+`flow.health`, under `worker_confinement`, so a filter violation is visible to
+the operator rather than looking like an ordinary crash.
+
+### What a confined Linux worker can still do
+
+The allowlist and its review are in
+[`syscall-admission.md`](../crates/basal-worker/sandbox/linux/syscall-admission.md),
+with native traces of the release worker. In short, the worker can:
+
+- read stdin and write stdout and stderr, the descriptors it was started with;
+- allocate, resize and release private, non-executable memory;
+- read the realtime, monotonic and own-thread CPU clocks, and sleep;
+- read random bytes from the kernel;
+- wait on and wake private futexes, handle its own signals, and signal itself
+  (an abort must still end it with SIGABRT, where the parent can see it);
+- exit.
+
+It cannot open files, create sockets, start processes or threads, make memory
+executable, map shared memory, or signal any other process.
+
+## macOS
+
+### Layers
+
+- **Seatbelt.** The worker applies the deny-by-default profile
+  [`worker.sb`](../crates/basal-worker/sandbox/worker.sb) to itself with
+  `sandbox_init` before its first read. The profile denies everything (files,
+  network, process execution, service lookups) and allows only signals to
+  the worker itself.
+- **Descriptor close.** Every inherited descriptor above stdio is closed
+  before the profile applies.
+- **Hardened runtime.** Both binaries are signed ad hoc with the hardened
+  runtime, a plain identifier and no entitlements, never `get-task-allow`, so
+  a debugger cannot attach to the worker. See [deploy.md](deploy.md).
+- **Disclaimed launch.** The module launches each worker disclaimed from its
+  own privacy (TCC) grants, so flow code can never use a Files & Folders or
+  similar grant the module holds. The module checks at startup that it can
+  launch this way, and exits rather than ever launching a worker plainly.
+
+The module accepts a macOS worker only if its `Welcome` reports Seatbelt.
+
+### What a confined macOS worker can still do
+
+Reads and writes on descriptors open before the profile applied are not file
+opens, so stdio keeps working. The worker can allocate memory, read clocks,
+and signal itself. **Thread creation is not blocked on macOS**: Seatbelt has no
+rule for it. A script cannot start a thread, because the JavaScript runtime
+exposes none, but a subverted engine could.
+
+This page makes no claim about the Mach rights a macOS worker holds at launch,
+or about executable memory in the macOS worker. That evidence has not been
+published yet.
+
+## Differences between the systems
+
+| | Linux | macOS |
+| --- | --- | --- |
+| Boundary | seccomp allowlist, fatal SIGSYS | Seatbelt profile, denied calls fail |
+| Second layer | Landlock | none |
+| Threads | blocked by seccomp | not blocked |
+| Ptrace and memory reads | blocked by non-dumpable | blocked by the hardened runtime |
+| Privacy grants | not applicable | worker launched disclaimed |
+| Health counter | SIGSYS deaths | none |
+
+A denied operation on Linux kills the worker; on macOS the call fails and the
+engine keeps running.
+
+### CPU bound of a pooled worker
+
+Each activation has a JavaScript CPU budget, measured on the engine thread's
+CPU clock, and a wall-clock deadline (60 seconds by default); the parent kills
+a worker that passes the deadline. Workers are pooled: a worker is bound to one
+flow and reused for that flow's later activations until it is retired after 256
+activations or after sitting idle for 10 minutes.
+
+On Linux the engine thread is the worker's only thread, so the CPU budget
+covers all of the worker's CPU use. On macOS a subverted engine could start
+another thread whose CPU the engine thread's clock does not count, and that
+thread could keep running after its activation returns. The CPU such a pooled
+worker can use is therefore bounded only by time: the wall-clock deadline
+while an activation runs, plus the idle-retire limit once the worker sits idle.
+
+### Budget exhaustion and retirement
+
+A worker that reports an exhausted CPU, memory or stack budget is retired and
+replaced before the flow's next activation, because the engine's state after
+such a failure is not trusted. A caught memory or stack failure does not retire
+the worker: if the script catches the error and its activation completes
+normally, the worker is reused for that flow.
+
+## Windows
+
+Windows confinement is not provided yet. The worker has no Windows sandbox,
+and the module refuses any worker whose `Welcome` reports no confinement, so
+no flow runs there.