ci: restore live FUSE and Slack coverage

.github/workflows/ci.yml

@@ -58,24 +58,16 @@ jobs:
             expect \
             patchelf \
             sqlite3
-      - name: Report Linux FUSE coverage
-        run: |
-          if [ -e /dev/fuse ]; then
-            echo "::notice::/dev/fuse is present. This workflow will run the real Linux FUSE mount smoke test."
-          else
-            echo "::error::/dev/fuse is not present. Real online-only FUSE mount smoke is not covered by this hosted runner."
-            exit 1
-          fi
       - name: Check Rust formatting
         run: cargo fmt --all -- --check
+      - name: Validate CI workflow contracts
+        run: tests/ci_workflow_contract.sh
       - name: Validate connector extension contracts
         run: |
           cargo test -p locality-connector --test manifest_contract --test conformance_testkit
           cargo test -p localityd --test connector_manifest
       - name: Run Rust tests
         run: cargo test --workspace --all-targets
-      - name: Run real Linux FUSE smoke test
-        run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/linux_fuse_smoke.sh
       - name: Run Amika snapshot shell test
         run: make test-init-amika-locality-snapshot
       - name: Install desktop dependencies
@@ -85,6 +77,26 @@ jobs:
         run: npm run build
         working-directory: apps/desktop
 
+  linux-fuse:
+    name: Linux FUSE smoke
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Install FUSE dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y fuse3 libfuse3-dev pkg-config sqlite3
+      - name: Verify hosted runner exposes FUSE
+        run: |
+          if [ ! -e /dev/fuse ]; then
+            echo "::error::/dev/fuse is not present. Real online-only FUSE mount smoke is not covered by this hosted runner."
+            exit 1
+          fi
+      - name: Run real Linux FUSE smoke test
+        run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh
+
   windows:
     name: Windows tests
     runs-on: windows-latest

.github/workflows/connector-live-e2e.yml

@@ -11,13 +11,18 @@ on:
       - "rust-toolchain.toml"
       - "crates/**"
       - "platform/linux/locality-fuse/**"
+      - "tests/linux-fuse-ci.Dockerfile"
+      - "tests/linux-fuse-ci-entrypoint.sh"
+      - "tests/run_linux_fuse_ci.sh"
       - "tests/live_connector_common.sh"
       - "tests/live_connector_common_selftest.sh"
       - "tests/live_google_docs_vfs_roundtrip.sh"
       - "tests/live_google_calendar_vfs_roundtrip.sh"
       - "tests/live_gmail_vfs_roundtrip.sh"
       - "tests/live_slack_vfs_read.sh"
       - "tests/live_linear_vfs_roundtrip.sh"
+      - "tests/resolve_linear_live_issue.py"
+      - "tests/resolve_linear_live_issue_selftest.sh"
   workflow_dispatch:
     inputs:
       force_oauth_refresh:
@@ -113,7 +118,7 @@ jobs:
           LOCALITY_LIVE_GOOGLE_DOCS_VFS: "1"
           LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }}
           LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-google-docs-live-credential.json', runner.temp) || '' }}
-        run: tests/live_google_docs_vfs_roundtrip.sh
+        run: tests/run_linux_fuse_ci.sh tests/live_google_docs_vfs_roundtrip.sh
       - name: Persist refreshed Google Docs credential
         if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }}
         shell: bash
@@ -189,7 +194,7 @@ jobs:
           LOCALITY_LIVE_GOOGLE_CALENDAR_VFS: "1"
           LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }}
           LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-google-calendar-live-credential.json', runner.temp) || '' }}
-        run: tests/live_google_calendar_vfs_roundtrip.sh
+        run: tests/run_linux_fuse_ci.sh tests/live_google_calendar_vfs_roundtrip.sh
       - name: Persist refreshed Google Calendar credential
         if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }}
         shell: bash
@@ -269,7 +274,7 @@ jobs:
           LOCALITY_LIVE_GMAIL_SEND: "1"
           LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }}
           LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-gmail-live-credential.json', runner.temp) || '' }}
-        run: tests/live_gmail_vfs_roundtrip.sh
+        run: tests/run_linux_fuse_ci.sh tests/live_gmail_vfs_roundtrip.sh
       - name: Persist refreshed Gmail credential
         if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }}
         shell: bash
@@ -319,20 +324,12 @@ jobs:
         env:
           LOCALITY_SLACK_LIVE_CREDENTIAL_JSON: ${{ secrets.LOCALITY_SLACK_LIVE_CREDENTIAL_JSON }}
           LOCALITY_SLACK_LIVE_CONVERSATION_ID: ${{ secrets.LOCALITY_SLACK_LIVE_CONVERSATION_ID }}
-          LOCALITY_LIVE_FORCE_OAUTH_REFRESH_INPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }}
-          LOCALITY_LIVE_PERSIST_ROTATED_OAUTH_SECRETS: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && '1' || '0' }}
           LOCALITY_SECRET_ROTATOR_TOKEN: ${{ secrets.LOCALITY_SECRET_ROTATOR_TOKEN }}
         run: |
           missing=()
           [ -n "$LOCALITY_SLACK_LIVE_CREDENTIAL_JSON" ] || missing+=("LOCALITY_SLACK_LIVE_CREDENTIAL_JSON")
           [ -n "$LOCALITY_SLACK_LIVE_CONVERSATION_ID" ] || missing+=("LOCALITY_SLACK_LIVE_CONVERSATION_ID")
-          if [ "$LOCALITY_LIVE_PERSIST_ROTATED_OAUTH_SECRETS" = "1" ]; then
-            if [ "$LOCALITY_LIVE_FORCE_OAUTH_REFRESH_INPUT" != "1" ]; then
-              echo "::error::persist_rotated_oauth_secrets requires force_oauth_refresh"
-              exit 1
-            fi
-            [ -n "$LOCALITY_SECRET_ROTATOR_TOKEN" ] || missing+=("LOCALITY_SECRET_ROTATOR_TOKEN")
-          fi
+          [ -n "$LOCALITY_SECRET_ROTATOR_TOKEN" ] || missing+=("LOCALITY_SECRET_ROTATOR_TOKEN")
           if [ "${#missing[@]}" -gt 0 ]; then
             echo "::error::Slack live e2e missing required environment secrets: ${missing[*]}"
             exit 1
@@ -341,17 +338,22 @@ jobs:
             echo "::error::/dev/fuse is unavailable; the live connector Linux FUSE e2e cannot run"
             exit 1
           fi
+          GH_TOKEN="$LOCALITY_SECRET_ROTATOR_TOKEN" \
+            gh api "repos/$GITHUB_REPOSITORY/environments/connector-live-e2e/secrets/public-key" \
+            >/dev/null
       - name: Run live Slack Linux FUSE e2e
         env:
           LOCALITY_SLACK_LIVE_CREDENTIAL_JSON: ${{ secrets.LOCALITY_SLACK_LIVE_CREDENTIAL_JSON }}
           LOCALITY_SLACK_LIVE_CONVERSATION_ID: ${{ secrets.LOCALITY_SLACK_LIVE_CONVERSATION_ID }}
           LOCALITY_SLACK_LIVE_TYPES: ${{ secrets.LOCALITY_SLACK_LIVE_TYPES }}
           LOCALITY_LIVE_SLACK_VFS: "1"
-          LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }}
-          LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-slack-live-credential.json', runner.temp) || '' }}
-        run: tests/live_slack_vfs_read.sh
+          # Slack refresh tokens rotate on every use, so every live run must
+          # publish the replacement credential before another run starts.
+          LOCALITY_LIVE_FORCE_OAUTH_REFRESH: "1"
+          LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ runner.temp }}/locality-slack-live-credential.json
+        run: tests/run_linux_fuse_ci.sh tests/live_slack_vfs_read.sh
       - name: Persist refreshed Slack credential
-        if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }}
+        if: ${{ always() }}
         shell: bash
         env:
           GH_TOKEN: ${{ secrets.LOCALITY_SECRET_ROTATOR_TOKEN }}
@@ -366,10 +368,19 @@ jobs:
             echo "::error::refreshed Slack credential was not exported; check the live e2e failure above"
             exit 1
           fi
-          gh secret set LOCALITY_SLACK_LIVE_CREDENTIAL_JSON \
-            --repo "$GITHUB_REPOSITORY" \
-            --env connector-live-e2e \
-            < "$ROTATED_CREDENTIAL_PATH"
+          for attempt in 1 2 3; do
+            if gh secret set LOCALITY_SLACK_LIVE_CREDENTIAL_JSON \
+              --repo "$GITHUB_REPOSITORY" \
+              --env connector-live-e2e \
+              < "$ROTATED_CREDENTIAL_PATH"; then
+              exit 0
+            fi
+            if [ "$attempt" -lt 3 ]; then
+              sleep "$attempt"
+            fi
+          done
+          echo "::error::failed to persist the rotated Slack credential after 3 attempts"
+          exit 1
 
   linear-live:
     name: Linear Linux FUSE live e2e
@@ -416,4 +427,4 @@ jobs:
           LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
           LOCALITY_LINEAR_LIVE_ISSUE_ID: ${{ secrets.LOCALITY_LINEAR_LIVE_ISSUE_ID }}
           LOCALITY_LIVE_LINEAR_VFS: "1"
-        run: tests/live_linear_vfs_roundtrip.sh
+        run: tests/run_linux_fuse_ci.sh tests/live_linear_vfs_roundtrip.sh

.github/workflows/e2e.yml

@@ -55,4 +55,4 @@ jobs:
             exit 1
           fi
       - name: Run real Linux FUSE smoke test
-        run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/linux_fuse_smoke.sh
+        run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh

.github/workflows/granola-live-e2e.yml

@@ -15,6 +15,9 @@ on:
       - "crates/localityd/**"
       - "crates/loc-cli/**"
       - "platform/linux/locality-fuse/**"
+      - "tests/linux-fuse-ci.Dockerfile"
+      - "tests/linux-fuse-ci-entrypoint.sh"
+      - "tests/run_linux_fuse_ci.sh"
       - "tests/live_granola_vfs_read.sh"
   schedule:
     - cron: "23 15 * * 2"
@@ -95,4 +98,4 @@ jobs:
             exit 1
           fi
       - name: Run live mounted-filesystem test
-        run: tests/live_granola_vfs_read.sh
+        run: tests/run_linux_fuse_ci.sh tests/live_granola_vfs_read.sh

.github/workflows/notion-live-e2e.yml

@@ -133,7 +133,7 @@ jobs:
           cargo build -p localityd
           cargo test -p locality-desktop live_mode_bidirectional_cloudstorage_markdown_e2e -- --ignored --exact --test-threads=1 --nocapture
       - name: Run Linux FUSE live Notion e2e from stored credential
-        run: env -u NOTION_TOKEN -u NOTION_AT tests/live_notion_vfs_push_pull.sh
+        run: tests/run_linux_fuse_ci.sh env -u NOTION_TOKEN -u NOTION_AT tests/live_notion_vfs_push_pull.sh
 
   windows-cloud-files-live:
     name: Windows Cloud Files live e2e

docs/e2e-behavior-coverage.md

@@ -110,6 +110,15 @@ script creates an isolated Locality state directory, runs through the real
 `loc`, `localityd`, and `locality-fuse` binaries, and keeps command/provider
 payloads out of test output.
 
+GitHub-hosted jobs run these unchanged scripts through
+`tests/run_linux_fuse_ci.sh`. New Ubuntu 22.04 and 24.04 hosted agents expose
+`/dev/fuse` but reject valid host-user mounts with `Operation not permitted`.
+The wrapper builds a pinned Rust/Linux image, passes the real host FUSE device
+into a privileged mount namespace, then drops back to the host runner UID before
+executing the script. This preserves the real unprivileged `fusermount3`,
+kernel directory, read, write, rename, and unmount paths instead of replacing
+them with mocks or allowing the tests to skip.
+
 For Google OAuth and Slack credentials, a local run can reuse a stored
 connection credential by reading its hex-encoded secret reference from
 `~/.loc/credentials`:
@@ -178,15 +187,23 @@ LOCALITY_LIVE_LINEAR_VFS=1 tests/live_linear_vfs_roundtrip.sh
 
 GitHub Actions runs `.github/workflows/connector-live-e2e.yml` for relevant
 changes on `main` and on manual dispatch. The workflow uses the
-`connector-live-e2e` environment and uploads no artifacts. Normal push and
-push runs validate refresh-capable OAuth credential JSON but never write back
-to GitHub Secrets. Manual dispatch can set `force_oauth_refresh=true` to
-exercise the broker refresh path. Manual dispatch can also set
+`connector-live-e2e` environment and uploads no artifacts. Google OAuth jobs
+validate refresh-capable credential JSON but do not write back to GitHub
+Secrets during normal push runs. Manual dispatch can set
+`force_oauth_refresh=true` to exercise their broker refresh paths, and can set
 `persist_rotated_oauth_secrets=true`; in that mode each OAuth job requires
 `LOCALITY_SECRET_ROTATOR_TOKEN`, exports its refreshed credential JSON after the
 first successful pull, and uses `gh secret set --env connector-live-e2e` to
 replace the corresponding environment secret.
 
+Slack is different because each successful Slack refresh invalidates the
+previous refresh token. Every Slack live job therefore requires
+`LOCALITY_SECRET_ROTATOR_TOKEN`, preflights its access to the environment
+Secrets public key, forces the existing live refresh assertion, and writes the
+rotated credential back even when a later live assertion fails. Workflow
+concurrency serializes runs so the next job cannot start from the superseded
+handle.
+
 ## Live Connector E2E Secrets
 
 The existing Granola workflow uses the `granola-live-e2e` environment:
@@ -210,7 +227,7 @@ The non-Notion connector workflow uses the `connector-live-e2e` environment:
 | `LOCALITY_SLACK_LIVE_TYPES` | no | Optional Slack mount type list for non-public conversations, for example `private_channel,im,mpim`. |
 | `LINEAR_API_KEY` | yes | Linear API key with access to the scratch issue. |
 | `LOCALITY_LINEAR_LIVE_ISSUE_ID` | yes | Stable scratch Linear issue id whose body can be edited and restored. |
-| `LOCALITY_SECRET_ROTATOR_TOKEN` | only for manual rotation | GitHub token used only when `workflow_dispatch` sets both `force_oauth_refresh=true` and `persist_rotated_oauth_secrets=true`; it must be able to update `connector-live-e2e` environment secrets. |
+| `LOCALITY_SECRET_ROTATOR_TOKEN` | yes for Slack; otherwise only for manual rotation | GitHub token able to update `connector-live-e2e` environment secrets. Slack uses it on every live run because Slack refresh tokens are single-use; Google jobs use it only when manual dispatch sets both `force_oauth_refresh=true` and `persist_rotated_oauth_secrets=true`. |
 
 ## Expected Behavior Coverage
 
@@ -290,7 +307,7 @@ Coverage labels:
 | `tests/live_google_calendar_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a stored Google Calendar credential, creates a calendar event from a local draft under the mounted filesystem, verifies the event projection after pull, and deletes the scratch event through the Calendar API. Covers the live draft-create path for Google Calendar. |
 | `tests/live_gmail_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a stored Gmail credential, creates an unsent Gmail UI draft from a mounted `draft/` Markdown file, verifies the projected Gmail draft maps to the created message, and deletes the draft through the Gmail API. Covers the live Gmail draft-create path; direct sends use the sibling `outbox/` folder and are tracked in E2E-042. |
 | `tests/live_slack_vfs_read.sh` | Live Linux FUSE product path | Seeds a stored Slack credential, mounts selected non-public Slack conversation types read-only, resolves the configured conversation by identity metadata, hydrates its `recent.md`, verifies status stays clean, and proves push is blocked before Slack writes. Covers the live Slack read-only projection and write guardrail. |
-| `tests/live_linear_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a Linear API key credential, mounts Linear through the real daemon and FUSE helper, finds the configured issue by frontmatter identity, appends a body marker, pushes and pulls it back, then restores only the original body under current generated frontmatter. Covers the live Linear issue edit/read-back/restore path. |
+| `tests/live_linear_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a Linear API key credential, mounts Linear through the real daemon and FUSE helper, resolves the configured issue through the local search index and verifies its projected frontmatter identity, appends a body marker, pushes and pulls it back, then restores only the original body under current generated frontmatter. Covers the live Linear issue edit/read-back/restore path without recursively hydrating unrelated workspace issues. |
 
 ## Live Notion Test Coverage Map
 

docs/linear-connector.md

@@ -110,6 +110,9 @@ ids as read-only before any `issueUpdate` mutation is attempted.
 mount/pull/diff/push flow, daemon, and Linux FUSE projection. It edits one
 existing scratch issue body through the mounted `page.md`, pushes the marker to
 Linear, pulls it back, and restores the original issue content before exiting.
+The test resolves the configured issue UUID through Locality's local search
+index and then verifies the projected `page.md` frontmatter identity. It does
+not recursively walk and hydrate unrelated Linear issues.
 
 Set the API key and scratch issue UUID before running it:
 

docs/linux-fuse.md

@@ -74,6 +74,15 @@ available. The fixture-backed smoke starts `localityd` with
 materialization, and local mutations exercise only the seeded daemon state
 instead of queueing connector-backed refresh work with the dummy token.
 
+GitHub Actions invokes the same smoke script through
+`tests/run_linux_fuse_ci.sh`. Current hosted agents expose `/dev/fuse` but deny
+host-user mounts, so the wrapper passes that device into a privileged container
+mount namespace and then runs the script as the numeric host runner user. The
+container keeps `/usr/bin/fusermount3` setuid and does not alter the smoke
+assertions, required-mode behavior, or filesystem operations being tested. Its
+Cargo target directory is isolated from the mounted workspace so cached
+host-built binaries cannot bypass the container build.
+
 ## Live Notion FUSE E2E
 
 The opt-in live FUSE script uses the real `loc`, `localityd`, and
@@ -95,11 +104,12 @@ then runs the FUSE mount, pull, edit, push, child create, child parent move out
 and back with Unix `mv`, child rename, and child delete/archive path with token
 environment variables removed from `loc` and `localityd`.
 
-The `notion-live-e2e` GitHub Actions workflow runs this script on
-`ubuntu-latest` when the live Notion environment is configured. That job seeds
-`~/.loc/credentials` from the `NOTION_TOKEN` secret before invoking the script
-with `NOTION_TOKEN` removed, so the CI path exercises the stored-credential
-product path rather than relying on direct token environment variables.
+The `notion-live-e2e` GitHub Actions workflow runs this script through the same
+privileged FUSE container on `ubuntu-latest` when the live Notion environment
+is configured. That job seeds `~/.loc/credentials` from the `NOTION_TOKEN`
+secret before invoking the script with `NOTION_TOKEN` removed, so the CI path
+exercises the stored-credential product path rather than relying on direct
+token environment variables.
 
 ### Existing Mount
 

docs/slack-connector.md

@@ -131,6 +131,12 @@ Use the full stored credential JSON. The live harness requires
 `access_token`, `oauth_broker_url`, `refresh_token_handle`, and numeric
 `expires_at` so it can exercise broker refresh when the token expires.
 
+The GitHub live job always forces that refresh assertion and persists the
+replacement `LOCALITY_SLACK_LIVE_CREDENTIAL_JSON` environment secret with
+`LOCALITY_SECRET_ROTATOR_TOKEN`. Slack refresh tokens are single-use, so a live
+job must not consume one without saving its replacement for the next serialized
+run.
+
 Set `LOCALITY_SLACK_LIVE_TYPES` when the target conversation is not covered by
 the default `private_channel,im,mpim` type set. Do not set `public_channel` for
 this live test.