diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3391a7bf..f5c0e0c2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,24 +58,16 @@ jobs: expect \ patchelf \ sqlite3 - - name: Report Linux FUSE coverage - run: | - if [ -e /dev/fuse ]; then - echo "::notice::/dev/fuse is present. This workflow will run the real Linux FUSE mount smoke test." - else - echo "::error::/dev/fuse is not present. Real online-only FUSE mount smoke is not covered by this hosted runner." - exit 1 - fi - name: Check Rust formatting run: cargo fmt --all -- --check + - name: Validate CI workflow contracts + run: tests/ci_workflow_contract.sh - name: Validate connector extension contracts run: | cargo test -p locality-connector --test manifest_contract --test conformance_testkit cargo test -p localityd --test connector_manifest - name: Run Rust tests run: cargo test --workspace --all-targets - - name: Run real Linux FUSE smoke test - run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/linux_fuse_smoke.sh - name: Run Amika snapshot shell test run: make test-init-amika-locality-snapshot - name: Install desktop dependencies @@ -85,6 +77,26 @@ jobs: run: npm run build working-directory: apps/desktop + linux-fuse: + name: Linux FUSE smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Install FUSE dependencies + run: | + sudo apt-get update + sudo apt-get install -y fuse3 libfuse3-dev pkg-config sqlite3 + - name: Verify hosted runner exposes FUSE + run: | + if [ ! -e /dev/fuse ]; then + echo "::error::/dev/fuse is not present. Real online-only FUSE mount smoke is not covered by this hosted runner." + exit 1 + fi + - name: Run real Linux FUSE smoke test + run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh + windows: name: Windows tests runs-on: windows-latest diff --git a/.github/workflows/connector-live-e2e.yml b/.github/workflows/connector-live-e2e.yml index 988bfaf0..6f29dac1 100644 --- a/.github/workflows/connector-live-e2e.yml +++ b/.github/workflows/connector-live-e2e.yml @@ -11,6 +11,9 @@ on: - "rust-toolchain.toml" - "crates/**" - "platform/linux/locality-fuse/**" + - "tests/linux-fuse-ci.Dockerfile" + - "tests/linux-fuse-ci-entrypoint.sh" + - "tests/run_linux_fuse_ci.sh" - "tests/live_connector_common.sh" - "tests/live_connector_common_selftest.sh" - "tests/live_google_docs_vfs_roundtrip.sh" @@ -18,6 +21,8 @@ on: - "tests/live_gmail_vfs_roundtrip.sh" - "tests/live_slack_vfs_read.sh" - "tests/live_linear_vfs_roundtrip.sh" + - "tests/resolve_linear_live_issue.py" + - "tests/resolve_linear_live_issue_selftest.sh" workflow_dispatch: inputs: force_oauth_refresh: @@ -113,7 +118,7 @@ jobs: LOCALITY_LIVE_GOOGLE_DOCS_VFS: "1" LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }} LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-google-docs-live-credential.json', runner.temp) || '' }} - run: tests/live_google_docs_vfs_roundtrip.sh + run: tests/run_linux_fuse_ci.sh tests/live_google_docs_vfs_roundtrip.sh - name: Persist refreshed Google Docs credential if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }} shell: bash @@ -189,7 +194,7 @@ jobs: LOCALITY_LIVE_GOOGLE_CALENDAR_VFS: "1" LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }} LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-google-calendar-live-credential.json', runner.temp) || '' }} - run: tests/live_google_calendar_vfs_roundtrip.sh + run: tests/run_linux_fuse_ci.sh tests/live_google_calendar_vfs_roundtrip.sh - name: Persist refreshed Google Calendar credential if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }} shell: bash @@ -269,7 +274,7 @@ jobs: LOCALITY_LIVE_GMAIL_SEND: "1" LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }} LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-gmail-live-credential.json', runner.temp) || '' }} - run: tests/live_gmail_vfs_roundtrip.sh + run: tests/run_linux_fuse_ci.sh tests/live_gmail_vfs_roundtrip.sh - name: Persist refreshed Gmail credential if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }} shell: bash @@ -319,20 +324,12 @@ jobs: env: LOCALITY_SLACK_LIVE_CREDENTIAL_JSON: ${{ secrets.LOCALITY_SLACK_LIVE_CREDENTIAL_JSON }} LOCALITY_SLACK_LIVE_CONVERSATION_ID: ${{ secrets.LOCALITY_SLACK_LIVE_CONVERSATION_ID }} - LOCALITY_LIVE_FORCE_OAUTH_REFRESH_INPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }} - LOCALITY_LIVE_PERSIST_ROTATED_OAUTH_SECRETS: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && '1' || '0' }} LOCALITY_SECRET_ROTATOR_TOKEN: ${{ secrets.LOCALITY_SECRET_ROTATOR_TOKEN }} run: | missing=() [ -n "$LOCALITY_SLACK_LIVE_CREDENTIAL_JSON" ] || missing+=("LOCALITY_SLACK_LIVE_CREDENTIAL_JSON") [ -n "$LOCALITY_SLACK_LIVE_CONVERSATION_ID" ] || missing+=("LOCALITY_SLACK_LIVE_CONVERSATION_ID") - if [ "$LOCALITY_LIVE_PERSIST_ROTATED_OAUTH_SECRETS" = "1" ]; then - if [ "$LOCALITY_LIVE_FORCE_OAUTH_REFRESH_INPUT" != "1" ]; then - echo "::error::persist_rotated_oauth_secrets requires force_oauth_refresh" - exit 1 - fi - [ -n "$LOCALITY_SECRET_ROTATOR_TOKEN" ] || missing+=("LOCALITY_SECRET_ROTATOR_TOKEN") - fi + [ -n "$LOCALITY_SECRET_ROTATOR_TOKEN" ] || missing+=("LOCALITY_SECRET_ROTATOR_TOKEN") if [ "${#missing[@]}" -gt 0 ]; then echo "::error::Slack live e2e missing required environment secrets: ${missing[*]}" exit 1 @@ -341,17 +338,22 @@ jobs: echo "::error::/dev/fuse is unavailable; the live connector Linux FUSE e2e cannot run" exit 1 fi + GH_TOKEN="$LOCALITY_SECRET_ROTATOR_TOKEN" \ + gh api "repos/$GITHUB_REPOSITORY/environments/connector-live-e2e/secrets/public-key" \ + >/dev/null - name: Run live Slack Linux FUSE e2e env: LOCALITY_SLACK_LIVE_CREDENTIAL_JSON: ${{ secrets.LOCALITY_SLACK_LIVE_CREDENTIAL_JSON }} LOCALITY_SLACK_LIVE_CONVERSATION_ID: ${{ secrets.LOCALITY_SLACK_LIVE_CONVERSATION_ID }} LOCALITY_SLACK_LIVE_TYPES: ${{ secrets.LOCALITY_SLACK_LIVE_TYPES }} LOCALITY_LIVE_SLACK_VFS: "1" - LOCALITY_LIVE_FORCE_OAUTH_REFRESH: ${{ github.event_name == 'workflow_dispatch' && inputs.force_oauth_refresh == true && '1' || '0' }} - LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true && format('{0}/locality-slack-live-credential.json', runner.temp) || '' }} - run: tests/live_slack_vfs_read.sh + # Slack refresh tokens rotate on every use, so every live run must + # publish the replacement credential before another run starts. + LOCALITY_LIVE_FORCE_OAUTH_REFRESH: "1" + LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ runner.temp }}/locality-slack-live-credential.json + run: tests/run_linux_fuse_ci.sh tests/live_slack_vfs_read.sh - name: Persist refreshed Slack credential - if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.persist_rotated_oauth_secrets == true }} + if: ${{ always() }} shell: bash env: GH_TOKEN: ${{ secrets.LOCALITY_SECRET_ROTATOR_TOKEN }} @@ -366,10 +368,19 @@ jobs: echo "::error::refreshed Slack credential was not exported; check the live e2e failure above" exit 1 fi - gh secret set LOCALITY_SLACK_LIVE_CREDENTIAL_JSON \ - --repo "$GITHUB_REPOSITORY" \ - --env connector-live-e2e \ - < "$ROTATED_CREDENTIAL_PATH" + for attempt in 1 2 3; do + if gh secret set LOCALITY_SLACK_LIVE_CREDENTIAL_JSON \ + --repo "$GITHUB_REPOSITORY" \ + --env connector-live-e2e \ + < "$ROTATED_CREDENTIAL_PATH"; then + exit 0 + fi + if [ "$attempt" -lt 3 ]; then + sleep "$attempt" + fi + done + echo "::error::failed to persist the rotated Slack credential after 3 attempts" + exit 1 linear-live: name: Linear Linux FUSE live e2e @@ -416,4 +427,4 @@ jobs: LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }} LOCALITY_LINEAR_LIVE_ISSUE_ID: ${{ secrets.LOCALITY_LINEAR_LIVE_ISSUE_ID }} LOCALITY_LIVE_LINEAR_VFS: "1" - run: tests/live_linear_vfs_roundtrip.sh + run: tests/run_linux_fuse_ci.sh tests/live_linear_vfs_roundtrip.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 55872744..b10950d5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -55,4 +55,4 @@ jobs: exit 1 fi - name: Run real Linux FUSE smoke test - run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/linux_fuse_smoke.sh + run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh diff --git a/.github/workflows/granola-live-e2e.yml b/.github/workflows/granola-live-e2e.yml index 36582738..d282cd94 100644 --- a/.github/workflows/granola-live-e2e.yml +++ b/.github/workflows/granola-live-e2e.yml @@ -15,6 +15,9 @@ on: - "crates/localityd/**" - "crates/loc-cli/**" - "platform/linux/locality-fuse/**" + - "tests/linux-fuse-ci.Dockerfile" + - "tests/linux-fuse-ci-entrypoint.sh" + - "tests/run_linux_fuse_ci.sh" - "tests/live_granola_vfs_read.sh" schedule: - cron: "23 15 * * 2" @@ -95,4 +98,4 @@ jobs: exit 1 fi - name: Run live mounted-filesystem test - run: tests/live_granola_vfs_read.sh + run: tests/run_linux_fuse_ci.sh tests/live_granola_vfs_read.sh diff --git a/.github/workflows/notion-live-e2e.yml b/.github/workflows/notion-live-e2e.yml index aec30514..0692c386 100644 --- a/.github/workflows/notion-live-e2e.yml +++ b/.github/workflows/notion-live-e2e.yml @@ -133,7 +133,7 @@ jobs: cargo build -p localityd cargo test -p locality-desktop live_mode_bidirectional_cloudstorage_markdown_e2e -- --ignored --exact --test-threads=1 --nocapture - name: Run Linux FUSE live Notion e2e from stored credential - run: env -u NOTION_TOKEN -u NOTION_AT tests/live_notion_vfs_push_pull.sh + run: tests/run_linux_fuse_ci.sh env -u NOTION_TOKEN -u NOTION_AT tests/live_notion_vfs_push_pull.sh windows-cloud-files-live: name: Windows Cloud Files live e2e diff --git a/docs/e2e-behavior-coverage.md b/docs/e2e-behavior-coverage.md index 5b658ba5..96887dc9 100644 --- a/docs/e2e-behavior-coverage.md +++ b/docs/e2e-behavior-coverage.md @@ -110,6 +110,15 @@ script creates an isolated Locality state directory, runs through the real `loc`, `localityd`, and `locality-fuse` binaries, and keeps command/provider payloads out of test output. +GitHub-hosted jobs run these unchanged scripts through +`tests/run_linux_fuse_ci.sh`. New Ubuntu 22.04 and 24.04 hosted agents expose +`/dev/fuse` but reject valid host-user mounts with `Operation not permitted`. +The wrapper builds a pinned Rust/Linux image, passes the real host FUSE device +into a privileged mount namespace, then drops back to the host runner UID before +executing the script. This preserves the real unprivileged `fusermount3`, +kernel directory, read, write, rename, and unmount paths instead of replacing +them with mocks or allowing the tests to skip. + For Google OAuth and Slack credentials, a local run can reuse a stored connection credential by reading its hex-encoded secret reference from `~/.loc/credentials`: @@ -178,15 +187,23 @@ LOCALITY_LIVE_LINEAR_VFS=1 tests/live_linear_vfs_roundtrip.sh GitHub Actions runs `.github/workflows/connector-live-e2e.yml` for relevant changes on `main` and on manual dispatch. The workflow uses the -`connector-live-e2e` environment and uploads no artifacts. Normal push and -push runs validate refresh-capable OAuth credential JSON but never write back -to GitHub Secrets. Manual dispatch can set `force_oauth_refresh=true` to -exercise the broker refresh path. Manual dispatch can also set +`connector-live-e2e` environment and uploads no artifacts. Google OAuth jobs +validate refresh-capable credential JSON but do not write back to GitHub +Secrets during normal push runs. Manual dispatch can set +`force_oauth_refresh=true` to exercise their broker refresh paths, and can set `persist_rotated_oauth_secrets=true`; in that mode each OAuth job requires `LOCALITY_SECRET_ROTATOR_TOKEN`, exports its refreshed credential JSON after the first successful pull, and uses `gh secret set --env connector-live-e2e` to replace the corresponding environment secret. +Slack is different because each successful Slack refresh invalidates the +previous refresh token. Every Slack live job therefore requires +`LOCALITY_SECRET_ROTATOR_TOKEN`, preflights its access to the environment +Secrets public key, forces the existing live refresh assertion, and writes the +rotated credential back even when a later live assertion fails. Workflow +concurrency serializes runs so the next job cannot start from the superseded +handle. + ## Live Connector E2E Secrets The existing Granola workflow uses the `granola-live-e2e` environment: @@ -210,7 +227,7 @@ The non-Notion connector workflow uses the `connector-live-e2e` environment: | `LOCALITY_SLACK_LIVE_TYPES` | no | Optional Slack mount type list for non-public conversations, for example `private_channel,im,mpim`. | | `LINEAR_API_KEY` | yes | Linear API key with access to the scratch issue. | | `LOCALITY_LINEAR_LIVE_ISSUE_ID` | yes | Stable scratch Linear issue id whose body can be edited and restored. | -| `LOCALITY_SECRET_ROTATOR_TOKEN` | only for manual rotation | GitHub token used only when `workflow_dispatch` sets both `force_oauth_refresh=true` and `persist_rotated_oauth_secrets=true`; it must be able to update `connector-live-e2e` environment secrets. | +| `LOCALITY_SECRET_ROTATOR_TOKEN` | yes for Slack; otherwise only for manual rotation | GitHub token able to update `connector-live-e2e` environment secrets. Slack uses it on every live run because Slack refresh tokens are single-use; Google jobs use it only when manual dispatch sets both `force_oauth_refresh=true` and `persist_rotated_oauth_secrets=true`. | ## Expected Behavior Coverage @@ -290,7 +307,7 @@ Coverage labels: | `tests/live_google_calendar_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a stored Google Calendar credential, creates a calendar event from a local draft under the mounted filesystem, verifies the event projection after pull, and deletes the scratch event through the Calendar API. Covers the live draft-create path for Google Calendar. | | `tests/live_gmail_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a stored Gmail credential, creates an unsent Gmail UI draft from a mounted `draft/` Markdown file, verifies the projected Gmail draft maps to the created message, and deletes the draft through the Gmail API. Covers the live Gmail draft-create path; direct sends use the sibling `outbox/` folder and are tracked in E2E-042. | | `tests/live_slack_vfs_read.sh` | Live Linux FUSE product path | Seeds a stored Slack credential, mounts selected non-public Slack conversation types read-only, resolves the configured conversation by identity metadata, hydrates its `recent.md`, verifies status stays clean, and proves push is blocked before Slack writes. Covers the live Slack read-only projection and write guardrail. | -| `tests/live_linear_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a Linear API key credential, mounts Linear through the real daemon and FUSE helper, finds the configured issue by frontmatter identity, appends a body marker, pushes and pulls it back, then restores only the original body under current generated frontmatter. Covers the live Linear issue edit/read-back/restore path. | +| `tests/live_linear_vfs_roundtrip.sh` | Live Linux FUSE product path | Seeds a Linear API key credential, mounts Linear through the real daemon and FUSE helper, resolves the configured issue through the local search index and verifies its projected frontmatter identity, appends a body marker, pushes and pulls it back, then restores only the original body under current generated frontmatter. Covers the live Linear issue edit/read-back/restore path without recursively hydrating unrelated workspace issues. | ## Live Notion Test Coverage Map diff --git a/docs/linear-connector.md b/docs/linear-connector.md index 06b8c716..ac818474 100644 --- a/docs/linear-connector.md +++ b/docs/linear-connector.md @@ -110,6 +110,9 @@ ids as read-only before any `issueUpdate` mutation is attempted. mount/pull/diff/push flow, daemon, and Linux FUSE projection. It edits one existing scratch issue body through the mounted `page.md`, pushes the marker to Linear, pulls it back, and restores the original issue content before exiting. +The test resolves the configured issue UUID through Locality's local search +index and then verifies the projected `page.md` frontmatter identity. It does +not recursively walk and hydrate unrelated Linear issues. Set the API key and scratch issue UUID before running it: diff --git a/docs/linux-fuse.md b/docs/linux-fuse.md index f2569323..da91bf93 100644 --- a/docs/linux-fuse.md +++ b/docs/linux-fuse.md @@ -74,6 +74,15 @@ available. The fixture-backed smoke starts `localityd` with materialization, and local mutations exercise only the seeded daemon state instead of queueing connector-backed refresh work with the dummy token. +GitHub Actions invokes the same smoke script through +`tests/run_linux_fuse_ci.sh`. Current hosted agents expose `/dev/fuse` but deny +host-user mounts, so the wrapper passes that device into a privileged container +mount namespace and then runs the script as the numeric host runner user. The +container keeps `/usr/bin/fusermount3` setuid and does not alter the smoke +assertions, required-mode behavior, or filesystem operations being tested. Its +Cargo target directory is isolated from the mounted workspace so cached +host-built binaries cannot bypass the container build. + ## Live Notion FUSE E2E The opt-in live FUSE script uses the real `loc`, `localityd`, and @@ -95,11 +104,12 @@ then runs the FUSE mount, pull, edit, push, child create, child parent move out and back with Unix `mv`, child rename, and child delete/archive path with token environment variables removed from `loc` and `localityd`. -The `notion-live-e2e` GitHub Actions workflow runs this script on -`ubuntu-latest` when the live Notion environment is configured. That job seeds -`~/.loc/credentials` from the `NOTION_TOKEN` secret before invoking the script -with `NOTION_TOKEN` removed, so the CI path exercises the stored-credential -product path rather than relying on direct token environment variables. +The `notion-live-e2e` GitHub Actions workflow runs this script through the same +privileged FUSE container on `ubuntu-latest` when the live Notion environment +is configured. That job seeds `~/.loc/credentials` from the `NOTION_TOKEN` +secret before invoking the script with `NOTION_TOKEN` removed, so the CI path +exercises the stored-credential product path rather than relying on direct +token environment variables. ### Existing Mount diff --git a/docs/slack-connector.md b/docs/slack-connector.md index 75562d44..8dca9d52 100644 --- a/docs/slack-connector.md +++ b/docs/slack-connector.md @@ -131,6 +131,12 @@ Use the full stored credential JSON. The live harness requires `access_token`, `oauth_broker_url`, `refresh_token_handle`, and numeric `expires_at` so it can exercise broker refresh when the token expires. +The GitHub live job always forces that refresh assertion and persists the +replacement `LOCALITY_SLACK_LIVE_CREDENTIAL_JSON` environment secret with +`LOCALITY_SECRET_ROTATOR_TOKEN`. Slack refresh tokens are single-use, so a live +job must not consume one without saving its replacement for the next serialized +run. + Set `LOCALITY_SLACK_LIVE_TYPES` when the target conversation is not covered by the default `private_channel,im,mpim` type set. Do not set `public_channel` for this live test. diff --git a/tests/ci_workflow_contract.sh b/tests/ci_workflow_contract.sh new file mode 100755 index 00000000..7f2ec4b8 --- /dev/null +++ b/tests/ci_workflow_contract.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +fail() { + echo "CI workflow contract failed: $*" >&2 + exit 1 +} + +job_block() { + local workflow="$1" + local job="$2" + + awk -v job="$job" ' + $0 == " " job ":" { + found = 1 + print + next + } + found && /^ [[:alnum:]_-]+:$/ { exit } + found { print } + ' "$ROOT/$workflow" +} + +assert_job_line() { + local workflow="$1" + local job="$2" + local expected="$3" + local block + + block="$(job_block "$workflow" "$job")" + [[ -n "$block" ]] || fail "$workflow is missing job $job" + grep -Fqx "$expected" <<<"$block" || + fail "$workflow job $job is missing exact line: $expected" +} + +assert_job_omits() { + local workflow="$1" + local job="$2" + local forbidden="$3" + local block + + block="$(job_block "$workflow" "$job")" + [[ -n "$block" ]] || fail "$workflow is missing job $job" + if grep -Fq "$forbidden" <<<"$block"; then + fail "$workflow job $job contains forbidden text: $forbidden" + fi +} + +# These jobs exercise real kernel FUSE mounts inside the privileged CI +# environment. Keep their product-path test commands exact so a runner +# workaround cannot silently reduce coverage. +assert_job_line ".github/workflows/ci.yml" "linux" " runs-on: ubuntu-latest" +assert_job_line ".github/workflows/ci.yml" "linux-fuse" " runs-on: ubuntu-latest" +assert_job_line ".github/workflows/ci.yml" "linux-fuse" \ + " run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh" + +assert_job_line ".github/workflows/e2e.yml" "linux-fuse" " runs-on: ubuntu-latest" +assert_job_line ".github/workflows/e2e.yml" "linux-fuse" \ + " run: LOCALITY_FUSE_SMOKE=1 LOCALITY_FUSE_SMOKE_REQUIRED=1 tests/run_linux_fuse_ci.sh tests/linux_fuse_smoke.sh" + +assert_job_line ".github/workflows/notion-live-e2e.yml" "linux-fuse-live" " runs-on: ubuntu-latest" +assert_job_line ".github/workflows/notion-live-e2e.yml" "linux-fuse-live" \ + " run: tests/run_linux_fuse_ci.sh env -u NOTION_TOKEN -u NOTION_AT tests/live_notion_vfs_push_pull.sh" + +assert_job_line ".github/workflows/granola-live-e2e.yml" "linux-fuse-live" " runs-on: ubuntu-latest" +assert_job_line ".github/workflows/granola-live-e2e.yml" "linux-fuse-live" \ + " run: tests/run_linux_fuse_ci.sh tests/live_granola_vfs_read.sh" + +connector_jobs=( + "google-docs-live" + "google-calendar-live" + "gmail-live" + "slack-live" + "linear-live" +) +connector_scripts=( + "tests/live_google_docs_vfs_roundtrip.sh" + "tests/live_google_calendar_vfs_roundtrip.sh" + "tests/live_gmail_vfs_roundtrip.sh" + "tests/live_slack_vfs_read.sh" + "tests/live_linear_vfs_roundtrip.sh" +) + +for index in "${!connector_jobs[@]}"; do + job="${connector_jobs[$index]}" + script="${connector_scripts[$index]}" + assert_job_line ".github/workflows/connector-live-e2e.yml" "$job" " runs-on: ubuntu-latest" + assert_job_line ".github/workflows/connector-live-e2e.yml" "$job" \ + " run: tests/run_linux_fuse_ci.sh $script" + assert_job_omits ".github/workflows/connector-live-e2e.yml" "$job" "continue-on-error" +done + +for workflow in \ + ".github/workflows/connector-live-e2e.yml" \ + ".github/workflows/granola-live-e2e.yml"; do + for dependency in \ + ' - "tests/linux-fuse-ci.Dockerfile"' \ + ' - "tests/linux-fuse-ci-entrypoint.sh"' \ + ' - "tests/run_linux_fuse_ci.sh"'; do + grep -Fqx "$dependency" "$ROOT/$workflow" || + fail "$workflow push paths must include $dependency" + done +done + +for dependency in \ + ' - "tests/resolve_linear_live_issue.py"' \ + ' - "tests/resolve_linear_live_issue_selftest.sh"'; do + grep -Fqx "$dependency" "$ROOT/.github/workflows/connector-live-e2e.yml" || + fail ".github/workflows/connector-live-e2e.yml push paths must include $dependency" +done + +# Slack refresh tokens are single-use. Every live run must force refresh, +# export the replacement, and persist it even if a later live assertion fails. +assert_job_line ".github/workflows/connector-live-e2e.yml" "slack-live" \ + ' LOCALITY_LIVE_FORCE_OAUTH_REFRESH: "1"' +assert_job_line ".github/workflows/connector-live-e2e.yml" "slack-live" \ + ' LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT: ${{ runner.temp }}/locality-slack-live-credential.json' +assert_job_line ".github/workflows/connector-live-e2e.yml" "slack-live" \ + ' if: ${{ always() }}' +assert_job_line ".github/workflows/connector-live-e2e.yml" "slack-live" \ + " gh api \"repos/\$GITHUB_REPOSITORY/environments/connector-live-e2e/secrets/public-key\" \\" + +grep -Fqx ' --privileged' "$ROOT/tests/run_linux_fuse_ci.sh" || + fail "Linux FUSE CI wrapper must keep Docker privileged mode enabled" +grep -Fqx ' --device /dev/fuse' "$ROOT/tests/run_linux_fuse_ci.sh" || + fail "Linux FUSE CI wrapper must pass the real FUSE device" +grep -Fqx ' && chmod 4755 /usr/bin/fusermount3 \' "$ROOT/tests/linux-fuse-ci.Dockerfile" || + fail "Linux FUSE CI image must preserve unprivileged fusermount semantics" +grep -Fqx 'ENV CARGO_TARGET_DIR=/tmp/locality-target' "$ROOT/tests/linux-fuse-ci.Dockerfile" || + fail "Linux FUSE CI image must not reuse host-built target artifacts" +grep -Fqx ' --env "LOCALITY_BIN=/tmp/locality-target/debug/loc"' "$ROOT/tests/run_linux_fuse_ci.sh" || + fail "Linux FUSE CI wrapper must resolve binaries from its isolated target directory" +grep -Fqx ' --volume "$target_root:/tmp/locality-target"' "$ROOT/tests/run_linux_fuse_ci.sh" || + fail "Linux FUSE CI wrapper must mount a disposable isolated target directory" + +echo "ok: CI workflow contracts are intact" diff --git a/tests/linux-fuse-ci-entrypoint.sh b/tests/linux-fuse-ci-entrypoint.sh new file mode 100755 index 00000000..cd900d73 --- /dev/null +++ b/tests/linux-fuse-ci-entrypoint.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +uid="${LOCALITY_CI_UID:?LOCALITY_CI_UID is required}" +gid="${LOCALITY_CI_GID:?LOCALITY_CI_GID is required}" + +if group_name="$(getent group "$gid" | cut -d: -f1)" && [[ -n "$group_name" ]]; then + : +else + group_name="locality-ci" + groupadd --gid "$gid" "$group_name" +fi + +if ! getent passwd "$uid" >/dev/null; then + useradd \ + --uid "$uid" \ + --gid "$gid" \ + --home-dir /tmp/locality-home \ + --no-create-home \ + --shell /bin/bash \ + locality-ci +fi + +exec setpriv \ + --reuid "$uid" \ + --regid "$gid" \ + --init-groups \ + "$@" diff --git a/tests/linux-fuse-ci.Dockerfile b/tests/linux-fuse-ci.Dockerfile new file mode 100644 index 00000000..1392b666 --- /dev/null +++ b/tests/linux-fuse-ci.Dockerfile @@ -0,0 +1,32 @@ +FROM rust:1.96.0-bookworm@sha256:5e2214abe154fe26e39f64488952e5c991eeed1d6d6da7cc8381ae83927f0cfc + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + fuse3 \ + git \ + jq \ + libfuse3-dev \ + pkg-config \ + procps \ + python3 \ + sqlite3 \ + util-linux \ + && chmod 4755 /usr/bin/fusermount3 \ + && install -d -m 1777 \ + /tmp/locality-cargo \ + /tmp/locality-home \ + /tmp/locality-target \ + && rm -rf /var/lib/apt/lists/* + +COPY linux-fuse-ci-entrypoint.sh /usr/local/bin/locality-fuse-ci-entrypoint +RUN chmod 0755 /usr/local/bin/locality-fuse-ci-entrypoint + +ENV CARGO_HOME=/tmp/locality-cargo +ENV CARGO_TARGET_DIR=/tmp/locality-target +ENV HOME=/tmp/locality-home +ENV RUSTUP_HOME=/usr/local/rustup +ENV RUSTUP_TOOLCHAIN=1.96.0 + +ENTRYPOINT ["/usr/local/bin/locality-fuse-ci-entrypoint"] diff --git a/tests/live_connector_common.sh b/tests/live_connector_common.sh index f25d7d71..dc84ee25 100755 --- a/tests/live_connector_common.sh +++ b/tests/live_connector_common.sh @@ -224,6 +224,7 @@ emit_live_debug_diagnostics() { status_report \ info_report \ doctor_report \ + issue_search_report \ drive_search_report \ calendar_search_report \ drafts_list_report \ diff --git a/tests/live_connector_common_selftest.sh b/tests/live_connector_common_selftest.sh index 3a415676..fee8bc6a 100755 --- a/tests/live_connector_common_selftest.sh +++ b/tests/live_connector_common_selftest.sh @@ -482,4 +482,5 @@ PATH="$fake_bin_dir:$PATH" wait_for_fuse "$locality_root" "$fake_fuse_pid" PATH="$old_path" +"$script_dir/resolve_linear_live_issue_selftest.sh" echo "live connector helper self-test passed" diff --git a/tests/live_linear_vfs_roundtrip.sh b/tests/live_linear_vfs_roundtrip.sh index 47a1975a..ba6239f7 100755 --- a/tests/live_linear_vfs_roundtrip.sh +++ b/tests/live_linear_vfs_roundtrip.sh @@ -48,6 +48,7 @@ fuse_log="$tmp_root/locality-fuse.log" command_log="$tmp_root/commands.err.log" mount_report="$tmp_root/mount.json" initial_pull_report="$tmp_root/initial-pull.json" +issue_search_report="$tmp_root/issue-search.json" diff_report="$tmp_root/diff.json" push_report="$tmp_root/push.json" pull_after_push_report="$tmp_root/pull-after-push.json" @@ -88,7 +89,23 @@ assert_changed_remote_id_matches_issue() { if [[ -z "$remote_id" ]]; then live_fail "$label did not include changed_remote_ids.0" fi - if [[ "$remote_id" != "$LOCALITY_LINEAR_LIVE_ISSUE_ID" ]]; then + if ! python3 - "$remote_id" "$LOCALITY_LINEAR_LIVE_ISSUE_ID" <<'PY' +import sys + +def canonical_issue_id(value): + value = value.strip() + compact = "".join( + character.lower() + for character in value + if character in "0123456789abcdefABCDEF" + ) + return compact if len(compact) == 32 else value + +raise SystemExit( + 0 if canonical_issue_id(sys.argv[1]) == canonical_issue_id(sys.argv[2]) else 1 +) +PY + then live_fail "$label changed_remote_ids.0 did not match the target Linear issue id" fi } @@ -164,90 +181,18 @@ find_issue_by_frontmatter() { local issue_id="$1" local match - if match="$(python3 - "$mount_root" "$issue_id" <<'PY' 2>>"$command_log" -import os -import pathlib -import re -import sys - -root = pathlib.Path(sys.argv[1]) -issue_id = sys.argv[2] + if ! LOCALITY_STATE_DIR="$state_root" "$loc_bin" search "$issue_id" \ + --connector linear \ + --limit 2 \ + --json >"$issue_search_report" 2>>"$command_log"; then + return 1 + fi -def frontmatter_lines(path): - try: - text = path.read_text(encoding="utf-8") - except OSError: - return None - lines = text.splitlines() - if not lines or lines[0].strip() != "---": - return None - frontmatter = [] - for line in lines[1:]: - if line.strip() == "---": - return frontmatter - frontmatter.append(line) - return None - -def scalar_value(line, key): - match = re.match(rf"^\s*{re.escape(key)}:\s*(.*?)\s*$", line) - if not match: - return None - value = match.group(1).strip() - if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): - value = value[1:-1] - return value - -def block_key(line): - match = re.match(r"^(\s*)([A-Za-z0-9_.-]+):\s*$", line) - if not match: - return None - return (len(match.group(1)), match.group(2)) - -matches = [] -for current_root, dirs, files in os.walk(root): - dirs.sort() - files.sort() - if "page.md" not in files: - continue - path = pathlib.Path(current_root) / "page.md" - frontmatter = frontmatter_lines(path) - if frontmatter is None: - continue - - has_linear_connector = False - has_issue_id = False - active_block = None - active_indent = -1 - for line in frontmatter: - stripped = line.strip() - if not stripped: - continue - indent = len(line) - len(line.lstrip(" ")) - if active_block is not None and indent <= active_indent: - active_block = None - active_indent = -1 - - block = block_key(line) - if block is not None: - active_indent, active_block = block - continue - - if active_block == "loc": - if scalar_value(line, "connector") == "linear": - has_linear_connector = True - if scalar_value(line, "id") == issue_id: - has_issue_id = True - - if has_linear_connector and has_issue_id: - matches.append(path) - -if len(matches) == 1: - print(matches[0]) - raise SystemExit(0) - -raise SystemExit(1) -PY - )"; then + if match="$(python3 "$script_dir/resolve_linear_live_issue.py" \ + "$issue_search_report" \ + "$mount_root" \ + "$mount_id" \ + "$issue_id" 2>>"$command_log")"; then [[ -n "$match" ]] || return 1 printf '%s\n' "$match" return 0 diff --git a/tests/resolve_linear_live_issue.py b/tests/resolve_linear_live_issue.py new file mode 100755 index 00000000..218dae47 --- /dev/null +++ b/tests/resolve_linear_live_issue.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import json +import os +import pathlib +import re +import sys + + +def fail(message): + raise SystemExit(message) + + +def frontmatter_lines(path): + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + fail(f"Linear issue page could not be read: {error}") + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + fail("Linear issue page.md did not start with frontmatter") + frontmatter = [] + for line in lines[1:]: + if line.strip() == "---": + return frontmatter + frontmatter.append(line) + fail("Linear issue page.md frontmatter was not terminated") + + +def scalar_value(line, key): + match = re.match(rf"^\s*{re.escape(key)}:\s*(.*?)\s*$", line) + if not match: + return None + value = match.group(1).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + value = value[1:-1] + return value + + +def block_key(line): + match = re.match(r"^(\s*)([A-Za-z0-9_.-]+):\s*$", line) + if not match: + return None + return (len(match.group(1)), match.group(2)) + + +def canonical_issue_id(value): + value = str(value).strip() + compact = "".join( + character.lower() + for character in value + if character in "0123456789abcdefABCDEF" + ) + return compact if len(compact) == 32 else value + + +def verify_frontmatter(path, issue_id): + has_linear_connector = False + has_issue_id = False + active_block = None + active_indent = -1 + for line in frontmatter_lines(path): + if not line.strip(): + continue + indent = len(line) - len(line.lstrip(" ")) + if active_block is not None and indent <= active_indent: + active_block = None + active_indent = -1 + block = block_key(line) + if block is not None: + active_indent, active_block = block + continue + if active_block == "loc": + has_linear_connector |= scalar_value(line, "connector") == "linear" + frontmatter_id = scalar_value(line, "id") + has_issue_id |= ( + frontmatter_id is not None + and canonical_issue_id(frontmatter_id) == canonical_issue_id(issue_id) + ) + if not has_linear_connector or not has_issue_id: + fail("Linear issue page.md frontmatter identity did not match the search result") + + +def main(): + if len(sys.argv) != 5: + fail( + "usage: resolve_linear_live_issue.py " + " " + ) + report_path = pathlib.Path(sys.argv[1]) + root = pathlib.Path(sys.argv[2]) + mount_id = sys.argv[3] + issue_id = sys.argv[4] + report = json.loads(report_path.read_text(encoding="utf-8")) + if report.get("ok") is not True: + fail("Linear issue search did not report ok=true") + matches = [ + result + for result in report.get("results", []) + if result.get("mount_id") == mount_id + and result.get("connector") == "linear" + and result.get("kind") == "page" + and canonical_issue_id(result.get("remote_id", "")) == canonical_issue_id(issue_id) + ] + if len(matches) != 1: + fail( + "Linear issue search expected one canonical remote-id match, " + f"got {len(matches)}" + ) + path = pathlib.Path(matches[0].get("absolute_path", "")) + root_normalized = os.path.abspath(root) + path_normalized = os.path.abspath(path) + if os.path.commonpath((root_normalized, path_normalized)) != root_normalized: + fail("Linear issue search returned a path outside the mount root") + if path.name != "page.md": + fail("Linear issue search did not return a page.md path") + verify_frontmatter(path, issue_id) + print(path) + + +if __name__ == "__main__": + main() diff --git a/tests/resolve_linear_live_issue_selftest.sh b/tests/resolve_linear_live_issue_selftest.sh new file mode 100755 index 00000000..bc1e5885 --- /dev/null +++ b/tests/resolve_linear_live_issue_selftest.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/loc-linear-lookup-selftest.XXXXXX")" +trap 'rm -rf "$tmp_root"' EXIT + +mount_root="$tmp_root/mount" +page_path="$mount_root/Teams/Engineering/Issues/Todo/ENG-1 Test/page.md" +report_path="$tmp_root/search.json" +canonical_issue_id="aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" +configured_issue_id="AAAAAAAABBBBCCCCDDDDEEEEEEEEEEEE" +mkdir -p "$(dirname "$page_path")" + +cat >"$page_path" <<'MARKDOWN' +--- +loc: + id: aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee + type: page + connector: linear +title: Test +--- +Body +MARKDOWN + +write_report() { + local absolute_path="$1" + local remote_id="${2:-$canonical_issue_id}" + python3 - "$report_path" "$absolute_path" "$remote_id" <<'PY' +import json +import pathlib +import sys + +pathlib.Path(sys.argv[1]).write_text( + json.dumps( + { + "ok": True, + "results": [ + { + "mount_id": "linear-live", + "connector": "linear", + "kind": "page", + "remote_id": sys.argv[3], + "absolute_path": sys.argv[2], + } + ], + } + ), + encoding="utf-8", +) +PY +} + +write_report "$page_path" +resolved="$(python3 "$script_dir/resolve_linear_live_issue.py" \ + "$report_path" "$mount_root" linear-live "$configured_issue_id")" +[[ "$resolved" == "$page_path" ]] + +write_report "$tmp_root/outside/page.md" +if python3 "$script_dir/resolve_linear_live_issue.py" \ + "$report_path" "$mount_root" linear-live "$configured_issue_id" >/dev/null 2>&1; then + echo "resolver accepted a path outside the mount root" >&2 + exit 1 +fi + +write_report "$page_path" ffffffff-bbbb-cccc-dddd-eeeeeeeeeeee +if python3 "$script_dir/resolve_linear_live_issue.py" \ + "$report_path" "$mount_root" linear-live "$configured_issue_id" >/dev/null 2>&1; then + echo "resolver accepted a non-matching search result" >&2 + exit 1 +fi + +write_report "$page_path" +sed -i.bak 's/connector: linear/connector: slack/' "$page_path" +rm -f "$page_path.bak" +if python3 "$script_dir/resolve_linear_live_issue.py" \ + "$report_path" "$mount_root" linear-live "$configured_issue_id" >/dev/null 2>&1; then + echo "resolver accepted mismatched projected frontmatter" >&2 + exit 1 +fi + +echo "Linear live issue resolver self-test passed" diff --git a/tests/run_linux_fuse_ci.sh b/tests/run_linux_fuse_ci.sh new file mode 100755 index 00000000..4d6ef14c --- /dev/null +++ b/tests/run_linux_fuse_ci.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "usage: tests/run_linux_fuse_ci.sh [args ...]" >&2 + exit 2 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +IMAGE="${LOCALITY_LINUX_FUSE_CI_IMAGE:-locality-linux-fuse-ci:rust-1.96.0}" +target_parent="${RUNNER_TEMP:-${TMPDIR:-/tmp}}" +target_root="$(mktemp -d "$target_parent/locality-fuse-target.XXXXXX")" +cleanup() { + rm -rf "$target_root" +} +trap cleanup EXIT + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required for the privileged Linux FUSE CI environment" >&2 + exit 1 +fi + +docker build \ + --file "$ROOT/tests/linux-fuse-ci.Dockerfile" \ + --tag "$IMAGE" \ + "$ROOT/tests" + +docker_args=( + run + --rm + --privileged + --device /dev/fuse + --security-opt apparmor=unconfined + --env "LOCALITY_CI_UID=$(id -u)" + --env "LOCALITY_CI_GID=$(id -g)" + --env "LOCALITY_BIN=/tmp/locality-target/debug/loc" + --env "LOCALITYD_BIN=/tmp/locality-target/debug/localityd" + --env "LOCALITY_FUSE_BIN=/tmp/locality-target/debug/locality-fuse" + --volume "$ROOT:/workspace" + --volume "$target_root:/tmp/locality-target" + --workdir /workspace +) + +if [[ -d "$HOME/.cargo/registry" ]]; then + docker_args+=(--volume "$HOME/.cargo/registry:/tmp/locality-cargo/registry") +fi +if [[ -d "$HOME/.cargo/git" ]]; then + docker_args+=(--volume "$HOME/.cargo/git:/tmp/locality-cargo/git") +fi +if [[ -d "$HOME/.loc/credentials" ]]; then + docker_args+=(--volume "$HOME/.loc/credentials:/tmp/locality-home/.loc/credentials") +fi +if [[ -n "${RUNNER_TEMP:-}" && -d "$RUNNER_TEMP" ]]; then + docker_args+=(--volume "$RUNNER_TEMP:$RUNNER_TEMP") +fi + +forwarded_env=( + GRANOLA_API_KEY + LINEAR_API_KEY + LOCALITY_FUSE_SMOKE + LOCALITY_FUSE_SMOKE_REQUIRED + LOCALITY_GMAIL_LIVE_CREDENTIAL_JSON + LOCALITY_GMAIL_LIVE_TO_EMAIL + LOCALITY_GOOGLE_CALENDAR_LIVE_CREDENTIAL_JSON + LOCALITY_GOOGLE_DOCS_LIVE_CREDENTIAL_JSON + LOCALITY_GOOGLE_DOCS_LIVE_WORKSPACE_FOLDER + LOCALITY_GRANOLA_LIVE_NOTE_ID + LOCALITY_LINEAR_LIVE_ISSUE_ID + LOCALITY_LIVE_FORCE_OAUTH_REFRESH + LOCALITY_LIVE_GMAIL_SEND + LOCALITY_LIVE_GMAIL_VFS + LOCALITY_LIVE_GOOGLE_CALENDAR_VFS + LOCALITY_LIVE_GOOGLE_DOCS_VFS + LOCALITY_LIVE_GRANOLA_VFS + LOCALITY_LIVE_LINEAR_VFS + LOCALITY_LIVE_NOTION_VFS_PUSH_PULL + LOCALITY_LIVE_ROTATED_CREDENTIAL_OUTPUT + LOCALITY_LIVE_SLACK_VFS + LOCALITY_NOTION_LIVE_PARENT_PAGE + LOCALITY_SLACK_LIVE_CONVERSATION_ID + LOCALITY_SLACK_LIVE_CREDENTIAL_JSON + LOCALITY_SLACK_LIVE_TYPES + NOTION_AT + NOTION_TOKEN +) + +for name in "${forwarded_env[@]}"; do + if [[ -n "${!name+x}" ]]; then + docker_args+=(--env "$name") + fi +done + +docker "${docker_args[@]}" "$IMAGE" "$@"