diff --git a/.github/workflows/snowflake-ci.yml b/.github/workflows/snowflake-ci.yml index 5a40cb7d7..af7169d4d 100644 --- a/.github/workflows/snowflake-ci.yml +++ b/.github/workflows/snowflake-ci.yml @@ -44,6 +44,7 @@ jobs: snowflake_paths=( ".github/workflows/snowflake-ci.yml" + ".github/workflows/snowflake-daily.yml" "Cargo.lock" "Cargo.toml" "crates/etl-api/Cargo.toml" @@ -102,12 +103,7 @@ jobs: snowflake_tests: name: Snowflake Tests needs: detect_snowflake - if: > - needs.detect_snowflake.outputs.required == 'true' && - ( - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - ) + if: needs.detect_snowflake.outputs.required == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 env: @@ -127,7 +123,6 @@ jobs: TESTS_DATABASE_PASSWORD: postgres TESTS_DATABASE_TLS_ENABLED: "true" TESTS_DATABASE_TLS_ROOT_CERT: ${{ github.workspace }}/target/postgres-tls/root.crt - TESTS_SNOWFLAKE_CONNECTION: ${{ secrets.TESTS_SNOWFLAKE_CONNECTION }} steps: - name: Checkout uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -140,16 +135,6 @@ jobs: with: key: test-snowflake - - name: Validate Snowflake Secrets - shell: bash - run: | - set -euo pipefail - - if [[ -z "${TESTS_SNOWFLAKE_CONNECTION}" ]]; then - echo "TESTS_SNOWFLAKE_CONNECTION secret is required for Snowflake CI." >&2 - exit 1 - fi - - name: Start Postgres run: | POSTGRES_IMAGE=postgres:18 \ @@ -161,8 +146,8 @@ jobs: with: tool: cargo-nextest - - name: Run Snowflake Tests - run: cargo x test-snowflake --credentials required + - name: Run Snowflake Tests Without Credentials + run: cargo x test-snowflake --credentials skip snowflake_gate: name: Snowflake Gate diff --git a/.github/workflows/snowflake-daily.yml b/.github/workflows/snowflake-daily.yml new file mode 100644 index 000000000..dade8499a --- /dev/null +++ b/.github/workflows/snowflake-daily.yml @@ -0,0 +1,82 @@ +name: Snowflake Daily Tests + +on: + schedule: + # Run paid integration tests once a day, away from the top of the hour. + - cron: "17 4 * * *" + +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + snowflake_tests: + name: Snowflake Credentialed Tests + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 60 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + POSTGRES_PORT: 5430 + POSTGRES_REPLICA_PORT: 6430 + POSTGRES_HOST: localhost + TESTS_DATABASE_HOST: localhost + TESTS_DATABASE_PORT: 5430 + TESTS_DATABASE_REPLICA_HOST: localhost + TESTS_DATABASE_REPLICA_PORT: 6430 + TESTS_DATABASE_START_PORT: 5430 + NUM_LOCAL_DATABASES: 1 + TESTS_DATABASE_USERNAME: postgres + TESTS_DATABASE_PASSWORD: postgres + TESTS_DATABASE_TLS_ENABLED: "true" + TESTS_DATABASE_TLS_ROOT_CERT: ${{ github.workspace }}/target/postgres-tls/root.crt + TESTS_SNOWFLAKE_CONNECTION: ${{ secrets.TESTS_SNOWFLAKE_CONNECTION }} + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Rust + uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 + + - name: Cache Cargo + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + key: test-snowflake + + - name: Validate Snowflake Secrets + shell: bash + run: | + set -euo pipefail + + if [[ -z "${TESTS_SNOWFLAKE_CONNECTION}" ]]; then + echo "TESTS_SNOWFLAKE_CONNECTION secret is required for Snowflake daily tests." >&2 + exit 1 + fi + + - name: Start Postgres + run: | + POSTGRES_IMAGE=postgres:18 \ + POSTGRES_VERSION=18 \ + cargo xtask postgres start --shards 1 --source-only + + - name: Install cargo-nextest + uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # v2.81.10 + with: + tool: cargo-nextest + + - name: Run Snowflake Tests + run: cargo x test-snowflake --credentials required + + - name: Summarize Snowflake Test Result + if: always() + env: + JOB_STATUS: ${{ job.status }} + run: | + echo "Snowflake daily tests: ${JOB_STATUS}." >> "$GITHUB_STEP_SUMMARY" diff --git a/crates/etl-destinations/src/snowflake/README.md b/crates/etl-destinations/src/snowflake/README.md index fad4b99f3..cccb6b301 100644 --- a/crates/etl-destinations/src/snowflake/README.md +++ b/crates/etl-destinations/src/snowflake/README.md @@ -17,6 +17,10 @@ cargo x test-snowflake This requires local Postgres to already be running. Run `cargo x init` first if the local development stack is not up. The command first runs the non-credentialed Snowflake destination preset, then runs the credentialed integration tiers when `TESTS_SNOWFLAKE_CONNECTION` is set. Use `--credentials skip` to run only the non-credentialed tier, or `--credentials required` to fail when credentials are missing. +GitHub Actions runs the credential-free tier for relevant pull request and `main` changes, retaining the `Snowflake Gate` check. Routine CI does not receive Snowflake credentials. The separate `Snowflake Daily Tests` workflow runs the credentialed API validator and destination integration tiers once daily at 04:17 UTC on the default branch, using the `TESTS_SNOWFLAKE_CONNECTION` repository secret. Scheduled runs begin after the workflow is merged and can be delayed by GitHub. + +Failures appear in the repository's Actions tab under `Snowflake Daily Tests`. + To run a specific destination test directly: ```bash @@ -60,7 +64,7 @@ on the command line. GitHub Actions uses the same one-var contract: -- `TESTS_SNOWFLAKE_CONNECTION` for `.github/workflows/snowflake-ci.yml`. +- `TESTS_SNOWFLAKE_CONNECTION` for `.github/workflows/snowflake-daily.yml`. - `BENCH_SNOWFLAKE_CONNECTION` for manual Snowflake benchmark workflow runs. Both repository secrets use the same JSON shape shown above. The workflows pass the JSON only