Ingest #303
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Ingest | |
| on: | |
| schedule: | |
| - cron: "0 */3 * * *" | |
| workflow_dispatch: | |
| concurrency: | |
| group: ingest | |
| cancel-in-progress: false | |
| jobs: | |
| ingest: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - uses: pnpm/action-setup@v6 | |
| with: | |
| version: 9.15.0 | |
| - uses: actions/setup-node@v7 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| - run: pnpm install --frozen-lockfile | |
| - name: Install Playwright Chromium (Citadel CDN bypass) | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Materialize Tesla state dump (optional secret) | |
| env: | |
| TESLA_STATE_JSON: ${{ secrets.TESLA_STATE_JSON }} | |
| run: | | |
| if [ -n "$TESLA_STATE_JSON" ]; then | |
| mkdir -p dumps | |
| printf '%s' "$TESLA_STATE_JSON" > dumps/tesla-state.json | |
| echo "OPENINTERN_TESLA_STATE_PATH=$GITHUB_WORKSPACE/dumps/tesla-state.json" >> "$GITHUB_ENV" | |
| fi | |
| - name: Require DATABASE_URL secret | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: | | |
| if [ -z "$DATABASE_URL" ]; then | |
| echo "::error::DATABASE_URL secret is not set. Add it under Settings → Secrets and variables → Actions (Neon connection string)." | |
| exit 1 | |
| fi | |
| - name: Run migrations | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| run: pnpm db:migrate | |
| - name: Run ingest | |
| id: ingest | |
| env: | |
| DATABASE_URL: ${{ secrets.DATABASE_URL }} | |
| OPENINTERN_BROWSER: "1" | |
| run: pnpm ingest | |
| - name: Notify on failure | |
| if: failure() | |
| uses: actions/github-script@v9 | |
| with: | |
| script: | | |
| const title = `Ingest failed (${new Date().toISOString().slice(0, 10)})`; | |
| const body = [ | |
| `Hourly ingest workflow failed on \`${context.sha.slice(0, 7)}\`.`, | |
| "", | |
| `Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, | |
| "", | |
| "Common causes: missing `DATABASE_URL` secret, Neon unreachable, or every company fetch failed.", | |
| "Check `/health` for per-company errors after a successful run.", | |
| ].join("\n"); | |
| const issues = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: "open", | |
| labels: "ingest-failure", | |
| per_page: 5, | |
| }); | |
| const existing = issues.data.find((i) => i.title.startsWith("Ingest failed")); | |
| if (existing) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.number, | |
| body, | |
| }); | |
| } else { | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title, | |
| body, | |
| labels: ["ingest-failure"], | |
| }); | |
| } |