Merge branch 'main' into main

Author: max-ostapenko

.github/linters/.trivyignore

@@ -0,0 +1,2 @@
+# Ignore the dataplexAdmin role issue
+AVD-GCP-0007

eslint.config.mjs .github/linters/eslint.config.mjseslint.config.mjs renamed to .github/linters/eslint.config.mjs

@@ -0,0 +1 @@
+ignorefile: ".github/linters/.trivyignore"

.github/linters/trivy.yaml

@@ -0,0 +1,4 @@
+rules:
+  unpinned-uses:
+    ignore:
+      - ci.yaml

.github/linters/zizmor.yaml

@@ -24,13 +24,13 @@ jobs:
         uses: actions/checkout@v5
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Lint Code Base
-        uses: super-linter/super-linter/slim@v8.0.0
+        uses: super-linter/super-linter/slim@v8.1.0
         env:
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          LINTER_RULES_PATH: .
           VALIDATE_JSCPD: false
           VALIDATE_JAVASCRIPT_PRETTIER: false
           VALIDATE_MARKDOWN_PRETTIER: false

.github/workflows/ci.yaml

@@ -7,6 +7,16 @@ publish('parsed_css', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page'],
     requirePartitionFilter: true
   },
+  columns: {
+    date: 'YYYY-MM-DD format of the HTTP Archive monthly crawl',
+    client: 'Test environment: desktop or mobile',
+    page: 'The URL of the page being tested',
+    is_root_page: 'Whether the page is the root of the origin.',
+    root_page: 'The URL of the root page being tested',
+    rank: 'Site popularity rank, from CrUX',
+    url: 'The URL of the request',
+    css: 'The parsed CSS, in JSON format'
+  },
   tags: ['crawl_complete']
 }).preOps(ctx => `
 SET @@RESERVATION='${constants.reservation_id}';
@@ -16,9 +26,7 @@ WHERE date = '${constants.currentMonth}'
   AND client = 'desktop';
 `).query(ctx => `
 SELECT
-  * EXCEPT(css),
-  NULL AS css_backup,
-  SAFE.PARSE_JSON(css, wide_number_mode=>'round') AS css
+  *
 FROM ${ctx.ref('crawl_staging', 'parsed_css')}
 WHERE date = '${constants.currentMonth}'
   AND client = 'desktop'
@@ -30,9 +38,7 @@ WHERE date = '${constants.currentMonth}'
 
 INSERT INTO ${ctx.self()}
 SELECT
-  * EXCEPT(css),
-  NULL AS css_backup,
-  SAFE.PARSE_JSON(css, wide_number_mode=>'round') AS css
+  *
 FROM ${ctx.ref('crawl_staging', 'parsed_css')}
 WHERE date = '${constants.currentMonth}'
   AND client = 'mobile'

definitions/output/crawl/parsed_css.js

@@ -4,6 +4,9 @@ FROM node:22-slim
 # Set the working directory
 WORKDIR /app
 
+# Create a non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
 # Copy package files first for better layer caching
 COPY package*.json ./
 
@@ -15,4 +18,13 @@ ENV EXPORT_CONFIG=""
 # Copy source code
 COPY . .
 
+# Change ownership of the app directory to the non-root user
+RUN chown -R appuser:appuser /app
+
+# Switch to non-root user
+USER appuser
+
+# No healthcheck needed for one-time job containers
+HEALTHCHECK NONE
+
 CMD ["node", "index.js"]

infra/bigquery-export/Dockerfile

@@ -3,6 +3,9 @@ FROM node:22-slim
 # Set the working directory
 WORKDIR /app
 
+# Create a non-root user
+RUN groupadd -r appuser && useradd -r -g appuser appuser
+
 # Copy package files first for better layer caching
 COPY package*.json ./
 
@@ -12,11 +15,21 @@ RUN npm ci --only=production --quiet --no-fund --no-audit && npm cache clean --f
 # Copy source code
 COPY . .
 
+# Change ownership of the app directory to the non-root user
+RUN chown -R appuser:appuser /app
+
+# Switch to non-root user
+USER appuser
+
 # Set default port (Cloud Run will override this)
 ENV PORT=8080
 
 # Expose port for Cloud Run
 EXPOSE 8080
 
+# Add healthcheck
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+  CMD node -e "require('http').get('http://localhost:$PORT/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) }).on('error', () => { process.exit(1) })" || exit 1
+
 # Start the function
 CMD ["npm", "start"]

infra/dataform-service/Dockerfile

@@ -223,14 +223,20 @@ async function mainHandler (req, res) {
 
   console.info(`Received request for path: ${path}`)
 
-  if (path === '/trigger' || path.startsWith('/trigger/')) {
+  if (path === '/health') {
+    // Health check endpoint
+    res.status(200).json({
+      status: 'healthy',
+      timestamp: new Date().toISOString()
+    })
+  } else if (path === '/trigger' || path.startsWith('/trigger/')) {
     await handleTrigger(req, res)
   } else if (path === '/') {
     await handleExport(req, res)
   } else {
     res.status(404).json({
       error: 'Not Found',
-      message: 'Available endpoints: /, /export'
+      message: 'Available endpoints: /, /trigger, /health'
     })
   }
 }

infra/dataform-service/index.js

@@ -2,8 +2,8 @@
   "name": "crawl-data",
   "author": "@max-ostapenko",
   "scripts": {
-    "format": "npx eslint --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
-    "lint": "npx eslint .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile",
+    "format": "npx eslint -c .github/linters/eslint.config.mjs --fix .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint . --fix; terraform -chdir=infra/tf fmt -recursive",
+    "lint": "npx eslint -c .github/linters/eslint.config.mjs .; npx markdownlint --ignore-path .gitignore --config package.json --configPointer /markdownlint .; dataform compile",
     "superlint": "docker run --platform linux/amd64 -e DEFAULT_BRANCH=main -e VALIDATE_GIT_COMMITLINT=false -e VALIDATE_TERRAFORM_TERRASCAN=false -e VALIDATE_TERRAFORM_TFLINT=false -e FIX_JSON_PRETTIER=true -e IGNORE_GITIGNORED_FILES=true -e VALIDATE_ALL_CODEBASE=true -e VALIDATE_JSCPD=false -e RUN_LOCAL=true -v ./:/tmp/lint ghcr.io/super-linter/super-linter:slim-latest"
   },
   "dependencies": {