Merge branch 'main' into smart-otter

max-ostapenko · web-flow · commit ada937425c4e · 2025-05-21T09:22:18.000+02:00
diff --git a/definitions/output/reports/tech_crux.js b/definitions/output/reports/tech_crux.js
@@ -9,8 +9,8 @@ publish('tech_crux', {
     clusterBy: ['geo', 'client', 'rank', 'technology'],
     requirePartitionFilter: true
   },
-  tags: ['tech_report'],
-  dependOnDependencyAssertions: true
+  dependOnDependencyAssertions: true,
+  tags: ['crux_ready']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
diff --git a/definitions/output/reports/tech_report_adoption.js b/definitions/output/reports/tech_report_adoption.js
@@ -8,7 +8,7 @@ publish('tech_report_adoption', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';
diff --git a/definitions/output/reports/tech_report_audits.js b/definitions/output/reports/tech_report_audits.js
@@ -8,7 +8,7 @@ publish('tech_report_audits', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMP FUNCTION GET_AUDITS(
   records ARRAY<STRUCT<
diff --git a/definitions/output/reports/tech_report_categories.js b/definitions/output/reports/tech_report_categories.js
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('tech_report_categories', {
   schema: 'reports',
   type: 'table',
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 WITH pages AS (
   SELECT DISTINCT
diff --git a/definitions/output/reports/tech_report_core_web_vitals.js b/definitions/output/reports/tech_report_core_web_vitals.js
@@ -8,7 +8,7 @@ publish('tech_report_core_web_vitals', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_VITALS(
   records ARRAY<STRUCT<
diff --git a/definitions/output/reports/tech_report_lighthouse.js b/definitions/output/reports/tech_report_lighthouse.js
@@ -8,7 +8,7 @@ publish('tech_report_lighthouse', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE(
   records ARRAY<STRUCT<
diff --git a/definitions/output/reports/tech_report_page_weight.js b/definitions/output/reports/tech_report_page_weight.js
@@ -8,7 +8,7 @@ publish('tech_report_page_weight', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT(
   records ARRAY<STRUCT<
diff --git a/definitions/output/reports/tech_report_technologies.js b/definitions/output/reports/tech_report_technologies.js
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('tech_report_technologies', {
   schema: 'reports',
   type: 'table',
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 WITH tech_origins AS (
   SELECT
diff --git a/definitions/output/reports/tech_report_versions.js b/definitions/output/reports/tech_report_versions.js
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('tech_report_versions', {
   schema: 'reports',
   type: 'table',
-  tags: ['tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 SELECT
   technology,
diff --git a/infra/bigquery-export/package-lock.json b/infra/bigquery-export/package-lock.json
diff --git a/infra/bigquery-export/package.json b/infra/bigquery-export/package.json
@@ -9,7 +9,7 @@
   "type": "module",
   "dependencies": {
     "@google-cloud/bigquery": "8.0.0",
-    "@google-cloud/firestore": "7.11.0",
+    "@google-cloud/firestore": "7.11.1",
     "@google-cloud/storage": "7.16.0"
   },
   "author": "@max-ostapenko"
diff --git a/infra/dataform-trigger/index.js b/infra/dataform-trigger/index.js
@@ -31,7 +31,7 @@ FROM crux, report;
     action: 'runDataformRepo',
     actionArgs: {
       repoName: 'crawl-data',
-      tags: ['tech_report']
+      tags: ['crux_ready']
     }
   },
   crawl_complete: {
diff --git a/infra/tf/.terraform.lock.hcl b/infra/tf/.terraform.lock.hcl
diff --git a/infra/tf/dataform.tf b/infra/tf/dataform.tf
@@ -20,12 +20,15 @@ locals {
     "core_web_vitals", // TODO: Remove after tech report migration
     "reports",
 
+    // Flattened tables for F1
+    "f1",
+
     // Service
     "dataform_assertions",
   ]
 
   dataform_service_account_roles = [
-    "roles/bigquery.jobUser",
+    "roles/bigquery.user",
     "roles/bigquery.connectionUser",
     "roles/dataform.serviceAgent",
   ]
diff --git a/infra/tf/dataform_trigger/main.tf b/infra/tf/dataform_trigger/main.tf
@@ -110,7 +110,7 @@ resource "google_cloud_scheduler_job" "bq-poller-crux-ready" {
   paused           = false
   project          = var.project
   region           = var.region
-  schedule         = "0 */8 8-14 * *"
+  schedule         = "0 8,12,16 8-14 * *"
   time_zone        = "Etc/UTC"
   http_target {
     body = base64encode(local.crux_ready_scheduler_body)
diff --git a/infra/tf/masthead/main.tf b/infra/tf/masthead/main.tf
@@ -49,15 +49,6 @@ resource "google_project_iam_member" "masthead_pubsub_publisherer_member" {
 }
 
 # 3. Grant Masthead Service Account roles
-resource "google_project_iam_custom_role" "masthead_bq_meta_reader" {
-  project     = var.project
-  description = "Masthead BigQuery assets metadata reader"
-  permissions = ["bigquery.datasets.get", "bigquery.tables.get", "bigquery.tables.list", "bigquery.routines.get", "bigquery.routines.list"]
-  role_id     = "masthead_bq_meta_reader"
-  stage       = "GA"
-  title       = "masthead_bq_meta_reader"
-}
-
 resource "google_project_iam_member" "masthead_pubsub_subscriber_member" {
   for_each = toset(["roles/bigquery.metadataViewer", "roles/bigquery.resourceViewer", "roles/pubsub.subscriber"])
 
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
