Merge branch 'main' into standard_reports

Author: max-ostapenko

definitions/declarations/chrome-ux-report.js

@@ -30,9 +30,3 @@ FROM ${ctx.ref(database, 'materialized', 'device_summary')}
 |> WHERE cnt_devices != 3 OR cnt_ranks != 10
 |> SELECT "Table data doesn't match 3 unique devices and 10 ranks" AS error_message
 `)
-
-declare({
-  database,
-  schema: 'experimental',
-  name: 'global'
-})

definitions/declarations/httparchive.js

@@ -40,3 +40,21 @@ ORDER BY cnt_pages DESC
     name: table
   })
 )
+
+operate('create_reservation_assignment')
+  .tags(['crawl_complete'])
+  .queries(ctx => `
+CREATE ASSIGNMENT
+\`httparchive.region-us.retrospective-reprocessing.pipeline\`
+OPTIONS (
+  assignee = 'projects/httparchive',
+  job_type = 'QUERY')
+`)
+
+operate('drop_reservation_assignment')
+  .dependencies(['requests_10k'])
+  .tags(['crawl_complete'])
+  .queries(ctx => `
+DROP ASSIGNMENT IF EXISTS
+\`httparchive.region-us.retrospective-reprocessing.pipeline\`
+`)

definitions/output/core_web_vitals/technology_matrix.js

@@ -0,0 +1,55 @@
+publish('technology_matrix', {
+  type: 'table',
+  schema: 'core_web_vitals',
+  bigquery: {
+    partitionBy: 'date',
+    clusterBy: ['client', 't1']
+  },
+  description: 'Used in: https://lookerstudio.google.com/u/2/reporting/0ad64c8f-644a-40f9-93e4-0ccd5f72b33d',
+  tags: ['crawl_complete']
+}).query(ctx => `
+WITH a AS (
+  SELECT
+    date,
+    client,
+    root_page,
+    technology AS t1,
+    ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c1
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies)
+  WHERE
+    date = '${constants.currentMonth}' AND
+    is_root_page
+), b AS (
+  SELECT
+    client,
+    root_page,
+    technology AS t2,
+    ARRAY_TO_STRING(ARRAY(SELECT category FROM UNNEST(categories) AS category ORDER BY category), ', ') AS c2
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies)
+  WHERE
+    date = '${constants.currentMonth}' AND
+    is_root_page
+)
+
+
+SELECT
+  date,
+  client,
+  t1,
+  c1,
+  t2,
+  c2,
+  COUNT(DISTINCT root_page) AS pages
+FROM a
+LEFT JOIN b
+USING (client, root_page)
+GROUP BY
+  date,
+  client,
+  t1,
+  c1,
+  t2,
+  c2
+`)

definitions/output/crawl/pages.js

@@ -47,7 +47,8 @@ publish('pages', {
     technologies: 'Technologies detected at runtime (see https://www.wappalyzer.com/)',
     metadata: 'Additional metadata about the test'
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}' AND

definitions/output/crawl/parsed_css.js

@@ -7,7 +7,8 @@ publish('parsed_css', {
     clusterBy: ['client', 'is_root_page', 'rank', 'page'],
     requirePartitionFilter: true
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${constants.currentMonth}'

definitions/output/crawl/requests.js

@@ -36,7 +36,8 @@ publish('requests', {
     },
     response_body: 'Text-based response body'
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete'],
+  dependencies: ['create_reservation_assignment']
 }).preOps(ctx => `
 FOR client_value IN (SELECT * FROM UNNEST(['desktop', 'mobile']) AS client) DO
   FOR is_root_page_value IN (SELECT * FROM UNNEST([TRUE, FALSE]) AS is_root_page) DO

definitions/output/reports/cwv_tech_adoption.js

@@ -8,7 +8,7 @@ publish('cwv_tech_adoption', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready', 'tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';

definitions/output/reports/cwv_tech_categories.js

@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('cwv_tech_categories', {
   schema: 'reports',
   type: 'table',
-  tags: ['crux_ready', 'tech_report']
+  tags: ['crux_ready']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
@@ -55,6 +55,14 @@ technology_stats AS (
   GROUP BY
     technology,
     categories
+),
+
+total_pages AS (
+  SELECT
+    client,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY client
 )
 
 SELECT
@@ -82,11 +90,5 @@ SELECT
     COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile
   ) AS origins,
   NULL AS technologies
-FROM (
-  SELECT
-    client,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY client
-)
+FROM total_pages
 `)

definitions/output/reports/cwv_tech_core_web_vitals.js

@@ -8,7 +8,7 @@ publish('cwv_tech_core_web_vitals', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready', 'tech_report']
+  tags: ['tech_report']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_VITALS(
   records ARRAY<STRUCT<

definitions/output/reports/cwv_tech_lighthouse.js

@@ -8,7 +8,7 @@ publish('cwv_tech_lighthouse', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready', 'tech_report']
+  tags: ['crux_ready']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE(
   records ARRAY<STRUCT<