Merge branch 'main' into standard_reports

Author: max-ostapenko

definitions/output/blink_features/features.js

@@ -6,7 +6,7 @@ publish('features', {
     partitionBy: 'yyyymmdd',
     clusterBy: ['client', 'rank']
   },
-  tags: ['crawl_complete']
+  tags: ['crawl_complete', 'blink_report']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE yyyymmdd = DATE '${constants.currentMonth}';

definitions/output/blink_features/usage.js

@@ -2,7 +2,7 @@ publish('usage', {
   schema: 'blink_features',
   type: 'incremental',
   protected: true,
-  tags: ['crawl_complete']
+  tags: ['crawl_complete', 'blink_report']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE yyyymmdd = REPLACE('${constants.currentMonth}', '-', '');

definitions/output/core_web_vitals/technologies.js

@@ -9,7 +9,7 @@ publish('technologies', {
     clusterBy: ['geo', 'app', 'rank', 'client'],
     requirePartitionFilter: true
   },
-  tags: ['crux_ready'],
+  tags: ['crux_ready', 'tech_report'],
   dependOnDependencyAssertions: true
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}

definitions/output/crawl/pages.js

@@ -73,7 +73,7 @@ WHERE date = '${constants.currentMonth}' AND
   ${constants.devRankFilter}
 `).postOps(ctx => `
 CREATE TEMP TABLE technologies_cleaned AS (
-  WITH wappalyzer AS (
+  WITH technologies AS (
     SELECT DISTINCT
       name AS technology,
       categories
@@ -101,13 +101,13 @@ CREATE TEMP TABLE technologies_cleaned AS (
     LEFT JOIN pages.categories AS category
     WHERE
       -- Technology is corrupted
-      technology NOT IN (SELECT DISTINCT technology FROM wappalyzer) OR
+      technology NOT IN (SELECT DISTINCT technology FROM technologies) OR
       -- Technology's category is corrupted
       CONCAT(technology, category) NOT IN (
         SELECT DISTINCT
           CONCAT(technology, category)
-        FROM wappalyzer
-        LEFT JOIN wappalyzer.categories AS category
+        FROM technologies
+        INNER JOIN technologies.categories AS category
       )
   ),
 
@@ -118,14 +118,14 @@ CREATE TEMP TABLE technologies_cleaned AS (
       page,
       ARRAY_AGG(STRUCT(
         pages.technology,
-        wappalyzer.categories,
+        technologies.categories,
         pages.info
       )) AS technologies
     FROM pages
     INNER JOIN impacted_pages
     USING (client, page)
-    INNER JOIN wappalyzer
-    ON pages.technology = wappalyzer.technology
+    INNER JOIN technologies
+    USING (technology)
     GROUP BY
       client,
       page
@@ -134,8 +134,10 @@ CREATE TEMP TABLE technologies_cleaned AS (
   SELECT
     client,
     page,
-    technologies
-  FROM reconstructed_technologies
+    reconstructed_technologies.technologies
+  FROM impacted_pages
+  LEFT JOIN reconstructed_technologies
+  USING(client,page)
 );
 
 -- Update the crawl.pages table with the cleaned and restored technologies

definitions/output/reports/cwv_tech_adoption.js

@@ -8,7 +8,7 @@ publish('cwv_tech_adoption', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).preOps(ctx => `
 DELETE FROM ${ctx.self()}
 WHERE date = '${pastMonth}';

definitions/output/reports/cwv_tech_categories.js

@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('cwv_tech_categories', {
   schema: 'reports',
   type: 'table',
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
 WITH pages AS (
@@ -71,5 +71,22 @@ GROUP BY
   category,
   description,
   origins
-ORDER BY category ASC
+
+UNION ALL
+
+SELECT
+  'ALL' AS category,
+  NULL AS description,
+  STRUCT(
+    COALESCE(MAX(IF(client = 'desktop', origins, 0))) AS desktop,
+    COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile
+  ) AS origins,
+  NULL AS technologies
+FROM (
+  SELECT
+    client,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY client
+)
 `)

definitions/output/reports/cwv_tech_core_web_vitals.js

@@ -8,7 +8,7 @@ publish('cwv_tech_core_web_vitals', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_VITALS(
   records ARRAY<STRUCT<

definitions/output/reports/cwv_tech_lighthouse.js

@@ -8,7 +8,7 @@ publish('cwv_tech_lighthouse', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE(
   records ARRAY<STRUCT<

definitions/output/reports/cwv_tech_page_weight.js

@@ -8,7 +8,7 @@ publish('cwv_tech_page_weight', {
     partitionBy: 'date',
     clusterBy: ['rank', 'geo']
   },
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).preOps(ctx => `
 CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT(
   records ARRAY<STRUCT<

definitions/output/reports/cwv_tech_technologies.js

@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
 publish('cwv_tech_technologies', {
   schema: 'reports',
   type: 'table',
-  tags: ['crux_ready']
+  tags: ['crux_ready', 'tech_report']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
 WITH pages AS (