Strict merge of CrUX and crawl (#85)

* versions

* technologies

* categories

* Update definitions/output/reports/tech_report_categories.js

Co-authored-by: Barry Pollard <barrypollard@google.com>

* all pages in categories = 'ALL'

---------

Co-authored-by: Barry Pollard <barrypollard@google.com>

Author: max-ostapenko

definitions/output/reports/tech_report_categories.js

@@ -23,6 +23,26 @@ category_descriptions AS (
   FROM ${ctx.ref('wappalyzer', 'categories')}
 ),
 
+crux AS (
+  SELECT
+    IF(device = 'desktop', 'desktop', 'mobile') AS client,
+    CONCAT(origin, '/') AS root_page
+  FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')}
+  WHERE
+    date = '${pastMonth}'
+    AND device IN ('desktop', 'phone')
+),
+
+merged_pages AS (
+  SELECT DISTINCT
+    client,
+    technologies,
+    root_page
+  FROM pages
+  INNER JOIN crux
+  USING (client, root_page)
+),
+
 category_stats AS (
   SELECT
     category,
@@ -35,8 +55,8 @@ category_stats AS (
       client,
       category,
       COUNT(DISTINCT root_page) AS origins
-    FROM pages
-    INNER JOIN pages.technologies AS tech
+    FROM merged_pages
+    INNER JOIN merged_pages.technologies AS tech
     INNER JOIN tech.categories AS category
     WHERE
       category IS NOT NULL
@@ -87,7 +107,7 @@ FROM (
   SELECT
     client,
     COUNT(DISTINCT root_page) AS origins
-  FROM pages
+  FROM merged_pages
   GROUP BY client
 )
 `).postOps(ctx => `

definitions/output/reports/tech_report_technologies.js

@@ -5,36 +5,17 @@ publish('tech_report_technologies', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-WITH pages AS (
-  SELECT DISTINCT
-    client,
-    root_page,
-    tech.technology
-  FROM ${ctx.ref('crawl', 'pages')} AS pages
-  INNER JOIN pages.technologies AS tech
-  WHERE
-    date = '${pastMonth}'
-    ${constants.devRankFilter}
-),
-
-tech_origins AS (
+WITH tech_origins AS (
   SELECT
-    technology,
-    STRUCT(
-      MAX(IF(client = 'desktop', origins, 0)) AS desktop,
-      MAX(IF(client = 'mobile', origins, 0)) AS mobile
-    ) AS origins
-  FROM (
-    SELECT
-      client,
-      technology,
-      COUNT(DISTINCT root_page) AS origins
-    FROM pages
-    GROUP BY
-      client,
-      technology
-  )
-  GROUP BY technology
+  technology,
+  adoption AS origins
+FROM ${ctx.ref('reports', 'tech_report_adoption')}
+WHERE
+  date = '${pastMonth}'
+  AND rank = 'ALL'
+  AND geo = 'ALL'
+  AND version = 'ALL'
+  ${constants.devRankFilter}
 ),
 
 technologies AS (
@@ -51,14 +32,6 @@ technologies AS (
     description,
     categories,
     icon
-),
-
-total_pages AS (
-  SELECT
-    client,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY client
 )
 
 SELECT
@@ -75,16 +48,14 @@ USING(technology)
 UNION ALL
 
 SELECT
-  'ALL' AS technology,
+  technology,
   NULL AS description,
   NULL AS icon,
   NULL AS category,
   NULL AS category_obj,
-  STRUCT(
-    MAX(IF(client = 'desktop', origins, 0)) AS desktop,
-    MAX(IF(client = 'mobile', origins, 0)) AS mobile
-  ) AS origins
-FROM total_pages
+  origins
+FROM tech_origins
+WHERE technology = 'ALL'
 `).postOps(ctx => `
   SELECT
     reports.run_export_job(

definitions/output/reports/tech_report_versions.js

@@ -5,61 +5,16 @@ publish('tech_report_versions', {
   type: 'table',
   tags: ['tech_report']
 }).query(ctx => `
-WITH pages AS (
-  SELECT DISTINCT
-    client,
-    root_page,
-    tech.technology,
-    REGEXP_EXTRACT(version, r'\\d+(?:\\.\\d+)?') AS version
-  FROM ${ctx.ref('crawl', 'pages')} AS pages
-  INNER JOIN pages.technologies AS tech
-  LEFT JOIN tech.info AS version
-  WHERE
-    date = '${pastMonth}'
-    ${constants.devRankFilter} AND
-    tech.technology IS NOT NULL
-),
-
-version_origins AS (
-  SELECT
-    client,
-    technology,
-    version,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  WHERE version IS NOT NULL
-  GROUP BY
-    client,
-    technology,
-    version
-),
-
-total_origins AS (
-  SELECT
-    client,
-    technology,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY
-    client,
-    technology
-)
-
 SELECT
-  client,
   technology,
   version,
-  origins
-FROM version_origins
-
-UNION ALL
-
-SELECT
-  client,
-  technology,
-  'ALL' AS version,
-  origins
-FROM total_origins
+  adoption AS origins
+FROM ${ctx.ref('reports', 'tech_report_adoption')}
+WHERE
+  date = '${pastMonth}'
+  AND rank = 'ALL'
+  AND geo = 'ALL'
+  ${constants.devRankFilter}
 `).postOps(ctx => `
   SELECT
     reports.run_export_job(