all metrics in the same table

Author: max-ostapenko

definitions/output/reports/reports_dynamic.js

@@ -33,42 +33,37 @@ for (
           date,
           metric,
           sql,
-          lense: { name: key, sql: value },
+          lens: { name: key, sql: value },
           devRankFilter: constants.devRankFilter
         })
       }
     })
   })
 }
 
-if (startDate === endDate) {
-  iterations.forEach((params, i) => {
-    publish(params.metric.id + '_' + params.sql.type + '_' + params.lens.name, {
-      type: 'incremental',
-      protected: true,
-      bigquery: params.sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-      schema: 'reports',
-      tags: ['crawl_complete', 'reports']
-    }).preOps(ctx => `
---DELETE FROM ${ctx.self()}
---WHERE date = '${params.date}';
-    `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${params.metric.id}", "type": "${params.sql.type}", "lense": "${params.lens.name}"} */` +
-params.sql.query(ctx, params)
-    )
-  })
-} else {
-  iterations.forEach((params, i) => {
-    operate(
-      params.metric.id + '_' + params.sql.type + '_' + params.lens.name + '_' + params.date)
-      .tags(['crawl_complete', 'reports'])
-      .queries(ctx => `
+iterations.forEach((params, i) => {
+  operate(
+    params.metric.id + '_' + params.sql.type + '_' + params.lens.name + '_' + params.date)
+    .tags(['crawl_complete', 'reports'])
+    .queries(ctx => `
+CREATE TABLE IF NOT EXISTS reports.${params.metric.id}_${params.sql.type} (
+  metric STRING,
+  date DATE,
+  client STRING,
+  lens STRING,
+  bin INT64,
+  volume INT64,
+  pdf FLOAT64,
+  cdf FLOAT64
+)
+PARTITION BY date
+CLUSTER BY client, lens;
+
 DELETE FROM reports.${params.metric.id}_${params.sql.type}
 WHERE date = '${params.date}';
 
 /* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${params.metric.id}", "type": "${params.sql.type}", "lense": "${params.lens.name}"} */
 INSERT INTO reports.${params.metric.id}_${params.sql.type}` +
 params.sql.query(ctx, params)
-      )
-  })
-}
+    )
+})

includes/reports.js

@@ -7,24 +7,31 @@ const config = {
         {
           type: 'histogram',
           query: DataformTemplateBuilder.create((ctx, params) => `
+WITH pages AS (
+  SELECT
+    date,
+    client,
+    CAST(FLOOR(INT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin
+  FROM crawl.pages
+  WHERE
+    date = '${params.date}' ${params.lens.sql} AND
+    is_root_page AND
+    INT64(summary.bytesTotal) > 0
+)
+
 SELECT
   *,
+  '${params.lens.name}' AS lens,
   SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf
 FROM (
   SELECT
     *,
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      date,
-      client,
-      CAST(FLOOR(INT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin,
+      *,
       COUNT(0) AS volume
-    FROM ${ctx.ref('crawl', 'pages')}
-    WHERE
-      date = '${params.date}' ${params.devRankFilter} ${params.lense.sql} AND
-      is_root_page AND
-      INT64(summary.bytesTotal) > 0
+    FROM pages
     GROUP BY
       date,
       client,
@@ -46,16 +53,17 @@ WITH pages AS (
     date,
     client,
     INT64(summary.bytesTotal) AS bytesTotal
-  FROM ${ctx.ref('crawl', 'pages')}
+  FROM crawl.pages
   WHERE
-    date = '${params.date}' ${params.devRankFilter} ${params.lense.sql} AND
+    date = '${params.date}' $ ${params.lens.sql} AND
     is_root_page AND
     INT64(summary.bytesTotal) > 0
 )
 
 SELECT
   date,
   client,
+  '${params.lens.name}' AS lens,
   UNIX_SECONDS(TIMESTAMP(date)) AS timestamp,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25,