bytesTotal tested

Author: max-ostapenko

definitions/output/reports/reports_dynamic.js

@@ -2,37 +2,47 @@ const configs = new reports.HTTPArchiveReports()
 const metrics = configs.listMetrics()
 const lenses = configs.lenses
 
-const bucket = 'httparchive'
-const storagePath = '/reports/dev/'
+const bucket = constants.bucket
+const storagePath = constants.storagePath
+const dataset = 'reports'
 
 // Adjust start and end dates to update reports retrospectively
-const startDate = '2024-12-01' // constants.currentMonth;
-const endDate = '2024-12-01' // constants.currentMonth;
+const startDate = constants.currentMonth; // '2025-07-01'
+const endDate = constants.currentMonth; // '2025-07-01'
 
-function generateExportPath (ctx, params) {
+function generateExportPath (params) {
+  objectName = storagePath
   if (params.sql.type === 'histogram') {
-    return `${storagePath}${params.date.replaceAll('-', '_')}/${params.metric.id}.json`
+    objectName = objectName + params.date.replaceAll('-', '_') + '/' + params.metric.id
   } else if (params.sql.type === 'timeseries') {
-    return `${storagePath}${params.metric.id}.json`
+    objectName = objectName + params.metric.id
   } else {
     throw new Error('Unknown SQL type')
   }
+  return objectName + '_test.json' // TODO: remove test suffix from the path
 }
 
-function generateExportQuery (ctx, params) {
+function generateExportQuery (params) {
   let query = ''
   if (params.sql.type === 'histogram') {
     query = `
-SELECT * EXCEPT(date)
-FROM \`reports.${params.sql.type}\`
+SELECT
+  * EXCEPT(date, metric, lens)
+FROM \`${dataset}.${params.tableName}\`
 WHERE date = '${params.date}'
+  AND metric = '${params.metric.id}'
+  AND lens = '${params.lens.name}'
+ORDER BY bin ASC
 `
   } else if (params.sql.type === 'timeseries') {
     query = `
 SELECT
   FORMAT_DATE('%Y_%m_%d', date) AS date,
-  * EXCEPT(date)
-FROM \`reports.${params.sql.type}\`
+  * EXCEPT(date, metric, lens)
+FROM \`${dataset}.${params.tableName}\`
+WHERE metric = '${params.metric.id}'
+  AND lens = '${params.lens.name}'
+ORDER BY date DESC
 `
   } else {
     throw new Error('Unknown SQL type')
@@ -60,44 +70,54 @@ for (
           metric,
           sql,
           lens: { name: key, sql: value },
-          devRankFilter: constants.devRankFilter
+          devRankFilter: constants.devRankFilter,
+          tableName: metric.id + '_' + sql.type
         })
       }
     })
   })
 }
 
 iterations.forEach((params, i) => {
-  operate(
-    params.metric.id + '_' + params.sql.type + '_' + params.lens.name + '_' + params.date)
+  operate(params.tableName + '_' + params.date + '_' + params.lens.name)
     .tags(['crawl_complete', 'reports'])
     .queries(ctx => `
-CREATE TABLE IF NOT EXISTS reports.${params.sql.type} (
-  date DATE,
-  lens STRING,
-  metric STRING,
-  client STRING,
-  data JSON
-)
+DECLARE job_config JSON;
+
+/* First report run
+CREATE TABLE IF NOT EXISTS ${dataset}.${params.tableName}
 PARTITION BY date
-CLUSTER BY metric, lens, client;
+CLUSTER BY metric, lens, client
+AS
+*/
 
-DELETE FROM reports.${params.sql.type}
+--/* Subsequent report run
+DELETE FROM ${dataset}.${params.tableName}
 WHERE date = '${params.date}'
-AND metric = '${params.metric.id}';
-
-INSERT INTO reports.${params.sql.type} ${params.sql.query(ctx, params)};
+  AND metric = '${params.metric.id}'
+  AND lens = '${params.lens.name}';
+INSERT INTO ${dataset}.${params.tableName}
+--*/
 
 SELECT
-reports.run_export_job(
-  JSON '''{
-    "destination": "cloud_storage",
-    "config": {
-      "bucket": "${bucket}",
-      "name": "${generateExportPath(ctx, params)}"
-    },
-    "query": "${generateExportQuery(ctx, params)}"
-  }'''
+  '${params.metric.id}' AS metric,
+  '${params.lens.name}' AS lens,
+  *
+FROM (
+  ${params.sql.query(ctx, params)}
 );
-  `)
+
+SET job_config = TO_JSON(
+  STRUCT(
+    "cloud_storage" AS destination,
+    STRUCT(
+      "httparchive" AS bucket,
+      "${generateExportPath(params)}" AS name
+    ) AS config,
+    r"${generateExportQuery(params)}" AS query
+  )
+);
+
+SELECT reports.run_export_job(job_config);
+    `)
 })

includes/constants.js

@@ -49,6 +49,7 @@ class DataformTemplateBuilder {
         if (typeof value === 'string') return `'${value}'`
         if (typeof value === 'number') return value.toString()
         if (typeof value === 'boolean') return value.toString()
+        if (typeof value === 'function') return value.toString()
 
         // For objects or arrays, use JSON.stringify
         return JSON.stringify(value)

includes/reports.js

@@ -12,9 +12,10 @@ WITH pages AS (
     date,
     client,
     CAST(FLOOR(FLOAT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin
-  FROM crawl.pages
+  FROM ${ctx.ref('crawl', 'pages')}
   WHERE
     date = '${params.date}'
+    ${params.devRankFilter}
     ${params.lens.sql}
     AND is_root_page
     AND FLOAT64(summary.bytesTotal) > 0
@@ -52,9 +53,10 @@ WITH pages AS (
     date,
     client,
     FLOAT64(summary.bytesTotal) AS bytesTotal
-  FROM crawl.pages
+  FROM ${ctx.ref('crawl', 'pages')}
   WHERE
     date = '${params.date}'
+    ${params.devRankFilter}
     ${params.lens.sql}
     AND is_root_page
     AND INT64(summary.bytesTotal) > 0
@@ -63,7 +65,7 @@ WITH pages AS (
 SELECT
   date,
   client,
-  UNIX_SECONDS(TIMESTAMP(date)) AS timestamp,
+  UNIX_DATE(date) * 1000 * 60 * 60 * 24 AS timestamp,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(101)] / 1024, 2) AS p10,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(251)] / 1024, 2) AS p25,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50,