Skip to content

Commit 28cebce

Browse files
max-ostapenkomax-ostapenko
authored
Merge branch 'main' into standard_reports
2 parents 6bb7ce4 + 88387c6 commit 28cebce

6 files changed

Lines changed: 231 additions & 48 deletions

File tree

‎definitions/declarations/httparchive.js‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,42 @@
1-
const stagingTables = ['pages', 'requests', 'parsed_css']
2-
for (const table of stagingTables) {
1+
// Staging tables source: https://github.com/HTTPArchive/crawl/blob/main/crawl.py
2+
['pages', 'requests', 'parsed_css'].forEach(table =>
33
declare({
44
schema: 'crawl_staging',
55
name: table
66
})
7-
}
7+
)
88

9-
declare({
10-
schema: 'wappalyzer',
11-
name: 'apps'
12-
})
9+
// See https://github.com/HTTPArchive/dataform/issues/43
10+
assert('corrupted_technology_values')
11+
.tags(['crawl_complete'])
12+
.query(ctx => `
13+
SELECT
14+
date,
15+
client,
16+
tech,
17+
COUNT(DISTINCT page) AS cnt_pages,
18+
ARRAY_AGG(DISTINCT page LIMIT 3) AS sample_pages
19+
FROM ${ctx.ref('crawl_staging', 'pages')} AS pages
20+
LEFT JOIN pages.technologies AS tech
21+
LEFT JOIN tech.categories AS category
22+
WHERE
23+
date = '${constants.currentMonth}' AND
24+
(
25+
tech.technology NOT IN (SELECT DISTINCT name FROM wappalyzer.technologies)
26+
OR category NOT IN (SELECT DISTINCT name FROM wappalyzer.categories)
27+
OR ARRAY_LENGTH(tech.categories) = 0
28+
)
29+
GROUP BY
30+
date,
31+
client,
32+
tech
33+
ORDER BY cnt_pages DESC
34+
`);
35+
36+
// Wappalyzer tables source: https://github.com/HTTPArchive/wappalyzer/blob/main/.github/workflows/upload.yml
37+
['technologies', 'categories'].forEach(table =>
38+
declare({
39+
schema: 'wappalyzer',
40+
name: table
41+
})
42+
)

‎definitions/output/crawl/pages.js‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,23 +52,97 @@ publish('pages', {
5252
DELETE FROM ${ctx.self()}
5353
WHERE date = '${constants.currentMonth}' AND
5454
client = 'desktop';
55-
`).query(ctx => `
55+
56+
INSERT INTO ${ctx.self()}
5657
SELECT
5758
*
5859
FROM ${ctx.ref('crawl_staging', 'pages')}
5960
WHERE date = '${constants.currentMonth}' AND
6061
client = 'desktop'
61-
${constants.devRankFilter}
62-
`).postOps(ctx => `
62+
${constants.devRankFilter};
63+
6364
DELETE FROM ${ctx.self()}
6465
WHERE date = '${constants.currentMonth}' AND
6566
client = 'mobile';
66-
67-
INSERT INTO ${ctx.self()}
67+
`).query(ctx => `
6868
SELECT
6969
*
7070
FROM ${ctx.ref('crawl_staging', 'pages')}
7171
WHERE date = '${constants.currentMonth}' AND
7272
client = 'mobile'
7373
${constants.devRankFilter}
74+
`).postOps(ctx => `
75+
CREATE TEMP TABLE technologies_cleaned AS (
76+
WITH wappalyzer AS (
77+
SELECT DISTINCT
78+
name AS technology,
79+
categories
80+
FROM ${ctx.ref('wappalyzer', 'technologies')}
81+
),
82+
83+
pages AS (
84+
SELECT
85+
client,
86+
page,
87+
tech.technology,
88+
tech.categories,
89+
tech.info
90+
FROM ${ctx.self()} AS pages
91+
LEFT JOIN pages.technologies AS tech
92+
WHERE date = '${constants.currentMonth}' ${constants.devRankFilter}
93+
),
94+
95+
-- Identify impacted pages
96+
impacted_pages AS (
97+
SELECT DISTINCT
98+
client,
99+
page
100+
FROM pages
101+
LEFT JOIN pages.categories AS category
102+
WHERE
103+
-- Technology is corrupted
104+
technology NOT IN (SELECT DISTINCT technology FROM wappalyzer) OR
105+
-- Technology's category is corrupted
106+
CONCAT(technology, category) NOT IN (
107+
SELECT DISTINCT
108+
CONCAT(technology, category)
109+
FROM wappalyzer
110+
LEFT JOIN wappalyzer.categories AS category
111+
)
112+
),
113+
114+
-- Keep valid technologies and use correct categories
115+
reconstructed_technologies AS (
116+
SELECT
117+
client,
118+
page,
119+
ARRAY_AGG(STRUCT(
120+
pages.technology,
121+
wappalyzer.categories,
122+
pages.info
123+
)) AS technologies
124+
FROM pages
125+
INNER JOIN impacted_pages
126+
USING (client, page)
127+
INNER JOIN wappalyzer
128+
ON pages.technology = wappalyzer.technology
129+
GROUP BY
130+
client,
131+
page
132+
)
133+
134+
SELECT
135+
client,
136+
page,
137+
technologies
138+
FROM reconstructed_technologies
139+
);
140+
141+
-- Update the crawl.pages table with the cleaned and restored technologies
142+
UPDATE ${ctx.self()} AS pages
143+
SET technologies = technologies_cleaned.technologies
144+
FROM technologies_cleaned
145+
WHERE pages.date = '${constants.currentMonth}' AND
146+
pages.client = technologies_cleaned.client AND
147+
pages.page = technologies_cleaned.page;
74148
`)

‎definitions/output/reports/cwv_tech_adoption.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ SELECT
2020
rank,
2121
geo,
2222
STRUCT(
23-
COALESCE(MAX(IF(client = 'desktop', origins, NULL))) AS desktop,
24-
COALESCE(MAX(IF(client = 'mobile', origins, NULL))) AS mobile
23+
COALESCE(MAX(IF(client = 'desktop', origins, 0))) AS desktop,
24+
COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile
2525
) AS adoption
2626
FROM ${ctx.ref('core_web_vitals', 'technologies')}
2727
WHERE date = '${pastMonth}'

‎definitions/output/reports/cwv_tech_categories.js‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,44 +7,69 @@ publish('cwv_tech_categories', {
77
}).query(ctx => `
88
/* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
99
WITH pages AS (
10-
SELECT
10+
SELECT DISTINCT
11+
client,
1112
root_page,
1213
technologies
1314
FROM ${ctx.ref('crawl', 'pages')}
1415
WHERE
15-
date = '${pastMonth}' AND
16-
client = 'mobile'
16+
date = '${pastMonth}'
1717
${constants.devRankFilter}
18-
), categories AS (
18+
),
19+
20+
category_descriptions AS (
21+
SELECT
22+
name AS category,
23+
description
24+
FROM ${ctx.ref('wappalyzer', 'categories')}
25+
),
26+
27+
category_stats AS (
1928
SELECT
2029
category,
21-
COUNT(DISTINCT root_page) AS origins
22-
FROM pages,
23-
UNNEST(technologies) AS t,
24-
UNNEST(t.categories) AS category
30+
STRUCT(
31+
COALESCE(MAX(IF(client = 'desktop', origins, 0))) AS desktop,
32+
COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile
33+
) AS origins
34+
FROM (
35+
SELECT
36+
client,
37+
category,
38+
COUNT(DISTINCT root_page) AS origins
39+
FROM pages
40+
LEFT JOIN pages.technologies AS tech
41+
LEFT JOIN tech.categories AS category
42+
GROUP BY
43+
client,
44+
category
45+
)
2546
GROUP BY category
26-
), technologies AS (
47+
),
48+
49+
technology_stats AS (
2750
SELECT
28-
category,
2951
technology,
30-
COUNT(DISTINCT root_page) AS origins
31-
FROM pages,
32-
UNNEST(technologies) AS t,
33-
UNNEST(t.categories) AS category
52+
category_obj AS categories,
53+
SUM(origins) AS total_origins
54+
FROM ${ctx.ref('reports', 'cwv_tech_technologies')}
3455
GROUP BY
35-
category,
36-
technology
56+
technology,
57+
categories
3758
)
3859
3960
SELECT
4061
category,
41-
categories.origins,
42-
ARRAY_AGG(technology IGNORE NULLS ORDER BY technologies.origins DESC) AS technologies
43-
FROM categories
44-
JOIN technologies
62+
description,
63+
origins,
64+
ARRAY_AGG(technology IGNORE NULLS ORDER BY technology_stats.total_origins DESC) AS technologies
65+
FROM category_stats
66+
INNER JOIN technology_stats
67+
ON category_stats.category IN UNNEST(technology_stats.categories)
68+
INNER JOIN category_descriptions
4569
USING (category)
4670
GROUP BY
4771
category,
48-
categories.origins
49-
ORDER BY categories.origins DESC
72+
description,
73+
origins
74+
ORDER BY category ASC
5075
`)

‎definitions/output/reports/cwv_tech_technologies.js‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,73 @@ publish('cwv_tech_technologies', {
66
tags: ['crux_ready']
77
}).query(ctx => `
88
/* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
9+
WITH pages AS (
10+
SELECT DISTINCT
11+
client,
12+
root_page,
13+
tech.technology
14+
FROM ${ctx.ref('crawl', 'pages')},
15+
UNNEST(technologies) AS tech
16+
WHERE
17+
date = '${pastMonth}'
18+
${constants.devRankFilter}
19+
),
20+
21+
tech_origins AS (
22+
SELECT
23+
client,
24+
technology,
25+
COUNT(DISTINCT root_page) AS origins
26+
FROM pages
27+
GROUP BY
28+
client,
29+
technology
30+
),
31+
32+
technologies AS (
33+
SELECT
34+
name AS technology,
35+
description,
36+
STRING_AGG(DISTINCT category, ', ' ORDER BY category ASC) AS category,
37+
categories AS category_obj,
38+
NULL AS similar_technologies
39+
FROM ${ctx.ref('wappalyzer', 'technologies')},
40+
UNNEST(categories) AS category
41+
GROUP BY
42+
technology,
43+
description,
44+
categories
45+
),
46+
47+
total_pages AS (
48+
SELECT
49+
client,
50+
COUNT(DISTINCT root_page) AS origins
51+
FROM pages
52+
GROUP BY client
53+
)
54+
955
SELECT
1056
client,
11-
app AS technology,
57+
technology,
1258
description,
1359
category,
14-
SPLIT(category, ",") AS category_obj,
60+
category_obj,
61+
similar_technologies,
62+
origins
63+
FROM tech_origins
64+
INNER JOIN technologies
65+
USING(technology)
66+
67+
UNION ALL
68+
69+
SELECT
70+
client,
71+
'ALL' AS technology,
72+
NULL AS description,
73+
NULL AS category,
74+
NULL AS category_obj,
1575
NULL AS similar_technologies,
1676
origins
17-
FROM ${ctx.ref('core_web_vitals', 'technologies')}
18-
LEFT JOIN ${ctx.ref('wappalyzer', 'apps')}
19-
ON app = name
20-
WHERE date = '${pastMonth}' AND
21-
geo = 'ALL' AND
22-
rank = 'ALL'
23-
ORDER BY origins DESC
77+
FROM total_pages
2478
`)

‎definitions/output/wappalyzer/tech_detections.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ tech_deprecated_gone_origins AS (
112112
-- aggregation of technology adoption/deprecation metrics
113113
SELECT
114114
DATE('${constants.currentMonth}') AS date,
115-
COALESCE(before_summary.technology, tech_adopted_existing_origins.technology, tech_adopted_new_origins.technology, apps.name) AS technology,
115+
COALESCE(before_summary.technology, tech_adopted_existing_origins.technology, tech_adopted_new_origins.technology, technologies.name) AS technology,
116116
117117
-- origins summary
118118
0-COALESCE(total_origins_deprecated_existing, 0) AS total_origins_deprecated_existing,
@@ -139,6 +139,6 @@ LEFT JOIN tech_deprecated_existing_origins
139139
ON before_summary.technology = tech_deprecated_existing_origins.technology
140140
LEFT JOIN tech_deprecated_gone_origins
141141
ON before_summary.technology = tech_deprecated_gone_origins.technology
142-
FULL OUTER JOIN wappalyzer.apps
143-
ON before_summary.technology = apps.name
142+
FULL OUTER JOIN wappalyzer.technologies
143+
ON before_summary.technology = technologies.name
144144
`)

0 commit comments

Comments
 (0)