Skip to content

Commit e8e6c86

Browse files
max-ostapenkomax-ostapenko
authored
Tech Report: Technologies - major.minor versions granularity (#48)
* versions * tech filter * new table with versions * typo * versions table * fix * no retries * tech_report_* tables * clusters renamed * lint * adjust export config * fix clustering * origin renamed * deduplicated good_cwv * include minor * fix * cleanup * pattern fix * tech detections only * fix * relaxed pattern * remove similar_technologies * cleanup
1 parent 65f3091 commit e8e6c86

19 files changed

+834
-13
lines changed

‎definitions/output/reports/cwv_tech_adoption.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ publish('cwv_tech_adoption', {
88
partitionBy: 'date',
99
clusterBy: ['rank', 'geo']
1010
},
11-
tags: ['crux_ready', 'tech_report']
11+
tags: ['crux_ready']
1212
}).preOps(ctx => `
1313
DELETE FROM ${ctx.self()}
1414
WHERE date = '${pastMonth}';

‎definitions/output/reports/cwv_tech_categories.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
33
publish('cwv_tech_categories', {
44
schema: 'reports',
55
type: 'table',
6-
tags: ['crux_ready', 'tech_report']
6+
tags: ['crux_ready']
77
}).query(ctx => `
88
/* {"dataform_trigger": "report_cwv_tech_complete", "name": "categories", "type": "dict"} */
99
WITH pages AS (

‎definitions/output/reports/cwv_tech_core_web_vitals.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ publish('cwv_tech_core_web_vitals', {
88
partitionBy: 'date',
99
clusterBy: ['rank', 'geo']
1010
},
11-
tags: ['crux_ready', 'tech_report']
11+
tags: ['tech_report']
1212
}).preOps(ctx => `
1313
CREATE TEMPORARY FUNCTION GET_VITALS(
1414
records ARRAY<STRUCT<

‎definitions/output/reports/cwv_tech_lighthouse.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ publish('cwv_tech_lighthouse', {
88
partitionBy: 'date',
99
clusterBy: ['rank', 'geo']
1010
},
11-
tags: ['crux_ready', 'tech_report']
11+
tags: ['crux_ready']
1212
}).preOps(ctx => `
1313
CREATE TEMPORARY FUNCTION GET_LIGHTHOUSE(
1414
records ARRAY<STRUCT<

‎definitions/output/reports/cwv_tech_page_weight.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ publish('cwv_tech_page_weight', {
88
partitionBy: 'date',
99
clusterBy: ['rank', 'geo']
1010
},
11-
tags: ['crux_ready', 'tech_report']
11+
tags: ['crux_ready']
1212
}).preOps(ctx => `
1313
CREATE TEMPORARY FUNCTION GET_PAGE_WEIGHT(
1414
records ARRAY<STRUCT<

‎definitions/output/reports/cwv_tech_technologies.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ const pastMonth = constants.fnPastMonth(constants.currentMonth)
33
publish('cwv_tech_technologies', {
44
schema: 'reports',
55
type: 'table',
6-
tags: ['crux_ready', 'tech_report']
6+
tags: ['crux_ready']
77
}).query(ctx => `
88
/* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
99
WITH pages AS (

‎definitions/output/reports/tech_crux.js

+295
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
const pastMonth = constants.fnPastMonth(constants.currentMonth)
2+
3+
publish('tech_crux', {
4+
schema: 'reports',
5+
type: 'incremental',
6+
protected: true,
7+
bigquery: {
8+
partitionBy: 'date',
9+
clusterBy: ['geo', 'client', 'rank', 'technology'],
10+
requirePartitionFilter: true
11+
},
12+
tags: ['tech_report'],
13+
dependOnDependencyAssertions: true
14+
}).preOps(ctx => `
15+
DELETE FROM ${ctx.self()}
16+
WHERE date = '${pastMonth}';
17+
18+
CREATE TEMP FUNCTION IS_GOOD(
19+
good FLOAT64,
20+
needs_improvement FLOAT64,
21+
poor FLOAT64
22+
) RETURNS BOOL AS (
23+
SAFE_DIVIDE(good, good + needs_improvement + poor) >= 0.75
24+
);
25+
26+
CREATE TEMP FUNCTION IS_NON_ZERO(
27+
good FLOAT64,
28+
needs_improvement FLOAT64,
29+
poor FLOAT64
30+
) RETURNS BOOL AS (
31+
good + needs_improvement + poor > 0
32+
);
33+
`).query(ctx => `
34+
WITH pages AS (
35+
SELECT
36+
client,
37+
page,
38+
root_page,
39+
technologies,
40+
summary,
41+
lighthouse
42+
FROM ${ctx.ref('crawl', 'pages')}
43+
WHERE
44+
date = '${pastMonth}'
45+
${constants.devRankFilter}
46+
),
47+
48+
geo_summary AS (
49+
SELECT
50+
\`chrome-ux-report\`.experimental.GET_COUNTRY(country_code) AS geo,
51+
rank,
52+
device,
53+
origin,
54+
avg_fcp,
55+
avg_fid,
56+
avg_inp,
57+
avg_lcp,
58+
avg_ttfb,
59+
fast_fcp,
60+
fast_fid,
61+
fast_inp,
62+
fast_lcp,
63+
fast_ttfb,
64+
slow_fcp,
65+
slow_fid,
66+
slow_inp,
67+
slow_lcp,
68+
slow_ttfb,
69+
small_cls,
70+
medium_cls,
71+
large_cls
72+
FROM ${ctx.ref('chrome-ux-report', 'materialized', 'country_summary')}
73+
WHERE
74+
yyyymm = CAST(FORMAT_DATE('%Y%m', '${pastMonth}') AS INT64) AND
75+
device IN ('desktop', 'phone')
76+
77+
UNION ALL
78+
79+
SELECT
80+
'ALL' AS geo,
81+
rank,
82+
device,
83+
origin,
84+
avg_fcp,
85+
avg_fid,
86+
avg_inp,
87+
avg_lcp,
88+
avg_ttfb,
89+
fast_fcp,
90+
fast_fid,
91+
fast_inp,
92+
fast_lcp,
93+
fast_ttfb,
94+
slow_fcp,
95+
slow_fid,
96+
slow_inp,
97+
slow_lcp,
98+
slow_ttfb,
99+
small_cls,
100+
medium_cls,
101+
large_cls
102+
FROM ${ctx.ref('chrome-ux-report', 'materialized', 'device_summary')}
103+
WHERE
104+
date = '${pastMonth}' AND
105+
device IN ('desktop', 'phone')
106+
),
107+
108+
crux AS (
109+
SELECT
110+
geo,
111+
CASE _rank
112+
WHEN 100000000 THEN 'ALL'
113+
WHEN 10000000 THEN 'Top 10M'
114+
WHEN 1000000 THEN 'Top 1M'
115+
WHEN 100000 THEN 'Top 100k'
116+
WHEN 10000 THEN 'Top 10k'
117+
WHEN 1000 THEN 'Top 1k'
118+
END AS rank,
119+
CONCAT(origin, '/') AS root_page,
120+
IF(device = 'desktop', 'desktop', 'mobile') AS client,
121+
122+
# CWV
123+
IS_NON_ZERO(fast_fid, avg_fid, slow_fid) AS any_fid,
124+
IS_GOOD(fast_fid, avg_fid, slow_fid) AS good_fid,
125+
IS_NON_ZERO(small_cls, medium_cls, large_cls) AS any_cls,
126+
IS_GOOD(small_cls, medium_cls, large_cls) AS good_cls,
127+
IS_NON_ZERO(fast_lcp, avg_lcp, slow_lcp) AS any_lcp,
128+
IS_GOOD(fast_lcp, avg_lcp, slow_lcp) AS good_lcp,
129+
IF('${pastMonth}' < '2024-01-01',
130+
(IS_GOOD(fast_fid, avg_fid, slow_fid) OR fast_fid IS NULL) AND
131+
IS_GOOD(small_cls, medium_cls, large_cls) AND
132+
IS_GOOD(fast_lcp, avg_lcp, slow_lcp),
133+
(IS_GOOD(fast_inp, avg_inp, slow_inp) OR fast_inp IS NULL) AND
134+
IS_GOOD(small_cls, medium_cls, large_cls) AND
135+
IS_GOOD(fast_lcp, avg_lcp, slow_lcp)
136+
) AS good_cwv,
137+
138+
# WV
139+
IS_NON_ZERO(fast_fcp, avg_fcp, slow_fcp) AS any_fcp,
140+
IS_GOOD(fast_fcp, avg_fcp, slow_fcp) AS good_fcp,
141+
IS_NON_ZERO(fast_ttfb, avg_ttfb, slow_ttfb) AS any_ttfb,
142+
IS_GOOD(fast_ttfb, avg_ttfb, slow_ttfb) AS good_ttfb,
143+
IS_NON_ZERO(fast_inp, avg_inp, slow_inp) AS any_inp,
144+
IS_GOOD(fast_inp, avg_inp, slow_inp) AS good_inp
145+
FROM geo_summary,
146+
UNNEST([1000, 10000, 100000, 1000000, 10000000, 100000000]) AS _rank
147+
WHERE rank <= _rank
148+
),
149+
150+
technologies AS (
151+
SELECT
152+
tech.technology,
153+
REGEXP_EXTRACT(version, r'\\d+(?:\\.\\d+)?') AS version,
154+
client,
155+
page
156+
FROM pages,
157+
UNNEST(technologies) AS tech,
158+
UNNEST(tech.info) AS version
159+
WHERE
160+
tech.technology IS NOT NULL AND
161+
REGEXP_EXTRACT(version, r'\\d+(?:\\.\\d+)?') IS NOT NULL
162+
163+
UNION ALL
164+
165+
SELECT
166+
tech.technology,
167+
'ALL' AS version,
168+
client,
169+
page
170+
FROM pages,
171+
UNNEST(technologies) AS tech
172+
WHERE
173+
tech.technology IS NOT NULL
174+
175+
176+
UNION ALL
177+
178+
SELECT
179+
'ALL' AS technology,
180+
'ALL' AS version,
181+
client,
182+
page
183+
FROM pages
184+
),
185+
186+
categories AS (
187+
SELECT
188+
tech.technology,
189+
ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category
190+
FROM pages,
191+
UNNEST(technologies) AS tech,
192+
UNNEST(tech.categories) AS category
193+
GROUP BY technology
194+
195+
UNION ALL
196+
197+
SELECT
198+
'ALL' AS technology,
199+
ARRAY_TO_STRING(ARRAY_AGG(DISTINCT category IGNORE NULLS ORDER BY category), ', ') AS category
200+
FROM pages,
201+
UNNEST(technologies) AS tech,
202+
UNNEST(tech.categories) AS category
203+
),
204+
205+
lab_metrics AS (
206+
SELECT
207+
client,
208+
page,
209+
root_page,
210+
SAFE.INT64(summary.bytesTotal) AS bytesTotal,
211+
SAFE.INT64(summary.bytesJS) AS bytesJS,
212+
SAFE.INT64(summary.bytesImg) AS bytesImg,
213+
SAFE.FLOAT64(lighthouse.categories.accessibility.score) AS accessibility,
214+
SAFE.FLOAT64(lighthouse.categories['best-practices'].score) AS best_practices,
215+
SAFE.FLOAT64(lighthouse.categories.performance.score) AS performance,
216+
SAFE.FLOAT64(lighthouse.categories.pwa.score) AS pwa,
217+
SAFE.FLOAT64(lighthouse.categories.seo.score) AS seo
218+
FROM pages
219+
),
220+
221+
lab_data AS (
222+
SELECT
223+
client,
224+
root_page,
225+
technology,
226+
version,
227+
ANY_VALUE(category) AS category,
228+
AVG(bytesTotal) AS bytesTotal,
229+
AVG(bytesJS) AS bytesJS,
230+
AVG(bytesImg) AS bytesImg,
231+
AVG(accessibility) AS accessibility,
232+
AVG(best_practices) AS best_practices,
233+
AVG(performance) AS performance,
234+
AVG(pwa) AS pwa,
235+
AVG(seo) AS seo
236+
FROM lab_metrics
237+
INNER JOIN technologies
238+
USING (client, page)
239+
INNER JOIN categories
240+
USING (technology)
241+
GROUP BY
242+
client,
243+
root_page,
244+
technology,
245+
version
246+
)
247+
248+
SELECT
249+
DATE('${pastMonth}') AS date,
250+
geo,
251+
client,
252+
rank,
253+
technology,
254+
version,
255+
COUNT(DISTINCT root_page) AS origins,
256+
257+
# CrUX data
258+
COUNTIF(good_fid) AS origins_with_good_fid,
259+
COUNTIF(good_cls) AS origins_with_good_cls,
260+
COUNTIF(good_lcp) AS origins_with_good_lcp,
261+
COUNTIF(good_fcp) AS origins_with_good_fcp,
262+
COUNTIF(good_ttfb) AS origins_with_good_ttfb,
263+
COUNTIF(good_inp) AS origins_with_good_inp,
264+
COUNTIF(any_fid) AS origins_with_any_fid,
265+
COUNTIF(any_cls) AS origins_with_any_cls,
266+
COUNTIF(any_lcp) AS origins_with_any_lcp,
267+
COUNTIF(any_fcp) AS origins_with_any_fcp,
268+
COUNTIF(any_ttfb) AS origins_with_any_ttfb,
269+
COUNTIF(any_inp) AS origins_with_any_inp,
270+
COUNTIF(good_cwv) AS origins_with_good_cwv,
271+
COUNTIF(any_lcp AND any_cls) AS origins_eligible_for_cwv,
272+
SAFE_DIVIDE(COUNTIF(good_cwv), COUNTIF(any_lcp AND any_cls)) AS pct_eligible_origins_with_good_cwv,
273+
274+
# Lighthouse data
275+
SAFE_CAST(APPROX_QUANTILES(accessibility, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_accessibility,
276+
SAFE_CAST(APPROX_QUANTILES(best_practices, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_best_practices,
277+
SAFE_CAST(APPROX_QUANTILES(performance, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_performance,
278+
SAFE_CAST(APPROX_QUANTILES(pwa, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_pwa,
279+
SAFE_CAST(APPROX_QUANTILES(seo, 1000)[OFFSET(500)] AS NUMERIC) AS median_lighthouse_score_seo,
280+
281+
# Page weight stats
282+
SAFE_CAST(APPROX_QUANTILES(bytesTotal, 1000)[OFFSET(500)] AS INT64) AS median_bytes_total,
283+
SAFE_CAST(APPROX_QUANTILES(bytesJS, 1000)[OFFSET(500)] AS INT64) AS median_bytes_js,
284+
SAFE_CAST(APPROX_QUANTILES(bytesImg, 1000)[OFFSET(500)] AS INT64) AS median_bytes_image
285+
286+
FROM lab_data
287+
INNER JOIN crux
288+
USING (client, root_page)
289+
GROUP BY
290+
geo,
291+
client,
292+
rank,
293+
technology,
294+
version
295+
`)

‎definitions/output/reports/tech_report_adoption.js

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const pastMonth = constants.fnPastMonth(constants.currentMonth)
2+
3+
publish('tech_report_adoption', {
4+
schema: 'reports',
5+
type: 'incremental',
6+
protected: true,
7+
bigquery: {
8+
partitionBy: 'date',
9+
clusterBy: ['rank', 'geo']
10+
},
11+
tags: ['tech_report']
12+
}).preOps(ctx => `
13+
DELETE FROM ${ctx.self()}
14+
WHERE date = '${pastMonth}';
15+
`).query(ctx => `
16+
/* {"dataform_trigger": "tech_report_complete", "date": "${pastMonth}", "name": "adoption", "type": "report"} */
17+
SELECT
18+
date,
19+
geo,
20+
rank,
21+
technology,
22+
version,
23+
STRUCT(
24+
MAX(IF(client = 'desktop', origins, 0)) AS desktop,
25+
MAX(IF(client = 'mobile', origins, 0)) AS mobile
26+
) AS adoption
27+
FROM ${ctx.ref('reports', 'tech_crux')}
28+
WHERE date = '${pastMonth}'
29+
GROUP BY
30+
date,
31+
geo,
32+
rank,
33+
technology,
34+
version
35+
`)

0 commit comments

Comments
 (0)