Tech report: Dedupe technology records (#60)

* versions

* tech filter

* new table with versions

* typo

* versions table

* fix

* no retries

* tech_report_* tables

* clusters renamed

* lint

* adjust export config

* fix clustering

* origin renamed

* deduplicated good_cwv

* include minor

* fix

* cleanup

* pattern fix

* tech detections only

* fix

* relaxed pattern

* remove hashing (#59)

* dedupe technologies

* cleanup

Author: max-ostapenko

definitions/output/reports/cwv_tech_categories.js

@@ -55,6 +55,14 @@ technology_stats AS (
   GROUP BY
     technology,
     categories
+),
+
+total_pages AS (
+  SELECT
+    client,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY client
 )
 
 SELECT
@@ -82,11 +90,5 @@ SELECT
     COALESCE(MAX(IF(client = 'mobile', origins, 0))) AS mobile
   ) AS origins,
   NULL AS technologies
-FROM (
-  SELECT
-    client,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY client
-)
+FROM total_pages
 `)

definitions/output/reports/tech_report_technologies.js

@@ -21,13 +21,22 @@ WITH pages AS (
 
 tech_origins AS (
   SELECT
-    client,
     technology,
-    COUNT(DISTINCT root_page) AS origins
-  FROM pages
-  GROUP BY
-    client,
-    technology
+    STRUCT(
+      MAX(IF(client = 'desktop', origins, 0)) AS desktop,
+      MAX(IF(client = 'mobile', origins, 0)) AS mobile
+    ) AS origins
+  FROM (
+    SELECT
+      client,
+      technology,
+      COUNT(DISTINCT root_page) AS origins
+    FROM pages
+    GROUP BY
+      client,
+      technology
+  )
+  GROUP BY technology
 ),
 
 technologies AS (
@@ -53,7 +62,6 @@ total_pages AS (
 )
 
 SELECT
-  client,
   technology,
   description,
   category,
@@ -66,11 +74,14 @@ USING(technology)
 UNION ALL
 
 SELECT
-  client,
   'ALL' AS technology,
   NULL AS description,
   NULL AS category,
   NULL AS category_obj,
-  origins
+  NULL AS similar_technologies,
+  STRUCT(
+    MAX(IF(client = 'desktop', origins, 0)) AS desktop,
+    MAX(IF(client = 'mobile', origins, 0)) AS mobile
+  ) AS origins
 FROM total_pages
 `)