Add UTM params support

Author: Dmytro Radchuk

dashboard/components/TopSourcesWidget/TopSourcesWidget.tsx

@@ -1,15 +1,27 @@
-import { BarList } from '@tremor/react'
+import { BarList, Flex, SelectBox, SelectBoxItem } from '@tremor/react'
 import Widget from '../Widget'
 import useTopSources from '../../lib/hooks/use-top-sources'
+import { OptionType } from '../../lib/types/options'
+import { UTMFilter } from '../../lib/types/top-sources'
 import { formatNumber } from '../../lib/utils'
 import { useMemo } from 'react'
 
+const utmFilterOptions: OptionType<UTMFilter>[] = [
+  { text: 'All', value: UTMFilter.All },
+  { text: 'UTM Medium', value: UTMFilter.Medium },
+  { text: 'UTM Source', value: UTMFilter.Source },
+  { text: 'UTM Campaign', value: UTMFilter.Campaign },
+  { text: 'UTM Content', value: UTMFilter.Content },
+  { text: 'UTM Term', value: UTMFilter.Term },
+]
+
 export default function TopSourcesWidget() {
-  const { data, status, warning } = useTopSources()
+  const { utmFilter, setUtmFilter, data, status, warning } = useTopSources()
+
   const chartData = useMemo(
     () =>
       (data?.data ?? []).map(d => ({
-        name: d.referrer,
+        name: (d.referrer || d.utm_filter) as string,
         value: d.visits,
         href: d.href,
       })),
@@ -18,7 +30,18 @@ export default function TopSourcesWidget() {
 
   return (
     <Widget>
-      <Widget.Title>Top Sources</Widget.Title>
+      <Flex justifyContent="justify-between">
+        <Widget.Title>Top Sources</Widget.Title>
+        <SelectBox
+          maxWidth="max-w-fit"
+          value={utmFilter}
+          onValueChange={setUtmFilter}
+        >
+          {utmFilterOptions.map(({ value, text }) => (
+            <SelectBoxItem key={value} value={value} text={text} />
+          ))}
+        </SelectBox>
+      </Flex>
       <Widget.Content
         status={status}
         noData={!chartData?.length}

dashboard/lib/hooks/use-top-sources.ts

@@ -1,26 +1,31 @@
+import { useState } from 'react'
 import { queryPipe } from '../api'
-import { TopSource, TopSources } from '../types/top-sources'
+import { TopSource, TopSources, UTMFilter } from '../types/top-sources'
 import useDateFilter from './use-date-filter'
 import useQuery from './use-query'
 
 async function getTopSources(
   date_from?: string,
-  date_to?: string
+  date_to?: string,
+  utm_filter?: UTMFilter
 ): Promise<TopSources> {
   const { data: queryData } = await queryPipe<TopSource>('top_sources', {
     limit: 8,
     date_from,
     date_to,
+    utm_filter,
   })
 
   const data: TopSource[] = [...queryData]
     .sort((a, b) => b.visits - a.visits)
-    .map(({ referrer, visits }) => ({
-      referrer: referrer || 'Direct',
+    .map(({ referrer, utm_filter, visits }) => ({
+      referrer: referrer || utm_filter || 'Direct',
       href: referrer ? `https://${referrer}` : undefined,
       visits,
     }))
-  const refs = data.map(({ referrer }) => referrer)
+  const refs = data.map(
+    ({ referrer, utm_filter }) => (referrer ?? utm_filter) as string
+  )
   const visits = data.map(({ visits }) => visits)
 
   return {
@@ -32,5 +37,16 @@ async function getTopSources(
 
 export default function useTopSources() {
   const { startDate, endDate } = useDateFilter()
-  return useQuery([startDate, endDate, 'topSources'], getTopSources)
+  const [utmFilter, setUtmFilter] = useState<UTMFilter>(UTMFilter.All)
+  const query = useQuery(
+    [
+      startDate,
+      endDate,
+      utmFilter === UTMFilter.All ? undefined : utmFilter,
+      'topSources',
+    ],
+    getTopSources
+  )
+
+  return { utmFilter, setUtmFilter, ...query }
 }

dashboard/lib/types/top-sources.ts

@@ -1,5 +1,6 @@
 export type TopSource = {
-  referrer: string
+  referrer?: string
+  utm_filter?: string
   visits: number
   href?: string
 }
@@ -9,3 +10,12 @@ export type TopSources = {
   refs: string[]
   visits: number[]
 }
+
+export enum UTMFilter {
+  All = 'all',
+  Source = 'utm_source',
+  Medium = 'utm_medium',
+  Campaign = 'utm_campaign',
+  Term = 'utm_term',
+  Content = 'utm_content',
+}

middleware/src/index.js

@@ -1,6 +1,7 @@
 (function(){
     const timezones = {"Asia/Barnaul":"RU","Africa/Nouakchott":"MR","Africa/Lusaka":"ZM","Asia/Pyongyang":"KP","Europe/Bratislava":"SK","America/Belize":"BZ","America/Maceio":"BR","Pacific/Chuuk":"FM","Indian/Comoro":"KM","Pacific/Palau":"PW","Asia/Jakarta":"ID","Africa/Windhoek":"NA","America/Chihuahua":"MX","America/Nome":"US","Africa/Mbabane":"SZ","Africa/Porto-Novo":"BJ","Europe/San_Marino":"SM","Pacific/Fakaofo":"TK","America/Denver":"US","Europe/Belgrade":"RS","America/Indiana/Tell_City":"US","America/Fortaleza":"BR","America/Halifax":"CA","Europe/Bucharest":"RO","America/Indiana/Petersburg":"US","Europe/Kirov":"RU","Europe/Athens":"GR","America/Argentina/Ushuaia":"AR","Europe/Monaco":"MC","Europe/Vilnius":"LT","Europe/Copenhagen":"DK","Pacific/Kanton":"KI","America/Caracas":"VE","Asia/Almaty":"KZ","Europe/Paris":"FR","Africa/Blantyre":"MW","Asia/Muscat":"OM","America/North_Dakota/Beulah":"US","America/Matamoros":"MX","Asia/Irkutsk":"RU","America/Costa_Rica":"CR","America/Araguaina":"BR","Atlantic/Canary":"ES","America/Santo_Domingo":"DO","America/Vancouver":"CA","Africa/Addis_Ababa":"ET","Africa/Accra":"GH","Pacific/Kwajalein":"MH","Asia/Baghdad":"IQ","Australia/Adelaide":"AU","Australia/Hobart":"AU","America/Guayaquil":"EC","America/Argentina/Tucuman":"AR","Australia/Lindeman":"AU","America/New_York":"US","Pacific/Fiji":"FJ","America/Antigua":"AG","Africa/Casablanca":"MA","America/Paramaribo":"SR","Africa/Cairo":"EG","America/Cayenne":"GF","America/Detroit":"US","Antarctica/Syowa":"AQ","Africa/Douala":"CM","America/Argentina/La_Rioja":"AR","Africa/Lagos":"NG","America/St_Barthelemy":"BL","Asia/Nicosia":"CY","Asia/Macau":"MO","Europe/Riga":"LV","Asia/Ashgabat":"TM","Indian/Antananarivo":"MG","America/Argentina/San_Juan":"AR","Asia/Aden":"YE","Asia/Tomsk":"RU","America/Asuncion":"PY","Pacific/Bougainville":"PG","Asia/Vientiane":"LA","America/Mazatlan":"MX","Africa/Luanda":"AO","Europe/Oslo":"NO","Africa/Kinshasa":"CD","Europe/Warsaw":"PL","America/Grand_Turk":"TC","Asia/Seoul":"KR","Africa/Tripoli":"LY","America/St_Thomas":"VI","Asia/Kathmandu":"NP","Pacific/Pitcairn":"PN","Pacific/Nauru":"NR","America/Curacao":"CW","Asia/Kabul":"AF","Pacific/Tongatapu":"TO","Europe/Simferopol":"UA","Asia/Ust-Nera":"RU","Africa/Mogadishu":"SO","Indian/Mayotte":"YT","Pacific/Niue":"NU","America/Thunder_Bay":"CA","Atlantic/Azores":"PT","Pacific/Gambier":"PF","Europe/Stockholm":"SE","Africa/Libreville":"GA","America/Punta_Arenas":"CL","America/Guatemala":"GT","America/Noronha":"BR","Europe/Helsinki":"FI","Asia/Gaza":"PS","Pacific/Kosrae":"FM","America/Aruba":"AW","America/Nassau":"BS","Asia/Choibalsan":"MN","America/Winnipeg":"CA","America/Anguilla":"AI","Asia/Thimphu":"BT","Asia/Beirut":"LB","Atlantic/Faroe":"FO","Europe/Berlin":"DE","Europe/Amsterdam":"NL","Pacific/Honolulu":"US","America/Regina":"CA","America/Scoresbysund":"GL","Europe/Vienna":"AT","Europe/Tirane":"AL","Africa/El_Aaiun":"EH","America/Creston":"CA","Asia/Qostanay":"KZ","Asia/Ho_Chi_Minh":"VN","Europe/Samara":"RU","Europe/Rome":"IT","Australia/Eucla":"AU","America/El_Salvador":"SV","America/Chicago":"US","Africa/Abidjan":"CI","Asia/Kamchatka":"RU","Pacific/Tarawa":"KI","America/Santiago":"CL","America/Bahia":"BR","Indian/Christmas":"CX","Asia/Atyrau":"KZ","Asia/Dushanbe":"TJ","Europe/Ulyanovsk":"RU","America/Yellowknife":"CA","America/Recife":"BR","Australia/Sydney":"AU","America/Fort_Nelson":"CA","Pacific/Efate":"VU","Europe/Saratov":"RU","Africa/Banjul":"GM","Asia/Omsk":"RU","Europe/Ljubljana":"SI","Europe/Budapest":"HU","Europe/Astrakhan":"RU","America/Argentina/Buenos_Aires":"AR","Pacific/Chatham":"NZ","America/Argentina/Salta":"AR","Africa/Niamey":"NE","Asia/Pontianak":"ID","Indian/Reunion":"RE","Asia/Hong_Kong":"HK","Antarctica/McMurdo":"AQ","Africa/Malabo":"GQ","America/Los_Angeles":"US","America/Argentina/Cordoba":"AR","Pacific/Pohnpei":"FM","America/Tijuana":"MX","America/Campo_Grande":"BR","America/Dawson_Creek":"CA","Asia/Novosibirsk":"RU","Pacific/Pago_Pago":"AS","Asia/Jerusalem":"IL","Europe/Sarajevo":"BA","Africa/Freetown":"SL","Asia/Yekaterinburg":"RU","America/Juneau":"US","Africa/Ouagadougou":"BF","Africa/Monrovia":"LR","Europe/Kiev":"UA","America/Argentina/San_Luis":"AR","Asia/Tokyo":"JP","Asia/Qatar":"QA","America/La_Paz":"BO","America/Bogota":"CO","America/Thule":"GL","Asia/Manila":"PH","Asia/Hovd":"MN","Asia/Tehran":"IR","Atlantic/Madeira":"PT","America/Metlakatla":"US","Europe/Vatican":"VA","Asia/Bishkek":"KG","Asia/Dili":"TL","Antarctica/Palmer":"AQ","Atlantic/Cape_Verde":"CV","Indian/Chagos":"IO","America/Kentucky/Monticello":"US","Africa/Algiers":"DZ","Africa/Maseru":"LS","Asia/Kuala_Lumpur":"MY","Africa/Khartoum":"SD","America/Argentina/Rio_Gallegos":"AR","America/Blanc-Sablon":"CA","Africa/Maputo":"MZ","America/Tortola":"VG","Atlantic/Bermuda":"BM","America/Argentina/Catamarca":"AR","America/Cayman":"KY","America/Puerto_Rico":"PR","Pacific/Majuro":"MH","Europe/Busingen":"DE","Pacific/Midway":"UM","Indian/Cocos":"CC","Asia/Singapore":"SG","America/Boise":"US","America/Nuuk":"GL","America/Goose_Bay":"CA","Australia/Broken_Hill":"AU","Africa/Dar_es_Salaam":"TZ","Africa/Asmara":"ER","Asia/Samarkand":"UZ","Asia/Tbilisi":"GE","America/Argentina/Jujuy":"AR","America/Indiana/Winamac":"US","America/Porto_Velho":"BR","Asia/Magadan":"RU","Europe/Zaporozhye":"UA","Antarctica/Casey":"AQ","Asia/Shanghai":"CN","Pacific/Norfolk":"NF","Europe/Guernsey":"GG","Australia/Brisbane":"AU","Antarctica/DumontDUrville":"AQ","America/Havana":"CU","America/Atikokan":"CA","America/Mexico_City":"MX","America/Rankin_Inlet":"CA","America/Cuiaba":"BR","America/Resolute":"CA","Africa/Ceuta":"ES","Arctic/Longyearbyen":"SJ","Pacific/Guam":"GU","Asia/Damascus":"SY","Asia/Colombo":"LK","Asia/Yerevan":"AM","America/Montserrat":"MS","America/Belem":"BR","Europe/Kaliningrad":"RU","Atlantic/South_Georgia":"GS","Asia/Tashkent":"UZ","Asia/Kolkata":"IN","America/St_Johns":"CA","Asia/Srednekolymsk":"RU","Asia/Yakutsk":"RU","Europe/Prague":"CZ","Africa/Djibouti":"DJ","Asia/Dubai":"AE","Europe/Uzhgorod":"UA","America/Edmonton":"CA","Asia/Famagusta":"CY","America/Indiana/Knox":"US","Asia/Hebron":"PS","Asia/Taipei":"TW","Europe/London":"GB","Africa/Dakar":"SN","Australia/Darwin":"AU","America/Glace_Bay":"CA","Antarctica/Vostok":"AQ","America/Indiana/Vincennes":"US","America/Nipigon":"CA","Asia/Kuwait":"KW","Pacific/Guadalcanal":"SB","America/Toronto":"CA","Africa/Gaborone":"BW","Africa/Bujumbura":"BI","Africa/Lubumbashi":"CD","America/Merida":"MX","America/Marigot":"MF","Europe/Zagreb":"HR","Pacific/Easter":"CL","America/Santarem":"BR","Pacific/Noumea":"NC","America/Sitka":"US","Atlantic/Stanley":"FK","Pacific/Funafuti":"TV","America/Iqaluit":"CA","America/Rainy_River":"CA","America/Anchorage":"US","America/Lima":"PE","Asia/Baku":"AZ","America/Indiana/Vevay":"US","Asia/Ulaanbaatar":"MN","America/Managua":"NI","Asia/Krasnoyarsk":"RU","Asia/Qyzylorda":"KZ","America/Eirunepe":"BR","Europe/Podgorica":"ME","Europe/Chisinau":"MD","Europe/Mariehamn":"AX","Europe/Volgograd":"RU","Africa/Nairobi":"KE","Europe/Isle_of_Man":"IM","America/Menominee":"US","Africa/Harare":"ZW","Asia/Anadyr":"RU","America/Moncton":"CA","Indian/Maldives":"MV","America/Whitehorse":"CA","Antarctica/Mawson":"AQ","Europe/Madrid":"ES","America/Argentina/Mendoza":"AR","America/Manaus":"BR","Africa/Bangui":"CF","Indian/Mauritius":"MU","Africa/Tunis":"TN","Australia/Lord_Howe":"AU","America/Kentucky/Louisville":"US","America/North_Dakota/Center":"US","Asia/Novokuznetsk":"RU","Asia/Makassar":"ID","America/Port_of_Spain":"TT","America/Bahia_Banderas":"MX","Pacific/Auckland":"NZ","America/Sao_Paulo":"BR","Asia/Dhaka":"BD","America/Pangnirtung":"CA","Europe/Dublin":"IE","Asia/Brunei":"BN","Africa/Brazzaville":"CG","America/Montevideo":"UY","America/Jamaica":"JM","America/Indiana/Indianapolis":"US","America/Kralendijk":"BQ","Europe/Gibraltar":"GI","Pacific/Marquesas":"PF","Pacific/Apia":"WS","Europe/Jersey":"JE","America/Phoenix":"US","Africa/Ndjamena":"TD","Asia/Karachi":"PK","Africa/Kampala":"UG","Asia/Sakhalin":"RU","America/Martinique":"MQ","Europe/Moscow":"RU","Africa/Conakry":"GN","America/Barbados":"BB","Africa/Lome":"TG","America/Ojinaga":"MX","America/Tegucigalpa":"HN","Asia/Bangkok":"TH","Africa/Johannesburg":"ZA","Europe/Vaduz":"LI","Africa/Sao_Tome":"ST","America/Cambridge_Bay":"CA","America/Lower_Princes":"SX","America/Miquelon":"PM","America/St_Kitts":"KN","Australia/Melbourne":"AU","Europe/Minsk":"BY","Asia/Vladivostok":"RU","Europe/Sofia":"BG","Antarctica/Davis":"AQ","Pacific/Galapagos":"EC","America/North_Dakota/New_Salem":"US","Asia/Amman":"JO","Pacific/Wallis":"WF","America/Hermosillo":"MX","Pacific/Kiritimati":"KI","Antarctica/Macquarie":"AU","America/Guyana":"GY","Asia/Riyadh":"SA","Pacific/Tahiti":"PF","America/St_Vincent":"VC","America/Cancun":"MX","America/Grenada":"GD","Pacific/Wake":"UM","America/Dawson":"CA","Europe/Brussels":"BE","Indian/Kerguelen":"TF","America/Yakutat":"US","Indian/Mahe":"SC","Atlantic/Reykjavik":"IS","America/Panama":"PA","America/Guadeloupe":"GP","Europe/Malta":"MT","Antarctica/Troll":"AQ","Asia/Jayapura":"ID","Asia/Bahrain":"BH","Asia/Chita":"RU","Europe/Tallinn":"EE","Asia/Khandyga":"RU","America/Rio_Branco":"BR","Atlantic/St_Helena":"SH","Africa/Juba":"SS","America/Adak":"US","Pacific/Saipan":"MP","America/St_Lucia":"LC","America/Inuvik":"CA","Europe/Luxembourg":"LU","Africa/Bissau":"GW","Asia/Oral":"KZ","America/Boa_Vista":"BR","Europe/Skopje":"MK","America/Port-au-Prince":"HT","Pacific/Port_Moresby":"PG","Europe/Andorra":"AD","America/Indiana/Marengo":"US","Africa/Kigali":"RW","Africa/Bamako":"ML","America/Dominica":"DM","Asia/Aqtobe":"KZ","Europe/Istanbul":"TR","Pacific/Rarotonga":"CK","America/Danmarkshavn":"GL","Europe/Zurich":"CH","Asia/Yangon":"MM","America/Monterrey":"MX","Europe/Lisbon":"PT","Asia/Kuching":"MY","Antarctica/Rothera":"AQ","Australia/Perth":"AU","Asia/Phnom_Penh":"KH","America/Swift_Current":"CA","Asia/Aqtau":"KZ","Asia/Urumqi":"CN"};
     const COOKIE_NAME = 'session-id';
+    const UTM_PARAMS = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content']
     let DATASOURCE = 'analytics_events';
 
     let proxy, token, host;
@@ -22,7 +23,7 @@
 
     function _getSessionId() {
         let cookie = {};
-        document.cookie.split(';').forEach(function(el) {
+        document.cookie.split(';').forEach(el => {
         let [key,value] = el.split('=');
         cookie[key.trim()] = value;
         })
@@ -119,6 +120,9 @@
             // ignore error
         }
 
+        const searchParams = new URLSearchParams(window.location.search);
+        const utmValues = UTM_PARAMS.map(param => ({ [param]: searchParams.get(param) ?? '' }))
+
         // Wait a bit for SPA routers
         setTimeout(() => {
             _sendEvent('page_hit',{
@@ -127,7 +131,8 @@
                 location: country,
                 referrer: document.referrer,
                 pathname: window.location.pathname,
-                href: window.location.href
+                href: window.location.href,
+                ...utmValues
             });
         }, 300);
     }

tinybird/endpoints/analytics_hits.pipe

@@ -18,6 +18,11 @@ SQL >
       JSONExtractString(payload, 'referrer') as referrer,
       JSONExtractString(payload, 'pathname') as pathname,
       JSONExtractString(payload, 'href') as href,
+      JSONExtractString(payload, 'utm_source') as utm_source,
+      JSONExtractString(payload, 'utm_medium') as utm_medium,
+      JSONExtractString(payload, 'utm_campaign') as utm_campaign,
+      JSONExtractString(payload, 'utm_term') as utm_term,
+      JSONExtractString(payload, 'utm_content') as utm_content,
       lower(JSONExtractString(payload, 'user-agent')) as user_agent
     FROM
       analytics_events
@@ -34,6 +39,11 @@ SQL >
       referrer,
       pathname,
       href,
+      utm_source,
+      utm_medium,
+      utm_campaign,
+      utm_term,
+      utm_content,
       case
         when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') then 'bot'
         when match(user_agent, 'android') then 'mobile-android'

tinybird/endpoints/top_sources.pipe

@@ -12,7 +12,11 @@ DESCRIPTION >
 SQL >
     %
     select
-      domainWithoutWWW(referrer) as referrer,
+      {% if defined(utm_filter) %}
+        {{column(utm_filter)}} as utm_filter,
+      {% else %}
+        domainWithoutWWW(referrer) as referrer,
+      {% end %}
       uniqMerge(visits) as visits,
       countMerge(hits) as hits
     from
@@ -28,8 +32,15 @@ SQL >
       {% else %}
         and date <= today()
       {% end %}
+      {% if defined(utm_filter) %}
+        and {{column(utm_filter)}} != ''
+      {% end %}
     group by
-      referrer
+      {% if defined(utm_filter) %}
+        {{column(utm_filter)}}
+      {% else %}
+        referrer
+      {% end %}
     order by
       visits desc
     limit {{Int32(skip, 0)}},{{Int32(limit, 50)}}

tinybird/pipes/analytics_sources.pipe

@@ -11,7 +11,12 @@ SQL >
       location,
       referrer,
       uniqState(session_id) AS visits,
-      countState() AS hits
+      countState() AS hits,
+      utm_source,
+      utm_medium,
+      utm_campaign,
+      utm_term,
+      utm_content
     from
       analytics_hits
     where
@@ -21,7 +26,12 @@ SQL >
       device,
       browser,
       location,
-      referrer
+      referrer,
+      utm_source,
+      utm_medium,
+      utm_campaign,
+      utm_term,
+      utm_content
 
 TYPE materialized
 DATASOURCE analytics_sources_mv