change pagestable to visitors and update clickhouse hostname column to array

db/clickhouse/migrations/08_update_hostname_view.sql

@@ -0,0 +1,253 @@
+-- create new hourly table
+CREATE TABLE umami.website_event_stats_hourly_new
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    region LowCardinality(String),
+    city String,
+    entry_url AggregateFunction(argMin, String, DateTime('UTC')),
+    exit_url AggregateFunction(argMax, String, DateTime('UTC')),
+    url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
+    url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
+    utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),
+    utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),
+    utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),
+    utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),
+    utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),
+    referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
+    page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
+    gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
+    twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    event_type UInt32,
+    event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
+    views SimpleAggregateFunction(sum, UInt64),
+    min_time SimpleAggregateFunction(min, DateTime('UTC')),
+    max_time SimpleAggregateFunction(max, DateTime('UTC')),
+    tag SimpleAggregateFunction(groupArrayArray, Array(String)),
+    distinct_id String,
+    created_at Datetime('UTC')
+)
+ENGINE = AggregatingMergeTree
+    PARTITION BY toYYYYMM(created_at)
+    ORDER BY (
+        website_id,
+        event_type,
+        toStartOfHour(created_at),
+        cityHash64(visit_id),
+        visit_id
+    )
+    SAMPLE BY cityHash64(visit_id);
+
+-- create view
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new
+TO umami.website_event_stats_hourly_new
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostnames as hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    utm_source,
+    utm_medium,
+    utm_campaign,
+    utm_content,
+    utm_term,
+    referrer_domain,
+    page_title,
+    gclid,
+    fbclid,
+    msclkid,
+    ttclid,
+    li_fat_id,
+    twclid,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    tag,
+    distinct_id,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
+    arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
+    arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
+    arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
+    arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
+    arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    arrayFilter(x -> x != '', groupArray(gclid)) gclid,
+    arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+    arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+    arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+    arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+    arrayFilter(x -> x != '', groupArray(twclid)) twclid,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    arrayFilter(x -> x != '', groupArray(tag)) tag,
+    distinct_id,
+    toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    event_type,
+    distinct_id,
+    timestamp);
+
+-- rename tables
+RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;
+RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;
+
+-- drop views
+DROP TABLE umami.website_event_stats_hourly_mv;
+DROP TABLE umami.website_event_stats_hourly_mv_new;
+
+-- recreate view
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
+TO umami.website_event_stats_hourly
+AS
+SELECT
+    website_id,
+    session_id,
+    visit_id,
+    hostnames as hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    entry_url,
+    exit_url,
+    url_paths as url_path,
+    url_query,
+    utm_source,
+    utm_medium,
+    utm_campaign,
+    utm_content,
+    utm_term,
+    referrer_domain,
+    page_title,
+    gclid,
+    fbclid,
+    msclkid,
+    ttclid,
+    li_fat_id,
+    twclid,
+    event_type,
+    event_name,
+    views,
+    min_time,
+    max_time,
+    tag,
+    distinct_id,
+    timestamp as created_at
+FROM (SELECT
+    website_id,
+    session_id,
+    visit_id,
+    arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    argMinState(url_path, created_at) entry_url,
+    argMaxState(url_path, created_at) exit_url,
+    arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+    arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+    arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
+    arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
+    arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
+    arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
+    arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
+    arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+    arrayFilter(x -> x != '', groupArray(gclid)) gclid,
+    arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+    arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+    arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+    arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+    arrayFilter(x -> x != '', groupArray(twclid)) twclid,
+    event_type,
+    if(event_type = 2, groupArray(event_name), []) event_name,
+    sumIf(1, event_type = 1) views,
+    min(created_at) min_time,
+    max(created_at) max_time,
+    arrayFilter(x -> x != '', groupArray(tag)) tag,
+    distinct_id,
+    toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+    session_id,
+    visit_id,
+    hostname,
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    country,
+    region,
+    city,
+    event_type,
+    distinct_id,
+    timestamp);

db/clickhouse/schema.sql

@@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourly
     website_id UUID,
     session_id UUID,
     visit_id UUID,
-    hostname LowCardinality(String),
+    hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
     browser LowCardinality(String),
     os LowCardinality(String),
     device LowCardinality(String),
@@ -143,7 +143,7 @@ SELECT
     website_id,
     session_id,
     visit_id,
-    hostname,
+    hostnames as hostname,
     browser,
     os,
     device,
@@ -181,7 +181,7 @@ FROM (SELECT
     website_id,
     session_id,
     visit_id,
-    hostname,
+    arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
     browser,
     os,
     device,
@@ -199,7 +199,7 @@ FROM (SELECT
     arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
     arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
     arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
-    arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+    arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
     arrayFilter(x -> x != '', groupArray(page_title)) page_title,
     arrayFilter(x -> x != '', groupArray(gclid)) gclid,
     arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,

src/components/metrics/PagesTable.tsx

@@ -62,7 +62,7 @@ export function PagesTable({ allowFilter, ...props }: PagesTableProps) {
       {...props}
       title={formatMessage(labels.pages)}
       type={view}
-      metric={formatMessage(labels.views)}
+      metric={formatMessage(labels.visitors)}
       dataFilter={emptyFilter}
       renderLabel={renderLink}
     >

src/lib/constants.ts

@@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';
 export const FILTER_PAGES = 'filter-pages';
 
 export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
-export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];
+export const EVENT_COLUMNS = [
+  'url',
+  'entry',
+  'exit',
+  'referrer',
+  'title',
+  'query',
+  'event',
+  'tag',
+  'host',
+];
 
 export const SESSION_COLUMNS = [
   'browser',
@@ -44,7 +54,6 @@ export const SESSION_COLUMNS = [
   'country',
   'city',
   'region',
-  'host',
 ];
 
 export const FILTER_GROUPS = {

src/queries/sql/getValues.ts

@@ -21,6 +21,12 @@ async function relationalQuery(
   const { rawQuery, getSearchSQL } = prisma;
   const params = {};
   let searchQuery = '';
+  let excludeDomain = '';
+
+  if (column === 'referrer_domain') {
+    excludeDomain = `and website_event.referrer_domain != website_event.hostname
+      and website_event.referrer_domain != ''`;
+  }
 
   if (search) {
     if (decodeURIComponent(search).includes(',')) {
@@ -49,6 +55,7 @@ async function relationalQuery(
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
       ${searchQuery}
+      ${excludeDomain}
     group by 1
     order by 2 desc
     limit 10
@@ -73,6 +80,11 @@ async function clickhouseQuery(
   const { rawQuery, getSearchSQL } = clickhouse;
   const params = {};
   let searchQuery = '';
+  let excludeDomain = '';
+
+  if (column === 'referrer_domain') {
+    excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
+  }
 
   if (search) {
     searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;
@@ -103,6 +115,7 @@ async function clickhouseQuery(
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       ${searchQuery}
+      ${excludeDomain}
     group by 1
     order by 2 desc
     limit 10

src/queries/sql/pageviews/getPageviewMetrics.ts

@@ -34,7 +34,7 @@ async function relationalQuery(
       ...filters,
       eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
     },
-    { joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' },
+    { joinSession: SESSION_COLUMNS.includes(type) },
   );
 
   let entryExitQuery = '';
@@ -66,7 +66,7 @@ async function relationalQuery(
   return rawQuery(
     `
     select ${column} x,
-      ${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y
+      count(distinct website_event.session_id) as y
     from website_event
     ${cohortQuery}
     ${joinSession}
@@ -126,7 +126,7 @@ async function clickhouseQuery(
 
     sql = `
     select ${column} x, 
-      ${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y
+      uniq(website_event.session_id) as y
     from website_event
     ${cohortQuery}
     ${entryExitQuery}
@@ -142,28 +142,27 @@ async function clickhouseQuery(
     `;
   } else {
     let groupByQuery = '';
-    let columnQuery = `arrayJoin(${column})`;
+    let columnQuery = `session_id s, arrayJoin(${column})`;
 
     if (column === 'referrer_domain') {
-      excludeDomain = `and t != hostname and t != ''`;
+      excludeDomain = `and t != ''`;
-      columnQuery = `session_id s, arrayJoin(${column})`;
     }
 
     if (type === 'entry') {
-      columnQuery = `visit_id x, argMinMerge(entry_url)`;
+      columnQuery = `session_id s, argMinMerge(entry_url)`;
     }
 
     if (type === 'exit') {
-      columnQuery = `visit_id x, argMaxMerge(exit_url)`;
+      columnQuery = `session_id s, argMaxMerge(exit_url)`;
     }
 
     if (type === 'entry' || type === 'exit') {
-      groupByQuery = 'group by x';
+      groupByQuery = 'group by s';
     }
 
     sql = `
     select g.t as x,
-      ${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y
+      uniq(s) as y
     from (
       select ${columnQuery} as t
       from website_event_stats_hourly website_event

src/queries/sql/sessions/getWebsiteSession.ts

@@ -17,7 +17,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
     select id,
       distinct_id as "distinctId",
       website_id as "websiteId",
-      hostname,
       browser,
       os,
       device,
@@ -37,7 +36,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
           session.distinct_id,
           website_event.visit_id,
           session.website_id,
-          website_event.hostname,
           session.browser,
           session.os,
           session.device,
@@ -54,8 +52,8 @@ async function relationalQuery(websiteId: string, sessionId: string) {
     join website_event on website_event.session_id = session.session_id
     where session.website_id = {{websiteId::uuid}}
       and session.session_id = {{sessionId::uuid}}
-    group by session.session_id, session.distinct_id, visit_id, session.website_id, website_event.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
+    group by session.session_id, session.distinct_id, visit_id, session.website_id, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
-    group by id, distinct_id, website_id, hostname, browser, os, device, screen, language, country, region, city;
+    group by id, distinct_id, website_id, browser, os, device, screen, language, country, region, city;
     `,
     { websiteId, sessionId },
   ).then(result => result?.[0]);
@@ -69,7 +67,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
     select id,
       websiteId,
       distinctId,
-      hostname,
       browser,
       os,
       device,
@@ -89,7 +86,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
               distinct_id as distinctId,
               visit_id,
               website_id as websiteId,
-              hostname,
               browser,
               os,
               device,
@@ -105,8 +101,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
         from website_event_stats_hourly
         where website_id = {websiteId:UUID}
           and session_id = {sessionId:UUID}
-        group by session_id, distinct_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, region, city) t
+        group by session_id, distinct_id, visit_id, website_id, browser, os, device, screen, language, country, region, city) t
-    group by id, websiteId, distinctId, hostname, browser, os, device, screen, language, country, region, city;
+    group by id, websiteId, distinctId, browser, os, device, screen, language, country, region, city;
     `,
     { websiteId, sessionId },
   ).then(result => result?.[0]);

src/queries/sql/sessions/getWebsiteSessions.ts

@@ -27,7 +27,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
     select
       session.session_id as "id",
       session.website_id as "websiteId",
-      website_event.hostname,
       session.browser,
       session.os,
       session.device,
@@ -58,7 +57,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
     }
     group by session.session_id, 
       session.website_id, 
-      website_event.hostname, 
       session.browser, 
       session.os, 
       session.device, 
@@ -84,7 +82,6 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
     select
       session_id as id,
       website_id as websiteId,
-      hostname,
       browser,
       os,
       device,
@@ -112,7 +109,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
            or (positionCaseInsensitive(device, {search:String}) > 0))`
         : ''
     }
-    group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city
+    group by session_id, website_id, browser, os, device, screen, language, country, region, city
     order by lastAt desc
     `,
     { ...params, search },