watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-04 04:37:11 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

change pagestable to visitors and update clickhouse hostname column to array

Browse source
This commit is contained in:
Francis Cao 2025-07-13 22:44:09 -07:00
parent 195619aeed
commit 2dcb9e21bd
8 changed files with 296 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

253
db/clickhouse/migrations/08_update_hostname_view.sql Normal file
View file

@ -0,0 +1,253 @@
-- create new hourly table
CREATE TABLE umami.website_event_stats_hourly_new
(
website_id UUID,
session_id UUID,
visit_id UUID,
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
region LowCardinality(String),
city String,
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),
referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
event_type UInt32,
event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
views SimpleAggregateFunction(sum, UInt64),
min_time SimpleAggregateFunction(min, DateTime('UTC')),
max_time SimpleAggregateFunction(max, DateTime('UTC')),
tag SimpleAggregateFunction(groupArrayArray, Array(String)),
distinct_id String,
created_at Datetime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (
website_id,
event_type,
toStartOfHour(created_at),
cityHash64(visit_id),
visit_id
)
SAMPLE BY cityHash64(visit_id);
-- create view
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new
TO umami.website_event_stats_hourly_new
AS
SELECT
website_id,
session_id,
visit_id,
hostnames as hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
distinct_id,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
distinct_id,
timestamp);
-- rename tables
RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;
RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;
-- drop views
DROP TABLE umami.website_event_stats_hourly_mv;
DROP TABLE umami.website_event_stats_hourly_mv_new;
-- recreate view
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
session_id,
visit_id,
hostnames as hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
distinct_id,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
distinct_id,
timestamp);

8
db/clickhouse/schema.sql
View file

@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourly
website_id UUID,
session_id UUID,
visit_id UUID,
hostname LowCardinality(String),
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
@ -143,7 +143,7 @@ SELECT
website_id,
session_id,
visit_id,
hostname,
hostnames as hostname,
browser,
os,
device,
@ -181,7 +181,7 @@ FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
@ -199,7 +199,7 @@ FROM (SELECT
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,

2
src/components/metrics/PagesTable.tsx
View file

@ -62,7 +62,7 @@ export function PagesTable({ allowFilter, ...props }: PagesTableProps) {
{...props}
title={formatMessage(labels.pages)}
type={view}
metric={formatMessage(labels.views)}
metric={formatMessage(labels.visitors)}
dataFilter={emptyFilter}
renderLabel={renderLink}
>

13
src/lib/constants.ts
View file

@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];
export const EVENT_COLUMNS = [
'url',
'entry',
'exit',
'referrer',
'title',
'query',
'event',
'tag',
'host',
];
export const SESSION_COLUMNS = [
'browser',
@ -44,7 +54,6 @@ export const SESSION_COLUMNS = [
'country',
'city',
'region',
'host',
];
export const FILTER_GROUPS = {

13
src/queries/sql/getValues.ts
View file

@ -21,6 +21,12 @@ async function relationalQuery(
const { rawQuery, getSearchSQL } = prisma;
const params = {};
let searchQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != website_event.hostname
and website_event.referrer_domain != ''`;
}
if (search) {
if (decodeURIComponent(search).includes(',')) {
@ -49,6 +55,7 @@ async function relationalQuery(
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${searchQuery}
${excludeDomain}
group by 1
order by 2 desc
limit 10
@ -73,6 +80,11 @@ async function clickhouseQuery(
const { rawQuery, getSearchSQL } = clickhouse;
const params = {};
let searchQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
}
if (search) {
searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;
@ -103,6 +115,7 @@ async function clickhouseQuery(
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${searchQuery}
${excludeDomain}
group by 1
order by 2 desc
limit 10

19
src/queries/sql/pageviews/getPageviewMetrics.ts
View file

@ -34,7 +34,7 @@ async function relationalQuery(
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
},
{ joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' },
{ joinSession: SESSION_COLUMNS.includes(type) },
);
let entryExitQuery = '';
@ -66,7 +66,7 @@ async function relationalQuery(
return rawQuery(
`
select ${column} x,
${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y
count(distinct website_event.session_id) as y
from website_event
${cohortQuery}
${joinSession}
@ -126,7 +126,7 @@ async function clickhouseQuery(
sql = `
select ${column} x,
${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y
uniq(website_event.session_id) as y
from website_event
${cohortQuery}
${entryExitQuery}
@ -142,28 +142,27 @@ async function clickhouseQuery(
`;
} else {
let groupByQuery = '';
let columnQuery = `arrayJoin(${column})`;
let columnQuery = `session_id s, arrayJoin(${column})`;
if (column === 'referrer_domain') {
excludeDomain = `and t != hostname and t != ''`;
columnQuery = `session_id s, arrayJoin(${column})`;
excludeDomain = `and t != ''`;
}
if (type === 'entry') {
columnQuery = `visit_id x, argMinMerge(entry_url)`;
columnQuery = `session_id s, argMinMerge(entry_url)`;
}
if (type === 'exit') {
columnQuery = `visit_id x, argMaxMerge(exit_url)`;
columnQuery = `session_id s, argMaxMerge(exit_url)`;
}
if (type === 'entry' || type === 'exit') {
groupByQuery = 'group by x';
groupByQuery = 'group by s';
}
sql = `
select g.t as x,
${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y
uniq(s) as y
from (
select ${columnQuery} as t
from website_event_stats_hourly website_event

12
src/queries/sql/sessions/getWebsiteSession.ts
View file

@ -17,7 +17,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
select id,
distinct_id as "distinctId",
website_id as "websiteId",
hostname,
browser,
os,
device,
@ -37,7 +36,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
session.distinct_id,
website_event.visit_id,
session.website_id,
website_event.hostname,
session.browser,
session.os,
session.device,
@ -54,8 +52,8 @@ async function relationalQuery(websiteId: string, sessionId: string) {
join website_event on website_event.session_id = session.session_id
where session.website_id = {{websiteId::uuid}}
and session.session_id = {{sessionId::uuid}}
group by session.session_id, session.distinct_id, visit_id, session.website_id, website_event.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
group by id, distinct_id, website_id, hostname, browser, os, device, screen, language, country, region, city;
group by session.session_id, session.distinct_id, visit_id, session.website_id, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
group by id, distinct_id, website_id, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
@ -69,7 +67,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
select id,
websiteId,
distinctId,
hostname,
browser,
os,
device,
@ -89,7 +86,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
distinct_id as distinctId,
visit_id,
website_id as websiteId,
hostname,
browser,
os,
device,
@ -105,8 +101,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
group by session_id, distinct_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, region, city) t
group by id, websiteId, distinctId, hostname, browser, os, device, screen, language, country, region, city;
group by session_id, distinct_id, visit_id, website_id, browser, os, device, screen, language, country, region, city) t
group by id, websiteId, distinctId, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);

5
src/queries/sql/sessions/getWebsiteSessions.ts
View file

@ -27,7 +27,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
select
session.session_id as "id",
session.website_id as "websiteId",
website_event.hostname,
session.browser,
session.os,
session.device,
@ -58,7 +57,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
}
group by session.session_id,
session.website_id,
website_event.hostname,
session.browser,
session.os,
session.device,
@ -84,7 +82,6 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
select
session_id as id,
website_id as websiteId,
hostname,
browser,
os,
device,
@ -112,7 +109,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
or (positionCaseInsensitive(device, {search:String}) > 0))`
: ''
}
group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city
group by session_id, website_id, browser, os, device, screen, language, country, region, city
order by lastAt desc
`,
{ ...params, search },