watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-05 05:07:15 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Renamed folder.

Browse source
This commit is contained in:
Mike Cao 2025-02-05 20:31:48 -08:00
parent 8525188e42
commit dcf0da7b14
39 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

107
src/queries/sql/events/getEventDataEvents.ts Normal file
View file

@ -0,0 +1,107 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getEventDataEvents(
...args: [websiteId: string, filters: QueryFilters]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { event } = filters;
const { params } = await parseFilters(websiteId, filters);
if (event) {
return rawQuery(
`
select
website_event.event_name as "eventName",
event_data.data_key as "propertyName",
event_data.data_type as "dataType",
event_data.string_value as "propertyValue",
count(*) as "total"
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
and website_event.event_name = {{event}}
group by website_event.event_name, event_data.data_key, event_data.data_type, event_data.string_value
order by 1 asc, 2 asc, 3 asc, 5 desc
`,
params,
);
}
return rawQuery(
`
select
website_event.event_name as "eventName",
event_data.data_key as "propertyName",
event_data.data_type as "dataType",
count(*) as "total"
from event_data
inner join website_event
on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
group by website_event.event_name, event_data.data_key, event_data.data_type
order by 1 asc, 2 asc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ eventName: string; propertyName: string; dataType: number; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { event } = filters;
const { params } = await parseFilters(websiteId, filters);
if (event) {
return rawQuery(
`
select
event_name as eventName,
data_key as propertyName,
data_type as dataType,
string_value as propertyValue,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_name = {event:String}
group by data_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 5 desc
limit 500
`,
params,
);
}
return rawQuery(
`
select
event_name as eventName,
data_key as propertyName,
data_type as dataType,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by data_key, data_type, event_name
order by 1 asc, 2 asc
limit 500
`,
params,
);
}

69
src/queries/sql/events/getEventDataFields.ts Normal file
View file

@ -0,0 +1,69 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getEventDataFields(
...args: [websiteId: string, filters: QueryFilters]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters, getDateSQL } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
data_key as "propertyName",
data_type as "dataType",
case
when data_type = 2 then replace(string_value, '.0000', '')
when data_type = 4 then ${getDateSQL('date_value', 'hour')}
else string_value
end as "value",
count(*) as "total"
from event_data
join website_event on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by data_key, data_type, value
order by 2 desc
limit 100
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
data_key as propertyName,
data_type as dataType,
multiIf(data_type = 2, replaceAll(string_value, '.0000', ''),
data_type = 4, toString(date_trunc('hour', date_value)),
string_value) as "value",
count(*) as "total"
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by data_key, data_type, value
order by 2 desc
limit 100
`,
params,
);
}

68
src/queries/sql/events/getEventDataProperties.ts Normal file
View file

@ -0,0 +1,68 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getEventDataProperties(
...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
) {
const { rawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
website_event.event_name as "eventName",
event_data.data_key as "propertyName",
count(*) as "total"
from event_data
join website_event on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by website_event.event_name, event_data.data_key
order by 3 desc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
): Promise<{ eventName: string; propertyName: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
event_name as eventName,
data_key as propertyName,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by event_name, data_key
order by 1, 3 desc
limit 500
`,
params,
);
}

72
src/queries/sql/events/getEventDataStats.ts Normal file
View file

@ -0,0 +1,72 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters } from '@/lib/types';
export async function getEventDataStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<{
events: number;
properties: number;
records: number;
}> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
}).then(results => results?.[0]);
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
count(distinct t.website_event_id) as "events",
count(distinct t.data_key) as "properties",
sum(t.total) as "records"
from (
select
website_event_id,
data_key,
count(*) as "total"
from event_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by website_event_id, data_key
) as t
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ events: number; properties: number; records: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
count(distinct t.event_id) as "events",
count(distinct t.data_key) as "properties",
sum(t.total) as "records"
from (
select
event_id,
data_key,
count(*) as "total"
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by event_id, data_key
) as t
`,
params,
);
}

34
src/queries/sql/events/getEventDataUsage.ts Normal file
View file

@ -0,0 +1,34 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from '@/lib/db';
export function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({
[PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function clickhouseQuery(
websiteIds: string[],
startDate: Date,
endDate: Date,
): Promise<{ websiteId: string; count: number }[]> {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
website_id as websiteId,
count(*) as count
from event_data
where created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_id in {websiteIds:Array(UUID)}
group by website_id
`,
{
websiteIds,
startDate,
endDate,
},
);
}

75
src/queries/sql/events/getEventDataValues.ts Normal file
View file

@ -0,0 +1,75 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getEventDataValues(
...args: [
websiteId: string,
filters: QueryFilters & { eventName?: string; propertyName?: string },
]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { eventName?: string; propertyName?: string },
) {
const { rawQuery, parseFilters, getDateSQL } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
case
when data_type = 2 then replace(string_value, '.0000', '')
when data_type = 4 then ${getDateSQL('date_value', 'hour')}
else string_value
end as "value",
count(*) as "total"
from event_data
join website_event on website_event.event_id = event_data.website_event_id
where event_data.website_id = {{websiteId::uuid}}
and event_data.created_at between {{startDate}} and {{endDate}}
and event_data.data_key = {{propertyName}}
and website_event.event_name = {{eventName}}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { eventName?: string; propertyName?: string },
): Promise<{ value: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
multiIf(data_type = 2, replaceAll(string_value, '.0000', ''),
data_type = 4, toString(date_trunc('hour', date_value)),
string_value) as "value",
count(*) as "total"
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and data_key = {propertyName:String}
and event_name = {eventName:String}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}

94
src/queries/sql/events/getEventMetrics.ts Normal file
View file

@ -0,0 +1,94 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters, WebsiteEventMetric } from '@/lib/types';
export async function getEventMetrics(
...args: [websiteId: string, filters: QueryFilters]
): Promise<WebsiteEventMetric[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { rawQuery, getDateSQL, parseFilters } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.customEvent,
});
return rawQuery(
`
select
event_name x,
${getDateSQL('website_event.created_at', unit, timezone)} t,
count(*) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1, 2
order by 2
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; t: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { rawQuery, getDateSQL, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.customEvent,
});
let sql = '';
if (filterQuery) {
sql = `
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
count(*) y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x, t
order by t
`;
} else {
sql = `
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
count(*) y
from (
select arrayJoin(event_name) as event_name,
created_at
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
) as g
group by x, t
order by t
`;
}
return rawQuery(sql, params).then(a => {
return Object.values(a).map(a => {
return { x: a.x, t: a.t, y: Number(a.y) };
});
});
}

38
src/queries/sql/events/getEventUsage.ts Normal file
View file

@ -0,0 +1,38 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from '@/lib/db';
export function getEventUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({
[PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function clickhouseQuery(
websiteIds: string[],
startDate: Date,
endDate: Date,
): Promise<{ websiteId: string; count: number }[]> {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
website_id as websiteId,
count(*) as count
from website_event
where website_id in {websiteIds:Array(UUID)}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by website_id
`,
{
websiteIds,
startDate,
endDate,
},
).then(a => {
return Object.values(a).map(a => {
return { websiteId: a.websiteId, count: Number(a.count) };
});
});
}

98
src/queries/sql/events/getWebsiteEvents.ts Normal file
View file

@ -0,0 +1,98 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, getDatabaseType, POSTGRESQL, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { PageParams, QueryFilters } from '@/lib/types';
export function getWebsiteEvents(
...args: [websiteId: string, filters: QueryFilters, pageParams?: PageParams]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) {
const { pagedRawQuery, parseFilters } = prisma;
const { query } = pageParams;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
const db = getDatabaseType();
const like = db === POSTGRESQL ? 'ilike' : 'like';
return pagedRawQuery(
`
with events as (
select
event_id as "id",
website_id as "websiteId",
session_id as "sessionId",
created_at as "createdAt",
url_path as "urlPath",
url_query as "urlQuery",
referrer_path as "referrerPath",
referrer_query as "referrerQuery",
referrer_domain as "referrerDomain",
page_title as "pageTitle",
event_type as "eventType",
event_name as "eventName"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
${filterQuery}
${
query
? `and ((event_name ${like} {{query}} and event_type = 2)
or (url_path ${like} {{query}} and event_type = 1))`
: ''
}
order by created_at desc
limit 1000)
select * from events
`,
{ ...params, query: `%${query}%` },
pageParams,
);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) {
const { pagedQuery, parseFilters } = clickhouse;
const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters);
const { query } = pageParams;
return pagedQuery(
`
with events as (
select
event_id as id,
website_id as websiteId,
session_id as sessionId,
created_at as createdAt,
url_path as urlPath,
url_query as urlQuery,
referrer_path as referrerPath,
referrer_query as referrerQuery,
referrer_domain as referrerDomain,
page_title as pageTitle,
event_type as eventType,
event_name as eventName
from website_event
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
${
query
? `and ((positionCaseInsensitive(event_name, {query:String}) > 0 and event_type = 2)
or (positionCaseInsensitive(url_path, {query:String}) > 0 and event_type = 1))`
: ''
}
order by created_at desc
limit 1000)
select * from events
`,
{ ...params, query },
pageParams,
);
}

195
src/queries/sql/events/saveEvent.ts Normal file
View file

@ -0,0 +1,195 @@
import { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import clickhouse from '@/lib/clickhouse';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import { uuid } from '@/lib/crypto';
import { saveEventData } from './saveEventData';
export async function saveEvent(args: {
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
referrerQuery?: string;
referrerDomain?: string;
pageTitle?: string;
eventName?: string;
eventData?: any;
hostname?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
city?: string;
tag?: string;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(args),
[CLICKHOUSE]: () => clickhouseQuery(args),
});
}
async function relationalQuery(data: {
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
referrerQuery?: string;
referrerDomain?: string;
pageTitle?: string;
eventName?: string;
eventData?: any;
tag?: string;
}) {
const {
websiteId,
sessionId,
visitId,
urlPath,
urlQuery,
referrerPath,
referrerQuery,
referrerDomain,
eventName,
eventData,
pageTitle,
tag,
} = data;
const websiteEventId = uuid();
const websiteEvent = prisma.client.websiteEvent.create({
data: {
id: websiteEventId,
websiteId,
sessionId,
visitId,
urlPath: urlPath?.substring(0, URL_LENGTH),
urlQuery: urlQuery?.substring(0, URL_LENGTH),
referrerPath: referrerPath?.substring(0, URL_LENGTH),
referrerQuery: referrerQuery?.substring(0, URL_LENGTH),
referrerDomain: referrerDomain?.substring(0, URL_LENGTH),
pageTitle: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag,
},
});
if (eventData) {
await saveEventData({
websiteId,
sessionId,
eventId: websiteEventId,
urlPath: urlPath?.substring(0, URL_LENGTH),
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
eventData,
});
}
return websiteEvent;
}
async function clickhouseQuery(data: {
websiteId: string;
sessionId: string;
visitId: string;
urlPath: string;
urlQuery?: string;
referrerPath?: string;
referrerQuery?: string;
referrerDomain?: string;
pageTitle?: string;
eventName?: string;
eventData?: any;
hostname?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
city?: string;
tag?: string;
}) {
const {
websiteId,
sessionId,
visitId,
urlPath,
urlQuery,
referrerPath,
referrerQuery,
referrerDomain,
pageTitle,
eventName,
eventData,
country,
subdivision1,
subdivision2,
city,
tag,
...args
} = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const eventId = uuid();
const createdAt = getUTCString();
const message = {
...args,
website_id: websiteId,
session_id: sessionId,
visit_id: visitId,
event_id: eventId,
country: country,
subdivision1:
country && subdivision1
? subdivision1.includes('-')
? subdivision1
: `${country}-${subdivision1}`
: null,
subdivision2: subdivision2,
city: city,
url_path: urlPath?.substring(0, URL_LENGTH),
url_query: urlQuery?.substring(0, URL_LENGTH),
referrer_path: referrerPath?.substring(0, URL_LENGTH),
referrer_query: referrerQuery?.substring(0, URL_LENGTH),
referrer_domain: referrerDomain?.substring(0, URL_LENGTH),
page_title: pageTitle?.substring(0, PAGE_TITLE_LENGTH),
event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag: tag,
created_at: createdAt,
};
if (kafka.enabled) {
await sendMessage('event', message);
} else {
await insert('website_event', [message]);
}
if (eventData) {
await saveEventData({
websiteId,
sessionId,
eventId,
urlPath: urlPath?.substring(0, URL_LENGTH),
eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
eventData,
createdAt,
});
}
return data;
}

91
src/queries/sql/events/saveEventData.ts Normal file
View file

@ -0,0 +1,91 @@
import { Prisma } from '@prisma/client';
import { DATA_TYPE } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { flattenJSON, getStringValue } from '@/lib/data';
import clickhouse from '@/lib/clickhouse';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import { DynamicData } from '@/lib/types';
export async function saveEventData(data: {
websiteId: string;
eventId: string;
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: DynamicData;
createdAt?: string;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(data),
[CLICKHOUSE]: () => clickhouseQuery(data),
});
}
async function relationalQuery(data: {
websiteId: string;
eventId: string;
eventData: DynamicData;
}): Promise<Prisma.BatchPayload> {
const { websiteId, eventId, eventData } = data;
const jsonKeys = flattenJSON(eventData);
// id, websiteEventId, eventStringValue
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteEventId: eventId,
websiteId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
}));
return prisma.client.eventData.createMany({
data: flattenedData,
});
}
async function clickhouseQuery(data: {
websiteId: string;
eventId: string;
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: DynamicData;
createdAt?: string;
}) {
const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const jsonKeys = flattenJSON(eventData);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
event_id: eventId,
url_path: urlPath,
event_name: eventName,
data_key: key,
data_type: dataType,
string_value: getStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: createdAt,
};
});
if (kafka.enabled) {
await sendMessage('event_data', messages);
} else {
await insert('event_data', messages);
}
return data;
}

44
src/queries/sql/getActiveVisitors.ts Normal file
View file

@ -0,0 +1,44 @@
import { subMinutes } from 'date-fns';
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
export async function getActiveVisitors(...args: [websiteId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string) {
const { rawQuery } = prisma;
const result = await rawQuery(
`
select count(distinct session_id) as visitors
from website_event
where website_id = {{websiteId::uuid}}
and created_at >= {{startDate}}
`,
{ websiteId, startDate: subMinutes(new Date(), 5) },
);
return result[0] ?? null;
}
async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {
const { rawQuery } = clickhouse;
const result = await rawQuery(
`
select
count(distinct session_id) as "visitors"
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime64}
`,
{ websiteId, startDate: subMinutes(new Date(), 5) },
);
return result[0] ?? null;
}

67
src/queries/sql/getRealtimeActivity.ts Normal file
View file

@ -0,0 +1,67 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
import { QueryFilters } from '@/lib/types';
export async function getRealtimeActivity(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { rawQuery, parseFilters } = prisma;
const { params, filterQuery, dateQuery } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
website_event.session_id as "sessionId",
website_event.event_name as "eventName",
website_event.created_at as "createdAt",
session.browser,
session.os,
session.device,
session.country,
website_event.url_path as "urlPath",
website_event.referrer_domain as "referrerDomain"
from website_event
inner join session
on session.session_id = website_event.session_id
where website_event.website_id = {{websiteId::uuid}}
${filterQuery}
${dateQuery}
order by website_event.created_at desc
limit 100
`,
params,
);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters): Promise<{ x: number }> {
const { rawQuery, parseFilters } = clickhouse;
const { params, filterQuery, dateQuery } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
session_id as sessionId,
event_name as eventName,
created_at as createdAt,
browser,
os,
device,
country,
url_path as urlPath,
referrer_domain as referrerDomain
from website_event
where website_id = {websiteId:UUID}
${filterQuery}
${dateQuery}
order by createdAt asc
limit 100
`,
params,
);
}

80
src/queries/sql/getRealtimeData.ts Normal file
View file

@ -0,0 +1,80 @@
import { getPageviewStats, getRealtimeActivity, getSessionStats } from '@/queries/index';
function increment(data: object, key: string) {
if (key) {
if (!data[key]) {
data[key] = 1;
} else {
data[key] += 1;
}
}
}
export async function getRealtimeData(
websiteId: string,
criteria: { startDate: Date; timezone: string },
) {
const { startDate, timezone } = criteria;
const filters = { startDate, endDate: new Date(), unit: 'minute', timezone };
const [activity, pageviews, sessions] = await Promise.all([
getRealtimeActivity(websiteId, filters),
getPageviewStats(websiteId, filters),
getSessionStats(websiteId, filters),
]);
const uniques = new Set();
const { countries, urls, referrers, events } = activity.reduce(
(
obj: { countries: any; urls: any; referrers: any; events: any },
event: {
sessionId: string;
urlPath: string;
referrerDomain: string;
country: string;
eventName: string;
},
) => {
const { countries, urls, referrers, events } = obj;
const { sessionId, urlPath, referrerDomain, country, eventName } = event;
if (!uniques.has(sessionId)) {
uniques.add(sessionId);
increment(countries, country);
events.push({ __type: 'session', ...event });
}
increment(urls, urlPath);
increment(referrers, referrerDomain);
events.push({ __type: eventName ? 'event' : 'pageview', ...event });
return obj;
},
{
countries: {},
urls: {},
referrers: {},
events: [],
},
);
return {
countries,
urls,
referrers,
events: events.reverse(),
series: {
views: pageviews,
visitors: sessions,
},
totals: {
views: pageviews.reduce((sum: number, { y }: { y: number }) => Number(sum) + Number(y), 0),
visitors: sessions.reduce((sum: number, { y }: { y: number }) => Number(sum) + Number(y), 0),
events: activity.filter(e => e.eventName).length,
countries: Object.keys(countries).length,
},
timestamp: Date.now(),
};
}

118
src/queries/sql/getValues.ts Normal file
View file

@ -0,0 +1,118 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
export async function getValues(
...args: [websiteId: string, column: string, startDate: Date, endDate: Date, search: string]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
column: string,
startDate: Date,
endDate: Date,
search: string,
) {
const { rawQuery, getSearchSQL } = prisma;
const params = {};
let searchQuery = '';
if (search) {
if (decodeURIComponent(search).includes(',')) {
searchQuery = `AND (${decodeURIComponent(search)
.split(',')
.slice(0, 5)
.map((value: string, index: number) => {
const key = `search${index}`;
params[key] = value;
return getSearchSQL(column, key).replace('and ', '');
})
.join(' OR ')})`;
} else {
searchQuery = getSearchSQL(column);
}
}
return rawQuery(
`
select ${column} as "value", count(*) as "count"
from website_event
inner join session
on session.session_id = website_event.session_id
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${searchQuery}
group by 1
order by 2 desc
limit 10
`,
{
websiteId,
startDate,
endDate,
search: `%${search}%`,
...params,
},
);
}
async function clickhouseQuery(
websiteId: string,
column: string,
startDate: Date,
endDate: Date,
search: string,
) {
const { rawQuery, getSearchSQL } = clickhouse;
const params = {};
let searchQuery = '';
if (search) {
searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;
}
if (search) {
if (decodeURIComponent(search).includes(',')) {
searchQuery = `AND (${decodeURIComponent(search)
.split(',')
.slice(0, 5)
.map((value: string, index: number) => {
const key = `search${index}`;
params[key] = value;
return getSearchSQL(column, key).replace('and ', '');
})
.join(' OR ')})`;
} else {
searchQuery = getSearchSQL(column);
}
}
return rawQuery(
`
select ${column} as "value", count(*) as "count"
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${searchQuery}
group by 1
order by 2 desc
limit 10
`,
{
websiteId,
startDate,
endDate,
search,
...params,
},
);
}

49
src/queries/sql/getWebsiteDateRange.ts Normal file
View file

@ -0,0 +1,49 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
import { DEFAULT_RESET_DATE } from '@/lib/constants';
export async function getWebsiteDateRange(...args: [websiteId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string) {
const { rawQuery, parseFilters } = prisma;
const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery(
`
select
min(created_at) as mindate,
max(created_at) as maxdate
from website_event
where website_id = {{websiteId::uuid}}
and created_at >= {{startDate}}
`,
params,
);
return result[0] ?? null;
}
async function clickhouseQuery(websiteId: string) {
const { rawQuery, parseFilters } = clickhouse;
const { params } = await parseFilters(websiteId, { startDate: new Date(DEFAULT_RESET_DATE) });
const result = await rawQuery(
`
select
min(created_at) as mindate,
max(created_at) as maxdate
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime64}
`,
params,
);
return result[0] ?? null;
}

121
src/queries/sql/getWebsiteStats.ts Normal file
View file

@ -0,0 +1,121 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
import { EVENT_COLUMNS } from '@/lib/constants';
export async function getWebsiteStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
website_event.session_id,
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time"
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1, 2
) as t
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[]
> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
sql = `
select
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime"
from (
select
session_id,
visit_id,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, visit_id
) as t;
`;
} else {
sql = `
select
sum(t.c) as "pageviews",
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
sumIf(1, t.c = 1) as "bounces",
sum(max_time-min_time) as "totaltime"
from (select
session_id,
visit_id,
sum(views) c,
min(min_time) min_time,
max(max_time) max_time
from umami.website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, visit_id
) as t;
`;
}
return rawQuery(sql, params);
}

183
src/queries/sql/pageviews/getPageviewMetrics.ts Normal file
View file

@ -0,0 +1,183 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getPageviewMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number | string,
offset?: number | string,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
) {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
let entryExitQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != {{websiteDomain}}
and website_event.referrer_domain is not null`;
}
if (type === 'entry' || type === 'exit') {
const aggregrate = type === 'entry' ? 'min' : 'max';
entryExitQuery = `
join (
select visit_id,
${aggregrate}(created_at) target_created_at
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
group by visit_id
) x
on x.visit_id = website_event.visit_id
and x.target_created_at = website_event.created_at
`;
}
return rawQuery(
`
select ${column} x, count(*) y
from website_event
${joinSession}
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${excludeDomain}
${filterQuery}
group by 1
order by 2 desc
limit ${limit}
offset ${offset}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
let excludeDomain = '';
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
let entryExitQuery = '';
if (column === 'referrer_domain') {
excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
}
if (type === 'entry' || type === 'exit') {
const aggregrate = type === 'entry' ? 'min' : 'max';
entryExitQuery = `
JOIN (select visit_id,
${aggregrate}(created_at) target_created_at
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
group by visit_id) x
ON x.visit_id = website_event.visit_id
and x.target_created_at = website_event.created_at`;
}
sql = `
select ${column} x, count(*) y
from website_event
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
group by x
order by y desc
limit ${limit}
offset ${offset}
`;
} else {
let groupByQuery = '';
if (column === 'referrer_domain') {
excludeDomain = `and t != {websiteDomain:String} and t != ''`;
}
let columnQuery = `arrayJoin(${column})`;
if (type === 'entry') {
columnQuery = `visit_id x, argMinMerge(entry_url)`;
}
if (type === 'exit') {
columnQuery = `visit_id x, argMaxMerge(exit_url)`;
}
if (type === 'entry' || type === 'exit') {
groupByQuery = 'group by x';
}
sql = `
select g.t as x,
count(*) as y
from (
select ${columnQuery} as t
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
${groupByQuery}) as g
group by x
order by y desc
limit ${limit}
offset ${offset}
`;
}
return rawQuery(sql, params).then((result: any) => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});
});
}

92
src/queries/sql/pageviews/getPageviewStats.ts Normal file
View file

@ -0,0 +1,92 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { QueryFilters } from '@/lib/types';
export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
${getDateSQL('website_event.created_at', unit, timezone)} x,
count(*) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
order by 1
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'utc', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
count(*) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
} else {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
sum(views)as y
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
}
return rawQuery(sql, params);
}

250
src/queries/sql/reports/getFunnel.ts Normal file
View file

@ -0,0 +1,250 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
const formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {
return steps.map((step: { type: string; value: string }, i: number) => {
const visitors = Number(results[i]?.count) || 0;
const previous = Number(results[i - 1]?.count) || 0;
const dropped = previous > 0 ? previous - visitors : 0;
const dropoff = 1 - visitors / previous;
const remaining = visitors / Number(results[0].count);
return {
...step,
visitors,
previous,
dropped,
dropoff,
remaining,
};
});
};
export async function getFunnel(
...args: [
websiteId: string,
criteria: {
windowMinutes: number;
startDate: Date;
endDate: Date;
steps: { type: string; value: string }[];
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
windowMinutes: number;
startDate: Date;
endDate: Date;
steps: { type: string; value: string }[];
},
): Promise<
{
value: string;
visitors: number;
dropoff: number;
}[]
> {
const { windowMinutes, startDate, endDate, steps } = criteria;
const { rawQuery, getAddIntervalQuery } = prisma;
const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);
function getFunnelQuery(
steps: { type: string; value: string }[],
windowMinutes: number,
): {
levelOneQuery: string;
levelQuery: string;
sumQuery: string;
params: string[];
} {
return steps.reduce(
(pv, cv, i) => {
const levelNumber = i + 1;
const startSum = i > 0 ? 'union ' : '';
const isURL = cv.type === 'url';
const column = isURL ? 'url_path' : 'event_name';
let operator = '=';
let paramValue = cv.value;
if (cv.value.startsWith('*') || cv.value.endsWith('*')) {
operator = 'like';
paramValue = cv.value.replace(/^\*|\*$/g, '%');
}
if (levelNumber === 1) {
pv.levelOneQuery = `
WITH level1 AS (
select distinct session_id, created_at
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and ${column} ${operator} {{${i}}}
)`;
} else {
pv.levelQuery += `
, level${levelNumber} AS (
select distinct we.session_id, we.created_at
from level${i} l
join website_event we
on l.session_id = we.session_id
where we.website_id = {{websiteId::uuid}}
and we.created_at between l.created_at and ${getAddIntervalQuery(
`l.created_at `,
`${windowMinutes} minute`,
)}
and we.${column} ${operator} {{${i}}}
and we.created_at <= {{endDate}}
)`;
}
pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
pv.params.push(paramValue);
return pv;
},
{
levelOneQuery: '',
levelQuery: '',
sumQuery: '',
params: [],
},
);
}
return rawQuery(
`
${levelOneQuery}
${levelQuery}
${sumQuery}
ORDER BY level;
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(formatResults(steps));
}
async function clickhouseQuery(
websiteId: string,
criteria: {
windowMinutes: number;
startDate: Date;
endDate: Date;
steps: { type: string; value: string }[];
},
): Promise<
{
value: string;
visitors: number;
dropoff: number;
}[]
> {
const { windowMinutes, startDate, endDate, steps } = criteria;
const { rawQuery } = clickhouse;
const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(
steps,
windowMinutes,
);
function getFunnelQuery(
steps: { type: string; value: string }[],
windowMinutes: number,
): {
levelOneQuery: string;
levelQuery: string;
sumQuery: string;
stepFilterQuery: string;
params: { [key: string]: string };
} {
return steps.reduce(
(pv, cv, i) => {
const levelNumber = i + 1;
const startSum = i > 0 ? 'union all ' : '';
const startFilter = i > 0 ? 'or' : '';
const isURL = cv.type === 'url';
const column = isURL ? 'url_path' : 'event_name';
let operator = '=';
let paramValue = cv.value;
if (cv.value.startsWith('*') || cv.value.endsWith('*')) {
operator = 'like';
paramValue = cv.value.replace(/^\*|\*$/g, '%');
}
if (levelNumber === 1) {
pv.levelOneQuery = `\n
level1 AS (
select *
from level0
where ${column} ${operator} {param${i}:String}
)`;
} else {
pv.levelQuery += `\n
, level${levelNumber} AS (
select distinct y.session_id as session_id,
y.url_path as url_path,
y.referrer_path as referrer_path,
y.event_name,
y.created_at as created_at
from level${i} x
join level0 y
on x.session_id = y.session_id
where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute
and y.${column} ${operator} {param${i}:String}
)`;
}
pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;
pv.params[`param${i}`] = paramValue;
return pv;
},
{
levelOneQuery: '',
levelQuery: '',
sumQuery: '',
stepFilterQuery: '',
params: {},
},
);
}
return rawQuery(
`
WITH level0 AS (
select distinct session_id, url_path, referrer_path, event_name, created_at
from umami.website_event
where (${stepFilterQuery})
and website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
),
${levelOneQuery}
${levelQuery}
select *
from (
${sumQuery}
) ORDER BY level;
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(formatResults(steps));
}

375
src/queries/sql/reports/getGoals.ts Normal file
View file

@ -0,0 +1,375 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getGoals(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string }[];
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string }[];
},
): Promise<any> {
const { startDate, endDate, goals } = criteria;
const { rawQuery } = prisma;
const urls = goals.filter(a => a.type === 'url');
const events = goals.filter(a => a.type === 'event');
const eventData = goals.filter(a => a.type === 'event-data');
const hasUrl = urls.length > 0;
const hasEvent = events.length > 0;
const hasEventData = eventData.length > 0;
function getParameters(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlParam = urls.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventParam = events.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventDataParam = eventData.reduce((acc, cv, i) => {
acc[`eventData${i}`] = cv.value;
acc[`property${i}`] = cv.property;
return acc;
}, {});
return {
urls: { ...urlParam, startDate, endDate, websiteId },
events: { ...eventParam, startDate, endDate, websiteId },
eventData: { ...eventDataParam, startDate, endDate, websiteId },
};
}
function getColumns(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlColumns = urls
.map((a, i) => `COUNT(CASE WHEN url_path = {{url${i}}} THEN 1 END) AS URL${i},`)
.join('\n')
.slice(0, -1);
const eventColumns = events
.map((a, i) => `COUNT(CASE WHEN event_name = {{event${i}}} THEN 1 END) AS EVENT${i},`)
.join('\n')
.slice(0, -1);
const eventDataColumns = eventData
.map(
(a, i) =>
`${
a.operator === 'average' ? 'avg' : a.operator
}(CASE WHEN event_name = {{eventData${i}}} AND data_key = {{property${i}}} THEN ${
a.operator === 'count' ? '1' : 'number_value'
} END) AS EVENT_DATA${i},`,
)
.join('\n')
.slice(0, -1);
return { urls: urlColumns, events: eventColumns, eventData: eventDataColumns };
}
function getWhere(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlWhere = urls.map((a, i) => `{{url${i}}}`).join(',');
const eventWhere = events.map((a, i) => `{{event${i}}}`).join(',');
const eventDataNameWhere = eventData.map((a, i) => `{{eventData${i}}}`).join(',');
const eventDataKeyWhere = eventData.map((a, i) => `{{property${i}}}`).join(',');
return {
urls: `and url_path in (${urlWhere})`,
events: `and event_name in (${eventWhere})`,
eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`,
};
}
const parameters = getParameters(urls, events, eventData);
const columns = getColumns(urls, events, eventData);
const where = getWhere(urls, events, eventData);
const urlResults = hasUrl
? await rawQuery(
`
select
${columns.urls}
from website_event
where website_id = {{websiteId::uuid}}
${where.urls}
and created_at between {{startDate}} and {{endDate}}
`,
parameters.urls,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => ({
...urls[i],
goal: Number(urls[i].goal),
result: Number(results[key]),
}));
})
: [];
const eventResults = hasEvent
? await rawQuery(
`
select
${columns.events}
from website_event
where website_id = {{websiteId::uuid}}
${where.events}
and created_at between {{startDate}} and {{endDate}}
`,
parameters.events,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) };
});
})
: [];
const eventDataResults = hasEventData
? await rawQuery(
`
select
${columns.eventData}
from website_event w
join event_data d
on d.website_event_id = w.event_id
where w.website_id = {{websiteId::uuid}}
${where.eventData}
and w.created_at between {{startDate}} and {{endDate}}
`,
parameters.eventData,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) };
});
})
: [];
return [...urlResults, ...eventResults, ...eventDataResults];
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string; property?: string }[];
},
): Promise<{ type: string; value: string; goal: number; result: number }[]> {
const { startDate, endDate, goals } = criteria;
const { rawQuery } = clickhouse;
const urls = goals.filter(a => a.type === 'url');
const events = goals.filter(a => a.type === 'event');
const eventData = goals.filter(a => a.type === 'event-data');
const hasUrl = urls.length > 0;
const hasEvent = events.length > 0;
const hasEventData = eventData.length > 0;
function getParameters(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlParam = urls.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventParam = events.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventDataParam = eventData.reduce((acc, cv, i) => {
acc[`eventData${i}`] = cv.value;
acc[`property${i}`] = cv.property;
return acc;
}, {});
return {
urls: { ...urlParam, startDate, endDate, websiteId },
events: { ...eventParam, startDate, endDate, websiteId },
eventData: { ...eventDataParam, startDate, endDate, websiteId },
};
}
function getColumns(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlColumns = urls
.map((a, i) => `countIf(url_path = {url${i}:String}) AS URL${i},`)
.join('\n')
.slice(0, -1);
const eventColumns = events
.map((a, i) => `countIf(event_name = {event${i}:String}) AS EVENT${i},`)
.join('\n')
.slice(0, -1);
const eventDataColumns = eventData
.map(
(a, i) =>
`${a.operator === 'average' ? 'avg' : a.operator}If(${
a.operator !== 'count' ? 'number_value, ' : ''
}event_name = {eventData${i}:String} AND data_key = {property${i}:String}) AS EVENT_DATA${i},`,
)
.join('\n')
.slice(0, -1);
return { url: urlColumns, events: eventColumns, eventData: eventDataColumns };
}
function getWhere(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlWhere = urls.map((a, i) => `{url${i}:String}`).join(',');
const eventWhere = events.map((a, i) => `{event${i}:String}`).join(',');
const eventDataNameWhere = eventData.map((a, i) => `{eventData${i}:String}`).join(',');
const eventDataKeyWhere = eventData.map((a, i) => `{property${i}:String}`).join(',');
return {
urls: `and url_path in (${urlWhere})`,
events: `and event_name in (${eventWhere})`,
eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`,
};
}
const parameters = getParameters(urls, events, eventData);
const columns = getColumns(urls, events, eventData);
const where = getWhere(urls, events, eventData);
const urlResults = hasUrl
? await rawQuery(
`
select
${columns.url}
from website_event
where website_id = {websiteId:UUID}
${where.urls}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.urls,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...urls[i], goal: Number(urls[i].goal), result: Number(results[key]) };
});
})
: [];
const eventResults = hasEvent
? await rawQuery(
`
select
${columns.events}
from website_event
where website_id = {websiteId:UUID}
${where.events}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.events,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) };
});
})
: [];
const eventDataResults = hasEventData
? await rawQuery(
`
select
${columns.eventData}
from event_data
where website_id = {websiteId:UUID}
${where.eventData}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.eventData,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) };
});
})
: [];
return [...urlResults, ...eventResults, ...eventDataResults];
}

138
src/queries/sql/reports/getInsights.ts Normal file
View file

@ -0,0 +1,138 @@
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { QueryFilters } from '@/lib/types';
export async function getInsights(
...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
fields: { name: string; type?: string }[],
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: EVENT_TYPE.pageView,
},
{
joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),
},
);
return rawQuery(
`
select
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",
${parseFieldsByName(fields)}
from (
select
${parseFields(fields)},
website_event.session_id,
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time"
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by ${parseFieldsByName(fields)},
website_event.session_id, website_event.visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
fields: { name: string; type?: string }[],
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime",
${parseFieldsByName(fields)}
from (
select
${parseFields(fields)},
session_id,
visit_id,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by ${parseFieldsByName(fields)},
session_id, visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
limit 500
`,
params,
).then(a => {
return Object.values(a).map(a => {
return {
...a,
views: Number(a.views),
visitors: Number(a.visitors),
visits: Number(a.visits),
bounces: Number(a.bounces),
totaltime: Number(a.totaltime),
};
});
});
}
function parseFields(fields: { name: any }[]) {
return fields.map(({ name }) => `${FILTER_COLUMNS[name]} as "${name}"`).join(',');
}
function parseFieldsByName(fields: { name: any }[]) {
return `${fields.map(({ name }) => name).join(',')}`;
}

272
src/queries/sql/reports/getJourney.ts Normal file
View file

@ -0,0 +1,272 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
interface JourneyResult {
e1: string;
e2: string;
e3: string;
e4: string;
e5: string;
e6: string;
e7: string;
count: number;
}
export async function getJourney(
...args: [
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
): Promise<JourneyResult[]> {
const { startDate, endDate, steps, startStep, endStep } = filters;
const { rawQuery } = prisma;
const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery(
steps,
startStep,
endStep,
);
function getJourneyQuery(
steps: number,
startStep?: string,
endStep?: string,
): {
sequenceQuery: string;
startStepQuery: string;
endStepQuery: string;
params: { [key: string]: string };
} {
const params = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
// create sequence query
let selectQuery = '';
let maxQuery = '';
let groupByQuery = '';
for (let i = 1; i <= steps; i++) {
const endQuery = i < steps ? ',' : '';
selectQuery += `s.e${i},`;
maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`;
groupByQuery += `s.e${i}${endQuery} `;
}
sequenceQuery = `\nsequences as (
select ${selectQuery}
count(*) count
FROM (
select visit_id,
${maxQuery}
FROM events
group by visit_id) s
group by ${groupByQuery})
`;
// create start Step params query
if (startStep) {
startStepQuery = `and e1 = {{startStep}}`;
params['startStep'] = startStep;
}
// create end Step params query
if (endStep) {
for (let i = 1; i < steps; i++) {
const startQuery = i === 1 ? 'and (' : '\nor ';
endStepQuery += `${startQuery}(e${i} = {{endStep}} and e${i + 1} is null) `;
}
endStepQuery += `\nor (e${steps} = {{endStep}}))`;
params['endStep'] = endStep;
}
return {
sequenceQuery,
startStepQuery,
endStepQuery,
params,
};
}
return rawQuery(
`
WITH events AS (
select distinct
visit_id,
referrer_path,
coalesce(nullIf(event_name, ''), url_path) event,
row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}),
${sequenceQuery}
select *
from sequences
where 1 = 1
${startStepQuery}
${endStepQuery}
order by count desc
limit 100
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(parseResult);
}
async function clickhouseQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
): Promise<JourneyResult[]> {
const { startDate, endDate, steps, startStep, endStep } = filters;
const { rawQuery } = clickhouse;
const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery(
steps,
startStep,
endStep,
);
function getJourneyQuery(
steps: number,
startStep?: string,
endStep?: string,
): {
sequenceQuery: string;
startStepQuery: string;
endStepQuery: string;
params: { [key: string]: string };
} {
const params = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
// create sequence query
let selectQuery = '';
let maxQuery = '';
let groupByQuery = '';
for (let i = 1; i <= steps; i++) {
const endQuery = i < steps ? ',' : '';
selectQuery += `s.e${i},`;
maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`;
groupByQuery += `s.e${i}${endQuery} `;
}
sequenceQuery = `\nsequences as (
select ${selectQuery}
count(*) count
FROM (
select visit_id,
${maxQuery}
FROM events
group by visit_id) s
group by ${groupByQuery})
`;
// create start Step params query
if (startStep) {
startStepQuery = `and e1 = {startStep:String}`;
params['startStep'] = startStep;
}
// create end Step params query
if (endStep) {
for (let i = 1; i < steps; i++) {
const startQuery = i === 1 ? 'and (' : '\nor ';
endStepQuery += `${startQuery}(e${i} = {endStep:String} and e${i + 1} is null) `;
}
endStepQuery += `\nor (e${steps} = {endStep:String}))`;
params['endStep'] = endStep;
}
return {
sequenceQuery,
startStepQuery,
endStepQuery,
params,
};
}
return rawQuery(
`
WITH events AS (
select distinct
visit_id,
coalesce(nullIf(event_name, ''), url_path) event,
row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number
from umami.website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}),
${sequenceQuery}
select *
from sequences
where 1 = 1
${startStepQuery}
${endStepQuery}
order by count desc
limit 100
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(parseResult);
}
function combineSequentialDuplicates(array: any) {
if (array.length === 0) return array;
const result = [array[0]];
for (let i = 1; i < array.length; i++) {
if (array[i] !== array[i - 1]) {
result.push(array[i]);
}
}
return result;
}
function parseResult(data: any) {
return data.map(({ e1, e2, e3, e4, e5, e6, e7, count }) => ({
items: combineSequentialDuplicates([e1, e2, e3, e4, e5, e6, e7]),
count: +Number(count),
}));
}

171
src/queries/sql/reports/getRetention.ts Normal file
View file

@ -0,0 +1,171 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getRetention(
...args: [
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
): Promise<
{
date: string;
day: number;
visitors: number;
returnVisitors: number;
percentage: number;
}[]
> {
const { startDate, endDate, timezone = 'UTC' } = filters;
const { getDateSQL, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma;
const unit = 'day';
return rawQuery(
`
WITH cohort_items AS (
select session_id,
${getDateSQL('created_at', unit, timezone)} as cohort_date
from session
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
),
user_activities AS (
select distinct
w.session_id,
${getDayDiffQuery(getDateSQL('created_at', unit, timezone), 'c.cohort_date')} as day_number
from website_event w
join cohort_items c
on w.session_id = c.session_id
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
),
cohort_size as (
select cohort_date,
count(*) as visitors
from cohort_items
group by 1
order by 1
),
cohort_date as (
select
c.cohort_date,
a.day_number,
count(*) as visitors
from user_activities a
join cohort_items c
on a.session_id = c.session_id
group by 1, 2
)
select
c.cohort_date as date,
c.day_number as day,
s.visitors,
c.visitors as "returnVisitors",
${getCastColumnQuery('c.visitors', 'float')} * 100 / s.visitors as percentage
from cohort_date c
join cohort_size s
on c.cohort_date = s.cohort_date
where c.day_number <= 31
order by 1, 2`,
{
websiteId,
startDate,
endDate,
},
);
}
async function clickhouseQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
): Promise<
{
date: string;
day: number;
visitors: number;
returnVisitors: number;
percentage: number;
}[]
> {
const { startDate, endDate, timezone = 'UTC' } = filters;
const { getDateSQL, rawQuery } = clickhouse;
const unit = 'day';
return rawQuery(
`
WITH cohort_items AS (
select
min(${getDateSQL('created_at', unit, timezone)}) as cohort_date,
session_id
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by session_id
),
user_activities AS (
select distinct
w.session_id,
(${getDateSQL('created_at', unit, timezone)} - c.cohort_date) / 86400 as day_number
from website_event w
join cohort_items c
on w.session_id = c.session_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
),
cohort_size as (
select cohort_date,
count(*) as visitors
from cohort_items
group by 1
order by 1
),
cohort_date as (
select
c.cohort_date,
a.day_number,
count(*) as visitors
from user_activities a
join cohort_items c
on a.session_id = c.session_id
group by 1, 2
)
select
c.cohort_date as date,
c.day_number as day,
s.visitors as visitors,
c.visitors returnVisitors,
c.visitors * 100 / s.visitors as percentage
from cohort_date c
join cohort_size s
on c.cohort_date = s.cohort_date
where c.day_number <= 31
order by 1, 2`,
{
websiteId,
startDate,
endDate,
},
);
}

274
src/queries/sql/reports/getRevenue.ts Normal file
View file

@ -0,0 +1,274 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, getDatabaseType, POSTGRESQL, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getRevenue(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
): Promise<{
chart: { x: string; t: string; y: number }[];
country: { name: string; value: number }[];
total: { sum: number; count: number; unique_count: number };
table: {
currency: string;
sum: number;
count: number;
unique_count: number;
}[];
}> {
const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria;
const { getDateSQL, rawQuery } = prisma;
const db = getDatabaseType();
const like = db === POSTGRESQL ? 'ilike' : 'like';
const chartRes = await rawQuery(
`
select
we.event_name x,
${getDateSQL('ed.created_at', unit, timezone)} t,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) y
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by x, t
order by t
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
const countryRes = await rawQuery(
`
select
s.country as name,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) value
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join session s
on s.session_id = we.session_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by s.country
`,
{ websiteId, startDate, endDate, currency },
);
const totalRes = await rawQuery(
`
select
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum,
count(distinct event_id) as count,
count(distinct session_id) as unique_count
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
`,
{ websiteId, startDate, endDate, currency },
).then(result => result?.[0]);
const tableRes = await rawQuery(
`
select
c.currency,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum,
count(distinct ed.website_event_id) as count,
count(distinct we.session_id) as unique_count
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id, string_value as currency
from event_data
where data_key ${like} '%currency%') c
on c.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by c.currency
order by sum desc;
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes };
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
): Promise<{
chart: { x: string; t: string; y: number }[];
country: { name: string; value: number }[];
total: { sum: number; count: number; unique_count: number };
table: {
currency: string;
sum: number;
count: number;
unique_count: number;
}[];
}> {
const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria;
const { getDateSQL, rawQuery } = clickhouse;
const chartRes = await rawQuery<
{
x: string;
t: string;
y: number;
}[]
>(
`
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) y
from event_data
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) currency
on currency.event_id = event_data.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
group by x, t
order by t
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
const countryRes = await rawQuery<
{
name: string;
value: number;
}[]
>(
`
select
s.country as name,
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as value
from event_data ed
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) c
on c.event_id = ed.event_id
join (select distinct website_id, session_id, country
from website_event_stats_hourly
where website_id = {websiteId:UUID}) s
on ed.website_id = s.website_id
and ed.session_id = s.session_id
where ed.website_id = {websiteId:UUID}
and ed.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(ed.data_key, 'revenue') > 0
group by s.country
`,
{ websiteId, startDate, endDate, currency },
);
const totalRes = await rawQuery<{
sum: number;
avg: number;
count: number;
unique_count: number;
}>(
`
select
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as sum,
uniqExact(event_id) as count,
uniqExact(session_id) as unique_count
from event_data
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) currency
on currency.event_id = event_data.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
`,
{ websiteId, startDate, endDate, currency },
).then(result => result?.[0]);
const tableRes = await rawQuery<
{
currency: string;
sum: number;
avg: number;
count: number;
unique_count: number;
}[]
>(
`
select
c.currency,
sum(coalesce(toDecimal64(ed.number_value, 2), toDecimal64(ed.string_value, 2))) as sum,
uniqExact(ed.event_id) as count,
uniqExact(ed.session_id) as unique_count
from event_data ed
join (select event_id, string_value as currency
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0) c
on c.event_id = ed.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
group by c.currency
order by sum desc;
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes };
}

75
src/queries/sql/reports/getRevenueValues.ts Normal file
View file

@ -0,0 +1,75 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA, getDatabaseType, POSTGRESQL } from '@/lib/db';
export async function getRevenueValues(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
) {
const { rawQuery } = prisma;
const { startDate, endDate } = criteria;
const db = getDatabaseType();
const like = db === POSTGRESQL ? 'ilike' : 'like';
return rawQuery(
`
select distinct string_value as currency
from event_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and data_key ${like} '%currency%'
order by currency
`,
{
websiteId,
startDate,
endDate,
},
);
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
) {
const { rawQuery } = clickhouse;
const { startDate, endDate } = criteria;
return rawQuery(
`
select distinct string_value as currency
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'currency') > 0
order by currency
`,
{
websiteId,
startDate,
endDate,
},
);
}

102
src/queries/sql/reports/getUTM.ts Normal file
View file

@ -0,0 +1,102 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getUTM(
...args: [
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
) {
const { startDate, endDate } = filters;
const { rawQuery } = prisma;
return rawQuery(
`
select url_query, count(*) as "num"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and coalesce(url_query, '') != ''
and event_type = 1
group by 1
`,
{
websiteId,
startDate,
endDate,
},
).then(result => parseParameters(result as any[]));
}
async function clickhouseQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
) {
const { startDate, endDate } = filters;
const { rawQuery } = clickhouse;
return rawQuery(
`
select url_query, count(*) as "num"
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and url_query != ''
and event_type = 1
group by 1
`,
{
websiteId,
startDate,
endDate,
},
).then(result => parseParameters(result as any[]));
}
function parseParameters(data: any[]) {
return data.reduce((obj, { url_query, num }) => {
try {
const searchParams = new URLSearchParams(url_query);
for (const [key, value] of searchParams) {
if (key.match(/^utm_(\w+)$/)) {
const name = value;
if (!obj[key]) {
obj[key] = { [name]: Number(num) };
} else if (!obj[key][name]) {
obj[key][name] = Number(num);
} else {
obj[key][name] += Number(num);
}
}
}
} catch {
// Ignore
}
return obj;
}, {});
}

36
src/queries/sql/sessions/createSession.ts Normal file
View file

@ -0,0 +1,36 @@
import { Prisma } from '@prisma/client';
import prisma from '@/lib/prisma';
export async function createSession(data: Prisma.SessionCreateInput) {
const {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
} = data;
return prisma.client.session.create({
data: {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
},
});
}

59
src/queries/sql/sessions/getSessionActivity.ts Normal file
View file

@ -0,0 +1,59 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getSessionActivity(
...args: [websiteId: string, sessionId: string, startDate: Date, endDate: Date]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
sessionId: string,
startDate: Date,
endDate: Date,
) {
return prisma.client.websiteEvent.findMany({
where: {
sessionId,
websiteId,
createdAt: { gte: startDate, lte: endDate },
},
take: 500,
orderBy: { createdAt: 'desc' },
});
}
async function clickhouseQuery(
websiteId: string,
sessionId: string,
startDate: Date,
endDate: Date,
) {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
created_at as createdAt,
url_path as urlPath,
url_query as urlQuery,
referrer_domain as referrerDomain,
event_id as eventId,
event_type as eventType,
event_name as eventName,
visit_id as visitId
from website_event
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
order by created_at desc
limit 500
`,
{ websiteId, sessionId, startDate, endDate },
);
}

56
src/queries/sql/sessions/getSessionData.ts Normal file
View file

@ -0,0 +1,56 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
export async function getSessionData(...args: [websiteId: string, sessionId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, sessionId: string) {
const { rawQuery } = prisma;
return rawQuery(
`
select
website_id as "websiteId",
session_id as "sessionId",
data_key as "dataKey",
data_type as "dataType",
replace(string_value, '.0000', '') as "stringValue",
number_value as "numberValue",
date_value as "dateValue",
created_at as "createdAt"
from session_data
where website_id = {{websiteId::uuid}}
and session_id = {{sessionId::uuid}}
order by data_key asc
`,
{ websiteId, sessionId },
);
}
async function clickhouseQuery(websiteId: string, sessionId: string) {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
website_id as websiteId,
session_id as sessionId,
data_key as dataKey,
data_type as dataType,
replace(string_value, '.0000', '') as stringValue,
number_value as numberValue,
date_value as dateValue,
created_at as createdAt
from session_data final
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
order by data_key asc
`,
{ websiteId, sessionId },
);
}

65
src/queries/sql/sessions/getSessionDataProperties.ts Normal file
View file

@ -0,0 +1,65 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getSessionDataProperties(
...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
) {
const { rawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
data_key as "propertyName",
count(*) as "total"
from session_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by data_key
order by 2 desc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
): Promise<{ propertyName: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
data_key as propertyName,
count(*) as total
from session_data final
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by data_key
order by 2 desc
limit 500
`,
params,
);
}

69
src/queries/sql/sessions/getSessionDataValues.ts Normal file
View file

@ -0,0 +1,69 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getSessionDataValues(
...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
) {
const { rawQuery, parseFilters, getDateSQL } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
case
when data_type = 2 then replace(string_value, '.0000', '')
when data_type = 4 then ${getDateSQL('date_value', 'hour')}
else string_value
end as "value",
count(*) as "total"
from session_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and data_key = {{propertyName}}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
multiIf(data_type = 2, replaceAll(string_value, '.0000', ''),
data_type = 4, toString(date_trunc('hour', date_value)),
string_value) as "value",
count(*) as "total"
from session_data final
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and data_key = {propertyName:String}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}

123
src/queries/sql/sessions/getSessionMetrics.ts Normal file
View file

@ -0,0 +1,123 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getSessionMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number | string,
offset?: number | string,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
) {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: EVENT_TYPE.pageView,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const includeCountry = column === 'city' || column === 'subdivision1';
return rawQuery(
`
select
${column} x,
count(distinct website_event.session_id) y
${includeCountry ? ', country' : ''}
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = {{eventType}}
${filterQuery}
group by 1
${includeCountry ? ', 3' : ''}
order by 2 desc
limit ${limit}
offset ${offset}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
sql = `
select
${column} x,
count(distinct session_id) y
${includeCountry ? ', country' : ''}
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
order by y desc
limit ${limit}
offset ${offset}
`;
} else {
sql = `
select
${column} x,
uniq(session_id) y
${includeCountry ? ', country' : ''}
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
order by y desc
limit ${limit}
offset ${offset}
`;
}
return rawQuery(sql, params).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y), country: a.country };
});
});
}

92
src/queries/sql/sessions/getSessionStats.ts Normal file
View file

@ -0,0 +1,92 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
${getDateSQL('website_event.created_at', unit, timezone)} x,
count(distinct website_event.session_id) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
order by 1
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'utc', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
count(distinct session_id) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
} else {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
uniq(session_id) as y
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
}
return rawQuery(sql, params);
}

109
src/queries/sql/sessions/getWebsiteSession.ts Normal file
View file

@ -0,0 +1,109 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
export async function getWebsiteSession(...args: [websiteId: string, sessionId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, sessionId: string) {
const { rawQuery, getTimestampDiffSQL } = prisma;
return rawQuery(
`
select id,
website_id as "websiteId",
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
min(min_time) as "firstAt",
max(max_time) as "lastAt",
count(distinct visit_id) as visits,
sum(views) as views,
sum(events) as events,
sum(${getTimestampDiffSQL('min_time', 'max_time')}) as "totaltime"
from (select
session.session_id as id,
website_event.visit_id,
session.website_id,
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city,
min(website_event.created_at) as min_time,
max(website_event.created_at) as max_time,
sum(case when website_event.event_type = 1 then 1 else 0 end) as views,
sum(case when website_event.event_type = 2 then 1 else 0 end) as events
from session
join website_event on website_event.session_id = session.session_id
where session.website_id = {{websiteId::uuid}}
and session.session_id = {{sessionId::uuid}}
group by session.session_id, visit_id, session.website_id, session.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.subdivision1, session.city) t
group by id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
}
async function clickhouseQuery(websiteId: string, sessionId: string) {
const { rawQuery, getDateStringSQL } = clickhouse;
return rawQuery(
`
select id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
uniq(visit_id) visits,
sum(views) as views,
sum(events) as events,
sum(max_time-min_time) as totaltime
from (select
session_id as id,
visit_id,
website_id as websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
min(min_time) as min_time,
max(max_time) as max_time,
sum(views) as views,
length(groupArrayArray(event_name)) as events
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
group by session_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city) t
group by id, websiteId, hostname, browser, os, device, screen, language, country, subdivision1, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
}

72
src/queries/sql/sessions/getWebsiteSessionStats.ts Normal file
View file

@ -0,0 +1,72 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getWebsiteSessionStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
const { parseFilters, rawQuery } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return rawQuery(
`
select
count(*) as "pageviews",
count(distinct website_event.session_id) as "visitors",
count(distinct website_event.visit_id) as "visits",
count(distinct session.country) as "countries",
sum(case when website_event.event_type = 2 then 1 else 0 end) as "events"
from website_event
join session on website_event.session_id = session.session_id
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return rawQuery(
`
select
sum(views) as "pageviews",
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
uniq(country) as "countries",
sum(length(event_name)) as "events"
from umami.website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
params,
);
}

102
src/queries/sql/sessions/getWebsiteSessions.ts Normal file
View file

@ -0,0 +1,102 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { PageParams, QueryFilters } from '@/lib/types';
export async function getWebsiteSessions(
...args: [websiteId: string, filters?: QueryFilters, pageParams?: PageParams]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams: PageParams) {
const { pagedRawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return pagedRawQuery(
`
with sessions as (
select
session.session_id as "id",
session.website_id as "websiteId",
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city,
min(website_event.created_at) as "firstAt",
max(website_event.created_at) as "lastAt",
count(distinct website_event.visit_id) as "visits",
sum(case when website_event.event_type = 1 then 1 else 0 end) as "views",
max(website_event.created_at) as "createdAt"
from website_event
join session on session.session_id = website_event.session_id
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by session.session_id,
session.website_id,
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city
order by max(website_event.created_at) desc
limit 1000)
select * from sessions
`,
params,
pageParams,
);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) {
const { pagedQuery, parseFilters, getDateStringSQL } = clickhouse;
const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters);
return pagedQuery(
`
with sessions as (
select
session_id as id,
website_id as websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
uniq(visit_id) as visits,
sumIf(views, event_type = 1) as views,
lastAt as createdAt
from website_event_stats_hourly
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
group by session_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city
order by lastAt desc
limit 1000)
select * from sessions
`,
params,
pageParams,
);
}

71
src/queries/sql/sessions/getWebsiteSessionsWeekly.ts Normal file
View file

@ -0,0 +1,71 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
import { QueryFilters } from '@/lib/types';
export async function getWebsiteSessionsWeekly(
...args: [websiteId: string, filters?: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc' } = filters;
const { rawQuery, getDateWeeklySQL, parseFilters } = prisma;
const { params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
${getDateWeeklySQL('created_at', timezone)} as time,
count(distinct session_id) as value
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by time
order by 2
`,
params,
).then(formatResults);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc' } = filters;
const { rawQuery, parseFilters } = clickhouse;
const { params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
formatDateTime(toDateTime(created_at, '${timezone}'), '%w:%H') as time,
count(distinct session_id) as value
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by time
order by time
`,
params,
).then(formatResults);
}
function formatResults(data: any) {
const days = [];
for (let i = 0; i < 7; i++) {
days.push([]);
for (let j = 0; j < 24; j++) {
days[i].push(
Number(
data.find(({ time }) => time === `${i}:${j.toString().padStart(2, '0')}`)?.value || 0,
),
);
}
}
return days;
}

109
src/queries/sql/sessions/saveSessionData.ts Normal file
View file

@ -0,0 +1,109 @@
import { DATA_TYPE } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import prisma from '@/lib/prisma';
import { DynamicData } from '@/lib/types';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import kafka from '@/lib/kafka';
import clickhouse from '@/lib/clickhouse';
export async function saveSessionData(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(data),
[CLICKHOUSE]: () => clickhouseQuery(data),
});
}
export async function relationalQuery(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
const { client } = prisma;
const { websiteId, sessionId, sessionData } = data;
const jsonKeys = flattenJSON(sessionData);
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteId,
sessionId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
}));
const existing = await client.sessionData.findMany({
where: {
sessionId,
},
select: {
id: true,
sessionId: true,
dataKey: true,
},
});
for (const data of flattenedData) {
const { sessionId, dataKey, ...props } = data;
const record = existing.find(e => e.sessionId === sessionId && e.dataKey === dataKey);
if (record) {
await client.sessionData.update({
where: {
id: record.id,
},
data: {
...props,
},
});
} else {
await client.sessionData.create({
data,
});
}
}
return flattenedData;
}
async function clickhouseQuery(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
const { websiteId, sessionId, sessionData } = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const createdAt = getUTCString();
const jsonKeys = flattenJSON(sessionData);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
data_key: key,
data_type: dataType,
string_value: getStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: createdAt,
};
});
if (kafka.enabled) {
await sendMessage('session_data', messages);
} else {
await insert('session_data', messages);
}
return data;
}