This commit is contained in:
badrAG 2025-02-24 20:06:12 +00:00
parent 6a8b804aff
commit d62055537e
26 changed files with 2599 additions and 86 deletions

View file

@ -0,0 +1,91 @@
import { Prisma } from '@prisma/client';
import { DATA_TYPE } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { flattenJSON, getStringValue } from '@/lib/data';
import clickhouse from '@/lib/clickhouse';
import kafka from '@/lib/kafka';
import prisma from '@/lib/prisma';
import { DynamicData } from '@/lib/types';
export async function saveEventData(data: {
websiteId: string;
eventId: string;
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: DynamicData;
createdAt?: string;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(data),
[CLICKHOUSE]: () => clickhouseQuery(data),
});
}
async function relationalQuery(data: {
websiteId: string;
eventId: string;
eventData: DynamicData;
}): Promise<Prisma.BatchPayload> {
const { websiteId, eventId, eventData } = data;
const jsonKeys = flattenJSON(eventData);
// id, websiteEventId, eventStringValue
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteEventId: eventId,
websiteId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
}));
return prisma.client.eventData.createMany({
data: flattenedData,
});
}
async function clickhouseQuery(data: {
websiteId: string;
eventId: string;
sessionId?: string;
urlPath?: string;
eventName?: string;
eventData: DynamicData;
createdAt?: string;
}) {
const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const jsonKeys = flattenJSON(eventData);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
event_id: eventId,
url_path: urlPath,
event_name: eventName,
data_key: key,
data_type: dataType,
string_value: getStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: createdAt,
};
});
if (kafka.enabled) {
await sendMessage('event_data', messages);
} else {
await insert('event_data', messages);
}
return data;
}

View file

@ -0,0 +1,182 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getPageviewMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number | string,
offset?: number | string,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
) {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
},
{ joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' },
);
let entryExitQuery = '';
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != session.hostname
and website_event.referrer_domain != ''`;
}
if (type === 'entry' || type === 'exit') {
const aggregrate = type === 'entry' ? 'min' : 'max';
entryExitQuery = `
join (
select visit_id,
${aggregrate}(created_at) target_created_at
from website_event
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
group by visit_id
) x
on x.visit_id = website_event.visit_id
and x.target_created_at = website_event.created_at
`;
}
return rawQuery(
`
select ${column} x,
${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y
from website_event
${joinSession}
${entryExitQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${excludeDomain}
${filterQuery}
group by 1
order by 2 desc
limit ${limit}
offset ${offset}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
});
let sql = '';
let excludeDomain = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
let entryExitQuery = '';
if (column === 'referrer_domain') {
excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
}
if (type === 'entry' || type === 'exit') {
const aggregrate = type === 'entry' ? 'min' : 'max';
entryExitQuery = `
JOIN (select visit_id,
${aggregrate}(created_at) target_created_at
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
group by visit_id) x
ON x.visit_id = website_event.visit_id
and x.target_created_at = website_event.created_at`;
}
sql = `
select ${column} x,
${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y
from website_event
${entryExitQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
group by x
order by y desc
limit ${limit}
offset ${offset}
`;
} else {
let groupByQuery = '';
let columnQuery = `arrayJoin(${column})`;
if (column === 'referrer_domain') {
excludeDomain = `and t != hostname and t != ''`;
columnQuery = `session_id s, arrayJoin(${column})`;
}
if (type === 'entry') {
columnQuery = `visit_id x, argMinMerge(entry_url)`;
}
if (type === 'exit') {
columnQuery = `visit_id x, argMaxMerge(exit_url)`;
}
if (type === 'entry' || type === 'exit') {
groupByQuery = 'group by x';
}
sql = `
select g.t as x,
${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y
from (
select ${columnQuery} as t
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
${groupByQuery}) as g
group by x
order by y desc
limit ${limit}
offset ${offset}
`;
}
return rawQuery(sql, params);
}

View file

@ -0,0 +1,92 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { QueryFilters } from '@/lib/types';
export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
${getDateSQL('website_event.created_at', unit, timezone)} x,
count(*) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
order by 1
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'utc', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
count(*) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
} else {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
sum(views)as y
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
}
return rawQuery(sql, params);
}

View file

@ -0,0 +1,375 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getGoals(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string }[];
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string }[];
},
): Promise<any> {
const { startDate, endDate, goals } = criteria;
const { rawQuery } = prisma;
const urls = goals.filter(a => a.type === 'url');
const events = goals.filter(a => a.type === 'event');
const eventData = goals.filter(a => a.type === 'event-data');
const hasUrl = urls.length > 0;
const hasEvent = events.length > 0;
const hasEventData = eventData.length > 0;
function getParameters(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlParam = urls.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventParam = events.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventDataParam = eventData.reduce((acc, cv, i) => {
acc[`eventData${i}`] = cv.value;
acc[`property${i}`] = cv.property;
return acc;
}, {});
return {
urls: { ...urlParam, startDate, endDate, websiteId },
events: { ...eventParam, startDate, endDate, websiteId },
eventData: { ...eventDataParam, startDate, endDate, websiteId },
};
}
function getColumns(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlColumns = urls
.map((a, i) => `COUNT(CASE WHEN url_path = {{url${i}}} THEN 1 END) AS URL${i},`)
.join('\n')
.slice(0, -1);
const eventColumns = events
.map((a, i) => `COUNT(CASE WHEN event_name = {{event${i}}} THEN 1 END) AS EVENT${i},`)
.join('\n')
.slice(0, -1);
const eventDataColumns = eventData
.map(
(a, i) =>
`${
a.operator === 'average' ? 'avg' : a.operator
}(CASE WHEN event_name = {{eventData${i}}} AND data_key = {{property${i}}} THEN ${
a.operator === 'count' ? '1' : 'number_value'
} END) AS EVENT_DATA${i},`,
)
.join('\n')
.slice(0, -1);
return { urls: urlColumns, events: eventColumns, eventData: eventDataColumns };
}
function getWhere(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlWhere = urls.map((a, i) => `{{url${i}}}`).join(',');
const eventWhere = events.map((a, i) => `{{event${i}}}`).join(',');
const eventDataNameWhere = eventData.map((a, i) => `{{eventData${i}}}`).join(',');
const eventDataKeyWhere = eventData.map((a, i) => `{{property${i}}}`).join(',');
return {
urls: `and url_path in (${urlWhere})`,
events: `and event_name in (${eventWhere})`,
eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`,
};
}
const parameters = getParameters(urls, events, eventData);
const columns = getColumns(urls, events, eventData);
const where = getWhere(urls, events, eventData);
const urlResults = hasUrl
? await rawQuery(
`
select
${columns.urls}
from website_event
where website_id = {{websiteId::uuid}}
${where.urls}
and created_at between {{startDate}} and {{endDate}}
`,
parameters.urls,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => ({
...urls[i],
goal: Number(urls[i].goal),
result: Number(results[key]),
}));
})
: [];
const eventResults = hasEvent
? await rawQuery(
`
select
${columns.events}
from website_event
where website_id = {{websiteId::uuid}}
${where.events}
and created_at between {{startDate}} and {{endDate}}
`,
parameters.events,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) };
});
})
: [];
const eventDataResults = hasEventData
? await rawQuery(
`
select
${columns.eventData}
from website_event w
join event_data d
on d.website_event_id = w.event_id
where w.website_id = {{websiteId::uuid}}
${where.eventData}
and w.created_at between {{startDate}} and {{endDate}}
`,
parameters.eventData,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) };
});
})
: [];
return [...urlResults, ...eventResults, ...eventDataResults];
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
goals: { type: string; value: string; goal: number; operator?: string; property?: string }[];
},
): Promise<{ type: string; value: string; goal: number; result: number }[]> {
const { startDate, endDate, goals } = criteria;
const { rawQuery } = clickhouse;
const urls = goals.filter(a => a.type === 'url');
const events = goals.filter(a => a.type === 'event');
const eventData = goals.filter(a => a.type === 'event-data');
const hasUrl = urls.length > 0;
const hasEvent = events.length > 0;
const hasEventData = eventData.length > 0;
function getParameters(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlParam = urls.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventParam = events.reduce((acc, cv, i) => {
acc[`${cv.type}${i}`] = cv.value;
return acc;
}, {});
const eventDataParam = eventData.reduce((acc, cv, i) => {
acc[`eventData${i}`] = cv.value;
acc[`property${i}`] = cv.property;
return acc;
}, {});
return {
urls: { ...urlParam, startDate, endDate, websiteId },
events: { ...eventParam, startDate, endDate, websiteId },
eventData: { ...eventDataParam, startDate, endDate, websiteId },
};
}
function getColumns(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlColumns = urls
.map((a, i) => `countIf(url_path = {url${i}:String}) AS URL${i},`)
.join('\n')
.slice(0, -1);
const eventColumns = events
.map((a, i) => `countIf(event_name = {event${i}:String}) AS EVENT${i},`)
.join('\n')
.slice(0, -1);
const eventDataColumns = eventData
.map(
(a, i) =>
`${a.operator === 'average' ? 'avg' : a.operator}If(${
a.operator !== 'count' ? 'number_value, ' : ''
}event_name = {eventData${i}:String} AND data_key = {property${i}:String}) AS EVENT_DATA${i},`,
)
.join('\n')
.slice(0, -1);
return { url: urlColumns, events: eventColumns, eventData: eventDataColumns };
}
function getWhere(
urls: { type: string; value: string; goal: number }[],
events: { type: string; value: string; goal: number }[],
eventData: {
type: string;
value: string;
goal: number;
operator?: string;
property?: string;
}[],
) {
const urlWhere = urls.map((a, i) => `{url${i}:String}`).join(',');
const eventWhere = events.map((a, i) => `{event${i}:String}`).join(',');
const eventDataNameWhere = eventData.map((a, i) => `{eventData${i}:String}`).join(',');
const eventDataKeyWhere = eventData.map((a, i) => `{property${i}:String}`).join(',');
return {
urls: `and url_path in (${urlWhere})`,
events: `and event_name in (${eventWhere})`,
eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`,
};
}
const parameters = getParameters(urls, events, eventData);
const columns = getColumns(urls, events, eventData);
const where = getWhere(urls, events, eventData);
const urlResults = hasUrl
? await rawQuery(
`
select
${columns.url}
from website_event
where website_id = {websiteId:UUID}
${where.urls}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.urls,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...urls[i], goal: Number(urls[i].goal), result: Number(results[key]) };
});
})
: [];
const eventResults = hasEvent
? await rawQuery(
`
select
${columns.events}
from website_event
where website_id = {websiteId:UUID}
${where.events}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.events,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) };
});
})
: [];
const eventDataResults = hasEventData
? await rawQuery(
`
select
${columns.eventData}
from event_data
where website_id = {websiteId:UUID}
${where.eventData}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
`,
parameters.eventData,
).then(a => {
const results = a[0];
return Object.keys(results).map((key, i) => {
return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) };
});
})
: [];
return [...urlResults, ...eventResults, ...eventDataResults];
}

View file

@ -0,0 +1,127 @@
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { QueryFilters } from '@/lib/types';
export async function getInsights(
...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
fields: { name: string; type?: string }[],
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: EVENT_TYPE.pageView,
},
{
joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),
},
);
return rawQuery(
`
select
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",
${parseFieldsByName(fields)}
from (
select
${parseFields(fields)},
website_event.session_id,
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
max(website_event.created_at) as "max_time"
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by ${parseFieldsByName(fields)},
website_event.session_id, website_event.visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
fields: { name: string; type?: string }[],
filters: QueryFilters,
): Promise<
{
x: string;
y: number;
}[]
> {
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime",
${parseFieldsByName(fields)}
from (
select
${parseFields(fields)},
session_id,
visit_id,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by ${parseFieldsByName(fields)},
session_id, visit_id
) as t
group by ${parseFieldsByName(fields)}
order by 1 desc, 2 desc
limit 500
`,
params,
);
}
function parseFields(fields: { name: any }[]) {
return fields.map(({ name }) => `${FILTER_COLUMNS[name]} as "${name}"`).join(',');
}
function parseFieldsByName(fields: { name: any }[]) {
return `${fields.map(({ name }) => name).join(',')}`;
}

View file

@ -0,0 +1,272 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
interface JourneyResult {
e1: string;
e2: string;
e3: string;
e4: string;
e5: string;
e6: string;
e7: string;
count: number;
}
export async function getJourney(
...args: [
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
): Promise<JourneyResult[]> {
const { startDate, endDate, steps, startStep, endStep } = filters;
const { rawQuery } = prisma;
const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery(
steps,
startStep,
endStep,
);
function getJourneyQuery(
steps: number,
startStep?: string,
endStep?: string,
): {
sequenceQuery: string;
startStepQuery: string;
endStepQuery: string;
params: { [key: string]: string };
} {
const params = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
// create sequence query
let selectQuery = '';
let maxQuery = '';
let groupByQuery = '';
for (let i = 1; i <= steps; i++) {
const endQuery = i < steps ? ',' : '';
selectQuery += `s.e${i},`;
maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`;
groupByQuery += `s.e${i}${endQuery} `;
}
sequenceQuery = `\nsequences as (
select ${selectQuery}
count(*) count
FROM (
select visit_id,
${maxQuery}
FROM events
group by visit_id) s
group by ${groupByQuery})
`;
// create start Step params query
if (startStep) {
startStepQuery = `and e1 = {{startStep}}`;
params['startStep'] = startStep;
}
// create end Step params query
if (endStep) {
for (let i = 1; i < steps; i++) {
const startQuery = i === 1 ? 'and (' : '\nor ';
endStepQuery += `${startQuery}(e${i} = {{endStep}} and e${i + 1} is null) `;
}
endStepQuery += `\nor (e${steps} = {{endStep}}))`;
params['endStep'] = endStep;
}
return {
sequenceQuery,
startStepQuery,
endStepQuery,
params,
};
}
return rawQuery(
`
WITH events AS (
select distinct
visit_id,
referrer_path,
coalesce(nullIf(event_name, ''), url_path) event,
row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}),
${sequenceQuery}
select *
from sequences
where 1 = 1
${startStepQuery}
${endStepQuery}
order by count desc
limit 100
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(parseResult);
}
async function clickhouseQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
steps: number;
startStep?: string;
endStep?: string;
},
): Promise<JourneyResult[]> {
const { startDate, endDate, steps, startStep, endStep } = filters;
const { rawQuery } = clickhouse;
const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery(
steps,
startStep,
endStep,
);
function getJourneyQuery(
steps: number,
startStep?: string,
endStep?: string,
): {
sequenceQuery: string;
startStepQuery: string;
endStepQuery: string;
params: { [key: string]: string };
} {
const params = {};
let sequenceQuery = '';
let startStepQuery = '';
let endStepQuery = '';
// create sequence query
let selectQuery = '';
let maxQuery = '';
let groupByQuery = '';
for (let i = 1; i <= steps; i++) {
const endQuery = i < steps ? ',' : '';
selectQuery += `s.e${i},`;
maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`;
groupByQuery += `s.e${i}${endQuery} `;
}
sequenceQuery = `\nsequences as (
select ${selectQuery}
count(*) count
FROM (
select visit_id,
${maxQuery}
FROM events
group by visit_id) s
group by ${groupByQuery})
`;
// create start Step params query
if (startStep) {
startStepQuery = `and e1 = {startStep:String}`;
params['startStep'] = startStep;
}
// create end Step params query
if (endStep) {
for (let i = 1; i < steps; i++) {
const startQuery = i === 1 ? 'and (' : '\nor ';
endStepQuery += `${startQuery}(e${i} = {endStep:String} and e${i + 1} is null) `;
}
endStepQuery += `\nor (e${steps} = {endStep:String}))`;
params['endStep'] = endStep;
}
return {
sequenceQuery,
startStepQuery,
endStepQuery,
params,
};
}
return rawQuery(
`
WITH events AS (
select distinct
visit_id,
coalesce(nullIf(event_name, ''), url_path) event,
row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number
from umami.website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}),
${sequenceQuery}
select *
from sequences
where 1 = 1
${startStepQuery}
${endStepQuery}
order by count desc
limit 100
`,
{
websiteId,
startDate,
endDate,
...params,
},
).then(parseResult);
}
function combineSequentialDuplicates(array: any) {
if (array.length === 0) return array;
const result = [array[0]];
for (let i = 1; i < array.length; i++) {
if (array[i] !== array[i - 1]) {
result.push(array[i]);
}
}
return result;
}
function parseResult(data: any) {
return data.map(({ e1, e2, e3, e4, e5, e6, e7, count }) => ({
items: combineSequentialDuplicates([e1, e2, e3, e4, e5, e6, e7]),
count: +Number(count),
}));
}

View file

@ -0,0 +1,274 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, getDatabaseType, POSTGRESQL, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getRevenue(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
): Promise<{
chart: { x: string; t: string; y: number }[];
country: { name: string; value: number }[];
total: { sum: number; count: number; unique_count: number };
table: {
currency: string;
sum: number;
count: number;
unique_count: number;
}[];
}> {
const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria;
const { getDateSQL, rawQuery } = prisma;
const db = getDatabaseType();
const like = db === POSTGRESQL ? 'ilike' : 'like';
const chartRes = await rawQuery(
`
select
we.event_name x,
${getDateSQL('ed.created_at', unit, timezone)} t,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) y
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by x, t
order by t
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
const countryRes = await rawQuery(
`
select
s.country as name,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) value
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join session s
on s.session_id = we.session_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by s.country
`,
{ websiteId, startDate, endDate, currency },
);
const totalRes = await rawQuery(
`
select
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum,
count(distinct event_id) as count,
count(distinct session_id) as unique_count
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id
from event_data
where data_key ${like} '%currency%'
and string_value = {{currency}}) currency
on currency.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
`,
{ websiteId, startDate, endDate, currency },
).then(result => result?.[0]);
const tableRes = await rawQuery(
`
select
c.currency,
sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum,
count(distinct ed.website_event_id) as count,
count(distinct we.session_id) as unique_count
from event_data ed
join website_event we
on we.event_id = ed.website_event_id
join (select website_event_id, string_value as currency
from event_data
where data_key ${like} '%currency%') c
on c.website_event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.created_at between {{startDate}} and {{endDate}}
and ed.data_key ${like} '%revenue%'
group by c.currency
order by sum desc;
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes };
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
unit: string;
timezone: string;
currency: string;
},
): Promise<{
chart: { x: string; t: string; y: number }[];
country: { name: string; value: number }[];
total: { sum: number; count: number; unique_count: number };
table: {
currency: string;
sum: number;
count: number;
unique_count: number;
}[];
}> {
const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria;
const { getDateSQL, rawQuery } = clickhouse;
const chartRes = await rawQuery<
{
x: string;
t: string;
y: number;
}[]
>(
`
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) y
from event_data
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) currency
on currency.event_id = event_data.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
group by x, t
order by t
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
const countryRes = await rawQuery<
{
name: string;
value: number;
}[]
>(
`
select
s.country as name,
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as value
from event_data ed
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) c
on c.event_id = ed.event_id
join (select distinct website_id, session_id, country
from website_event_stats_hourly
where website_id = {websiteId:UUID}) s
on ed.website_id = s.website_id
and ed.session_id = s.session_id
where ed.website_id = {websiteId:UUID}
and ed.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(ed.data_key, 'revenue') > 0
group by s.country
`,
{ websiteId, startDate, endDate, currency },
);
const totalRes = await rawQuery<{
sum: number;
avg: number;
count: number;
unique_count: number;
}>(
`
select
sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as sum,
uniqExact(event_id) as count,
uniqExact(session_id) as unique_count
from event_data
join (select event_id
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0
and string_value = {currency:String}) currency
on currency.event_id = event_data.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
`,
{ websiteId, startDate, endDate, currency },
).then(result => result?.[0]);
const tableRes = await rawQuery<
{
currency: string;
sum: number;
avg: number;
count: number;
unique_count: number;
}[]
>(
`
select
c.currency,
sum(coalesce(toDecimal64(ed.number_value, 2), toDecimal64(ed.string_value, 2))) as sum,
uniqExact(ed.event_id) as count,
uniqExact(ed.session_id) as unique_count
from event_data ed
join (select event_id, string_value as currency
from event_data
where positionCaseInsensitive(data_key, 'currency') > 0) c
on c.event_id = ed.event_id
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'revenue') > 0
group by c.currency
order by sum desc;
`,
{ websiteId, startDate, endDate, unit, timezone, currency },
);
return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes };
}

View file

@ -0,0 +1,75 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA, getDatabaseType, POSTGRESQL } from '@/lib/db';
export async function getRevenueValues(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
) {
const { rawQuery } = prisma;
const { startDate, endDate } = criteria;
const db = getDatabaseType();
const like = db === POSTGRESQL ? 'ilike' : 'like';
return rawQuery(
`
select distinct string_value as currency
from event_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and data_key ${like} '%currency%'
order by currency
`,
{
websiteId,
startDate,
endDate,
},
);
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
},
) {
const { rawQuery } = clickhouse;
const { startDate, endDate } = criteria;
return rawQuery(
`
select distinct string_value as currency
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and positionCaseInsensitive(data_key, 'currency') > 0
order by currency
`,
{
websiteId,
startDate,
endDate,
},
);
}

View file

@ -0,0 +1,102 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getUTM(
...args: [
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
) {
const { startDate, endDate } = filters;
const { rawQuery } = prisma;
return rawQuery(
`
select url_query, count(*) as "num"
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and coalesce(url_query, '') != ''
and event_type = 1
group by 1
`,
{
websiteId,
startDate,
endDate,
},
).then(result => parseParameters(result as any[]));
}
async function clickhouseQuery(
websiteId: string,
filters: {
startDate: Date;
endDate: Date;
timezone?: string;
},
) {
const { startDate, endDate } = filters;
const { rawQuery } = clickhouse;
return rawQuery(
`
select url_query, count(*) as "num"
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and url_query != ''
and event_type = 1
group by 1
`,
{
websiteId,
startDate,
endDate,
},
).then(result => parseParameters(result as any[]));
}
function parseParameters(data: any[]) {
return data.reduce((obj, { url_query, num }) => {
try {
const searchParams = new URLSearchParams(url_query);
for (const [key, value] of searchParams) {
if (key.match(/^utm_(\w+)$/)) {
const name = value;
if (!obj[key]) {
obj[key] = { [name]: Number(num) };
} else if (!obj[key][name]) {
obj[key][name] = Number(num);
} else {
obj[key][name] += Number(num);
}
}
}
} catch {
// Ignore
}
return obj;
}, {});
}

View file

@ -0,0 +1,59 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
export async function getSessionActivity(
...args: [websiteId: string, sessionId: string, startDate: Date, endDate: Date]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
sessionId: string,
startDate: Date,
endDate: Date,
) {
return prisma.client.websiteEvent.findMany({
where: {
sessionId,
websiteId,
createdAt: { gte: startDate, lte: endDate },
},
take: 500,
orderBy: { createdAt: 'desc' },
});
}
async function clickhouseQuery(
websiteId: string,
sessionId: string,
startDate: Date,
endDate: Date,
) {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
created_at as createdAt,
url_path as urlPath,
url_query as urlQuery,
referrer_domain as referrerDomain,
event_id as eventId,
event_type as eventType,
event_name as eventName,
visit_id as visitId
from website_event
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
order by created_at desc
limit 500
`,
{ websiteId, sessionId, startDate, endDate },
);
}

View file

@ -0,0 +1,56 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
export async function getSessionData(...args: [websiteId: string, sessionId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, sessionId: string) {
const { rawQuery } = prisma;
return rawQuery(
`
select
website_id as "websiteId",
session_id as "sessionId",
data_key as "dataKey",
data_type as "dataType",
replace(string_value, '.0000', '') as "stringValue",
number_value as "numberValue",
date_value as "dateValue",
created_at as "createdAt"
from session_data
where website_id = {{websiteId::uuid}}
and session_id = {{sessionId::uuid}}
order by data_key asc
`,
{ websiteId, sessionId },
);
}
async function clickhouseQuery(websiteId: string, sessionId: string) {
const { rawQuery } = clickhouse;
return rawQuery(
`
select
website_id as websiteId,
session_id as sessionId,
data_key as dataKey,
data_type as dataType,
replace(string_value, '.0000', '') as stringValue,
number_value as numberValue,
date_value as dateValue,
created_at as createdAt
from session_data final
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
order by data_key asc
`,
{ websiteId, sessionId },
);
}

View file

@ -0,0 +1,70 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getSessionDataProperties(
...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
) {
const { rawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
data_key as "propertyName",
count(distinct d.session_id) as "total"
from website_event e
join session_data d
on d.session_id = e.session_id
where e.website_id = {{websiteId::uuid}}
and e.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by 1
order by 2 desc
limit 500
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
): Promise<{ propertyName: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { propertyName: 'data_key' },
});
return rawQuery(
`
select
data_key as propertyName,
count(distinct d.session_id) as total
from website_event e
join session_data d final
on d.session_id = e.session_id
where e.website_id = {websiteId:UUID}
and e.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and d.data_key != ''
${filterQuery}
group by 1
order by 2 desc
limit 500
`,
params,
);
}

View file

@ -0,0 +1,73 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import { QueryFilters, WebsiteEventData } from '@/lib/types';
export async function getSessionDataValues(
...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }]
): Promise<WebsiteEventData[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
) {
const { rawQuery, parseFilters, getDateSQL } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
case
when data_type = 2 then replace(string_value, '.0000', '')
when data_type = 4 then ${getDateSQL('date_value', 'hour')}
else string_value
end as "value",
count(distinct d.session_id) as "total"
from website_event e
join session_data d
on d.session_id = e.session_id
where e.website_id = {{websiteId::uuid}}
and e.created_at between {{startDate}} and {{endDate}}
and d.data_key = {{propertyName}}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters & { propertyName?: string },
): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
multiIf(data_type = 2, replaceAll(string_value, '.0000', ''),
data_type = 4, toString(date_trunc('hour', date_value)),
string_value) as "value",
uniq(d.session_id) as "total"
from website_event e
join session_data d final
on d.session_id = e.session_id
where e.website_id = {websiteId:UUID}
and e.created_at between {startDate:DateTime64} and {endDate:DateTime64}
and d.data_key = {propertyName:String}
${filterQuery}
group by value
order by 2 desc
limit 100
`,
params,
);
}

View file

@ -0,0 +1,119 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getSessionMetrics(
...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number | string,
offset?: number | string,
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
) {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(
websiteId,
{
...filters,
eventType: EVENT_TYPE.pageView,
},
{
joinSession: SESSION_COLUMNS.includes(type),
},
);
const includeCountry = column === 'city' || column === 'subdivision1';
return rawQuery(
`
select
${column} x,
count(distinct website_event.session_id) y
${includeCountry ? ', country' : ''}
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = {{eventType}}
${filterQuery}
group by 1
${includeCountry ? ', 3' : ''}
order by 2 desc
limit ${limit}
offset ${offset}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
type: string,
filters: QueryFilters,
limit: number | string = 500,
offset: number | string = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
sql = `
select
${column} x,
count(distinct session_id) y
${includeCountry ? ', country' : ''}
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
order by y desc
limit ${limit}
offset ${offset}
`;
} else {
sql = `
select
${column} x,
uniq(session_id) y
${includeCountry ? ', country' : ''}
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
${includeCountry ? ', country' : ''}
order by y desc
limit ${limit}
offset ${offset}
`;
}
return rawQuery(sql, params);
}

View file

@ -0,0 +1,92 @@
import clickhouse from '@/lib/clickhouse';
import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc', unit = 'day' } = filters;
const { getDateSQL, parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
return rawQuery(
`
select
${getDateSQL('website_event.created_at', unit, timezone)} x,
count(distinct website_event.session_id) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
order by 1
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'utc', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateSQL } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
let sql = '';
if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
count(distinct session_id) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
} else {
sql = `
select
g.t as x,
g.y as y
from (
select
${getDateSQL('website_event.created_at', unit, timezone)} as t,
uniq(session_id) as y
from website_event_stats_hourly website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`;
}
return rawQuery(sql, params);
}

View file

@ -0,0 +1,109 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
export async function getWebsiteSession(...args: [websiteId: string, sessionId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, sessionId: string) {
const { rawQuery, getTimestampDiffSQL } = prisma;
return rawQuery(
`
select id,
website_id as "websiteId",
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
min(min_time) as "firstAt",
max(max_time) as "lastAt",
count(distinct visit_id) as visits,
sum(views) as views,
sum(events) as events,
sum(${getTimestampDiffSQL('min_time', 'max_time')}) as "totaltime"
from (select
session.session_id as id,
website_event.visit_id,
session.website_id,
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city,
min(website_event.created_at) as min_time,
max(website_event.created_at) as max_time,
sum(case when website_event.event_type = 1 then 1 else 0 end) as views,
sum(case when website_event.event_type = 2 then 1 else 0 end) as events
from session
join website_event on website_event.session_id = session.session_id
where session.website_id = {{websiteId::uuid}}
and session.session_id = {{sessionId::uuid}}
group by session.session_id, visit_id, session.website_id, session.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.subdivision1, session.city) t
group by id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
}
async function clickhouseQuery(websiteId: string, sessionId: string) {
const { rawQuery, getDateStringSQL } = clickhouse;
return rawQuery(
`
select id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
uniq(visit_id) visits,
sum(views) as views,
sum(events) as events,
sum(max_time-min_time) as totaltime
from (select
session_id as id,
visit_id,
website_id as websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
min(min_time) as min_time,
max(max_time) as max_time,
sum(views) as views,
length(groupArrayArray(event_name)) as events
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
group by session_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city) t
group by id, websiteId, hostname, browser, os, device, screen, language, country, subdivision1, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
}

View file

@ -0,0 +1,72 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { QueryFilters } from '@/lib/types';
export async function getWebsiteSessionStats(
...args: [websiteId: string, filters: QueryFilters]
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
const { parseFilters, rawQuery } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return rawQuery(
`
select
count(*) as "pageviews",
count(distinct website_event.session_id) as "visitors",
count(distinct website_event.visit_id) as "visits",
count(distinct session.country) as "countries",
sum(case when website_event.event_type = 2 then 1 else 0 end) as "events"
from website_event
join session on website_event.session_id = session.session_id
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
filters: QueryFilters,
): Promise<
{ pageviews: number; visitors: number; visits: number; countries: number; events: number }[]
> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return rawQuery(
`
select
sum(views) as "pageviews",
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
uniq(country) as "countries",
sum(length(event_name)) as "events"
from umami.website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
`,
params,
);
}

View file

@ -0,0 +1,102 @@
import clickhouse from '@/lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import { PageParams, QueryFilters } from '@/lib/types';
export async function getWebsiteSessions(
...args: [websiteId: string, filters?: QueryFilters, pageParams?: PageParams]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams: PageParams) {
const { pagedRawQuery, parseFilters } = prisma;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
});
return pagedRawQuery(
`
with sessions as (
select
session.session_id as "id",
session.website_id as "websiteId",
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city,
min(website_event.created_at) as "firstAt",
max(website_event.created_at) as "lastAt",
count(distinct website_event.visit_id) as "visits",
sum(case when website_event.event_type = 1 then 1 else 0 end) as "views",
max(website_event.created_at) as "createdAt"
from website_event
join session on session.session_id = website_event.session_id
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by session.session_id,
session.website_id,
session.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.city
order by max(website_event.created_at) desc
limit 1000)
select * from sessions
`,
params,
pageParams,
);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) {
const { pagedQuery, parseFilters, getDateStringSQL } = clickhouse;
const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters);
return pagedQuery(
`
with sessions as (
select
session_id as id,
website_id as websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
uniq(visit_id) as visits,
sumIf(views, event_type = 1) as views,
lastAt as createdAt
from website_event_stats_hourly
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
group by session_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city
order by lastAt desc
limit 1000)
select * from sessions
`,
params,
pageParams,
);
}

View file

@ -0,0 +1,71 @@
import prisma from '@/lib/prisma';
import clickhouse from '@/lib/clickhouse';
import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db';
import { QueryFilters } from '@/lib/types';
export async function getWebsiteSessionsWeekly(
...args: [websiteId: string, filters?: QueryFilters]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc' } = filters;
const { rawQuery, getDateWeeklySQL, parseFilters } = prisma;
const { params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
${getDateWeeklySQL('created_at', timezone)} as time,
count(distinct session_id) as value
from website_event
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by time
order by 2
`,
params,
).then(formatResults);
}
async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const { timezone = 'utc' } = filters;
const { rawQuery, parseFilters } = clickhouse;
const { params } = await parseFilters(websiteId, filters);
return rawQuery(
`
select
formatDateTime(toDateTime(created_at, '${timezone}'), '%w:%H') as time,
count(distinct session_id) as value
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by time
order by time
`,
params,
).then(formatResults);
}
function formatResults(data: any) {
const days = [];
for (let i = 0; i < 7; i++) {
days.push([]);
for (let j = 0; j < 24; j++) {
days[i].push(
Number(
data.find(({ time }) => time === `${i}:${j.toString().padStart(2, '0')}`)?.value || 0,
),
);
}
}
return days;
}

View file

@ -0,0 +1,109 @@
import { DATA_TYPE } from '@/lib/constants';
import { uuid } from '@/lib/crypto';
import { flattenJSON, getStringValue } from '@/lib/data';
import prisma from '@/lib/prisma';
import { DynamicData } from '@/lib/types';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import kafka from '@/lib/kafka';
import clickhouse from '@/lib/clickhouse';
export async function saveSessionData(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
return runQuery({
[PRISMA]: () => relationalQuery(data),
[CLICKHOUSE]: () => clickhouseQuery(data),
});
}
export async function relationalQuery(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
const { client } = prisma;
const { websiteId, sessionId, sessionData } = data;
const jsonKeys = flattenJSON(sessionData);
const flattenedData = jsonKeys.map(a => ({
id: uuid(),
websiteId,
sessionId,
dataKey: a.key,
stringValue: getStringValue(a.value, a.dataType),
numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
dataType: a.dataType,
}));
const existing = await client.sessionData.findMany({
where: {
sessionId,
},
select: {
id: true,
sessionId: true,
dataKey: true,
},
});
for (const data of flattenedData) {
const { sessionId, dataKey, ...props } = data;
const record = existing.find(e => e.sessionId === sessionId && e.dataKey === dataKey);
if (record) {
await client.sessionData.update({
where: {
id: record.id,
},
data: {
...props,
},
});
} else {
await client.sessionData.create({
data,
});
}
}
return flattenedData;
}
async function clickhouseQuery(data: {
websiteId: string;
sessionId: string;
sessionData: DynamicData;
}) {
const { websiteId, sessionId, sessionData } = data;
const { insert, getUTCString } = clickhouse;
const { sendMessage } = kafka;
const createdAt = getUTCString();
const jsonKeys = flattenJSON(sessionData);
const messages = jsonKeys.map(({ key, value, dataType }) => {
return {
website_id: websiteId,
session_id: sessionId,
data_key: key,
data_type: dataType,
string_value: getStringValue(value, dataType),
number_value: dataType === DATA_TYPE.number ? value : null,
date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
created_at: createdAt,
};
});
if (kafka.enabled) {
await sendMessage('session_data', messages);
} else {
await insert('session_data', messages);
}
return data;
}

View file

@ -7,9 +7,9 @@ import {
LOCALE_CONFIG,
THEME_CONFIG,
TIMEZONE_CONFIG,
} from 'lib/constants';
import { getItem } from 'next-basics';
import { getTimezone } from 'lib/date';
} from '@/lib/constants';
import { getItem } from '@/lib/storage';
import { getTimezone } from '@/lib/date';
function getDefaultTheme() {
return typeof window !== 'undefined'

View file

@ -1,12 +1,14 @@
import { create } from 'zustand';
import { DASHBOARD_CONFIG, DEFAULT_WEBSITE_LIMIT } from 'lib/constants';
import { getItem, setItem } from 'next-basics';
import { DASHBOARD_CONFIG, DEFAULT_WEBSITE_LIMIT } from '@/lib/constants';
import { getItem, setItem } from '@/lib/storage';
export const initialState = {
showCharts: true,
limit: DEFAULT_WEBSITE_LIMIT,
websiteOrder: [],
websiteActive: [],
editing: false,
isEdited: false,
};
const store = create(() => ({ ...initialState, ...getItem(DASHBOARD_CONFIG) }));

View file

@ -1,8 +1,8 @@
import { create } from 'zustand';
import { produce } from 'immer';
import semver from 'semver';
import { CURRENT_VERSION, VERSION_CHECK, UPDATES_URL } from 'lib/constants';
import { getItem } from 'next-basics';
import { CURRENT_VERSION, VERSION_CHECK, UPDATES_URL } from '@/lib/constants';
import { getItem } from '@/lib/storage';
const initialState = {
current: CURRENT_VERSION,

View file

@ -1,6 +1,6 @@
import { create } from 'zustand';
import { produce } from 'immer';
import { DateRange } from 'lib/types';
import { DateRange } from '@/lib/types';
const store = create(() => ({}));
@ -18,4 +18,18 @@ export function setWebsiteDateRange(websiteId: string, dateRange: DateRange) {
);
}
export function setWebsiteDateCompare(websiteId: string, dateCompare: string) {
store.setState(
produce(state => {
if (!state[websiteId]) {
state[websiteId] = {};
}
state[websiteId].dateCompare = dateCompare;
return state;
}),
);
}
export default store;

View file

@ -1,29 +0,0 @@
.zh-CN {
font-family: '方体', 'PingFang SC', '黑体', 'Heiti SC', 'Microsoft JhengHei UI',
'Microsoft JhengHei', Roboto, Noto, 'Noto Sans CJK SC', sans-serif !important;
}
.zh-TW {
font-family: '方體', 'PingFang TC', '黑體', 'Heiti TC', 'Microsoft JhengHei UI',
'Microsoft JhengHei', Roboto, Noto, 'Noto Sans CJK TC', sans-serif !important;
}
.ja-JP {
font-family: '游ゴシック体', YuGothic, 'ヒラギノ丸ゴ', 'Hiragino Sans', 'Yu Gothic UI',
'Meiryo UI', 'MS Gothic', Roboto, Noto, 'Noto Sans CJK JP', sans-serif !important;
}
.ko-KR {
font-family: 'Nanum Gothic', 'Apple SD Gothic Neo', 'Malgun Gothic', Roboto, Noto,
'Noto Sans CJK KR', sans-serif !important;
}
.ar-SA {
font-family: 'Geeza Pro', 'Arabic Typesetting', Roboto, Noto, 'Noto Naskh Arabic',
'Times New Roman', serif !important;
}
.he-IL {
font-family: 'New Peninim MT', 'Arial Hebrew', Gisha, 'Times New Roman', Roboto, Noto,
'Noto Sans Hebrew', sans-serif !important;
}

View file

@ -3,12 +3,13 @@
screen: { width, height },
navigator: { language },
location,
localStorage,
document,
history,
top,
} = window;
const { hostname, pathname, search } = location;
const { hostname, href, origin } = location;
const { currentScript, referrer } = document;
const localStorage = href.startsWith('data:') ? undefined : window.localStorage;
if (!currentScript) return;
@ -21,6 +22,7 @@
const tag = attr(_data + 'tag');
const autoTrack = attr(_data + 'auto-track') !== _false;
const excludeSearch = attr(_data + 'exclude-search') === _true;
const excludeHash = attr(_data + 'exclude-hash') === _true;
const domain = attr(_data + 'domains') || '';
const domains = domain.split(',').map(n => n.trim());
const host =
@ -33,36 +35,14 @@
/* Helper functions */
const encode = str => {
if (!str) {
return undefined;
}
try {
const result = decodeURI(str);
if (result !== str) {
return result;
}
} catch {
return str;
}
return encodeURI(str);
};
const parseURL = url => {
return excludeSearch ? url.split('?')[0] : url;
};
const getPayload = () => ({
website,
hostname,
screen,
language,
title: encode(title),
url: encode(currentUrl),
referrer: encode(currentRef),
title,
hostname,
url: currentUrl,
referrer: currentRef,
tag: tag ? tag : undefined,
});
@ -72,7 +52,17 @@
if (!url) return;
currentRef = currentUrl;
currentUrl = parseURL(url.toString());
currentUrl = new URL(url, location.href);
if (excludeSearch) {
currentUrl.search = '';
}
if (excludeHash) {
currentUrl.hash = '';
}
currentUrl = currentUrl.toString();
if (currentUrl !== currentRef) {
setTimeout(track, delayDuration);
@ -169,7 +159,9 @@
e.preventDefault();
}
return trackElement(parentElement).then(() => {
if (!external) location.href = href;
if (!external) {
(target === '_top' ? top.location : location).href = href;
}
});
}
} else if (parentElement.tagName === 'BUTTON') {
@ -187,31 +179,51 @@
/* Tracking functions */
const trackingDisabled = () =>
disabled ||
!website ||
(localStorage && localStorage.getItem('umami.disabled')) ||
(domain && !domains.includes(hostname));
const send = async (payload, type = 'event') => {
if (trackingDisabled()) return;
const headers = {
'Content-Type': 'application/json',
};
if (typeof cache !== 'undefined') {
headers['x-umami-cache'] = cache;
}
try {
const res = await fetch(endpoint, {
method: 'POST',
body: JSON.stringify({ type, payload }),
headers,
credentials: 'omit',
});
const text = await res.text();
return (cache = text);
} catch {
const data = await res.json();
if (data) {
disabled = !!data.disabled;
cache = data.cache;
}
} catch (e) {
/* empty */
}
};
const init = () => {
if (!initialized) {
track();
handlePathChanges();
handleTitleChanges();
handleClicks();
initialized = true;
}
};
const track = (obj, data) => {
if (typeof obj === 'string') {
return send({
@ -238,26 +250,18 @@
};
}
let currentUrl = `${pathname}${search}`;
let currentRef = referrer !== hostname ? referrer : '';
let currentUrl = href;
let currentRef = referrer.startsWith(origin) ? '' : referrer;
let title = document.title;
let cache;
let initialized;
let disabled = false;
if (autoTrack && !trackingDisabled()) {
handlePathChanges();
handleTitleChanges();
handleClicks();
const init = () => {
if (document.readyState === 'complete' && !initialized) {
track();
initialized = true;
}
};
document.addEventListener('readystatechange', init, true);
init();
if (document.readyState === 'complete') {
init();
} else {
document.addEventListener('readystatechange', init, true);
}
}
})(window);