Merge branch 'dev' into bug/um-362-relational-funnels-query

This commit is contained in:
Francis Cao 2023-07-27 13:53:49 -07:00
commit a03574e8d4
78 changed files with 1110 additions and 852 deletions

View file

@ -1,7 +1,7 @@
import { Prisma, Team, TeamWebsite } from '@prisma/client';
import prisma from 'lib/prisma';
import { uuid } from 'lib/crypto';
import { ROLES } from 'lib/constants';
import { uuid } from 'next-basics';
export async function getTeam(where: Prisma.TeamWhereInput): Promise<Team> {
return prisma.client.team.findFirst({

View file

@ -1,5 +1,5 @@
import { Prisma, TeamUser } from '@prisma/client';
import { uuid } from 'lib/crypto';
import { uuid } from 'next-basics';
import prisma from 'lib/prisma';
export async function getTeamUserById(teamUserId: string): Promise<TeamUser> {

View file

@ -1,6 +1,6 @@
import { Prisma, Team, TeamUser, TeamWebsite, Website } from '@prisma/client';
import { ROLES } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { uuid } from 'next-basics';
import prisma from 'lib/prisma';
export async function getTeamWebsite(

View file

@ -0,0 +1,120 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getEventDataEvents(
...args: [
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
]
): Promise<WebsiteEventDataFields[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { field, event } = filters;
if (event) {
return rawQuery(
`
select
we.event_name as event,
ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId:uuid}}
and ed.event_key = {{field}}
and ed.created_at between {{startDate}} and {{endDate}}
and we.event_name = {{event}}
group by ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
{ ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
return rawQuery(
`
select
we.event_name as event,
ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
inner join website_event as we
on we.event_id = ed.website_event_id
where ed.website_id = {{websiteId::uuid}}
and ed.event_key = {{field}}
and ed.created_at between {{startDate}} and {{endDate}}
group by we.event_name, ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
async function clickhouseQuery(
websiteId: string,
startDate: Date,
endDate: Date,
filters: { field?: string; event?: string },
) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { event } = filters;
if (event) {
return rawQuery(
`
select
event_name as event,
event_key as field,
data_type as type,
string_value as value,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_name = {event:String}
group by event_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100
`,
{ ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
return rawQuery(
`
select
event_name as event,
event_key as field,
data_type as type,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, data_type, event_name
order by 1 asc, 2 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}

View file

@ -2,18 +2,11 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { WebsiteEventDataFields } from 'lib/types';
import { loadWebsite } from 'lib/query';
import { DEFAULT_CREATED_AT } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getEventDataFields(
...args: [
websiteId: string,
startDate: Date,
endDate: Date,
field?: string,
event?: string,
withEventNames?: boolean,
]
...args: [websiteId: string, startDate: Date, endDate: Date, field?: string]
): Promise<WebsiteEventDataFields[]> {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
@ -21,168 +14,82 @@ export async function getEventDataFields(
});
}
async function relationalQuery(
websiteId: string,
startDate: Date,
endDate: Date,
field: string,
event: string,
withEventNames: boolean,
) {
const { toUuid, rawQuery } = prisma;
async function relationalQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
if (field) {
if (event) {
return rawQuery(
`select ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
join website_event as e on e.event_id = ed.website_event_id
where ed.website_id = $1${toUuid()}
and ed.event_key = $2
and ed.created_at >= $3
and ed.created_at between $4 and $5
and e.event_name = $6
group by ed.event_key, ed.string_value
order by 3 desc, 2 desc, 1 asc
`,
[websiteId, field, resetDate, startDate, endDate, event] as any,
);
}
return rawQuery(
`select event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = $1${toUuid()}
and event_key = $2
and created_at >= $3
and created_at between $4 and $5
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
`,
[websiteId, field, resetDate, startDate, endDate] as any,
);
}
if (withEventNames) {
return rawQuery(
`select
ed.event_key as field,
ed.data_type as type,
count(ed.*) as total,
e.event_name as event
from event_data as ed
join website_event as e on e.event_id = ed.website_event_id
where ed.website_id = $1${toUuid()}
and ed.created_at >= $2
and ed.created_at between $3 and $4
group by e.event_name, ed.event_key, ed.data_type
order by 3 desc, 2 asc, 1 asc
`,
[websiteId, resetDate, startDate, endDate] as any,
);
}
return rawQuery(
`select
event_key as field,
data_type as type,
count(*) as total
from event_data
where website_id = $1${toUuid()}
and created_at >= $2
and created_at between $3 and $4
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
`,
[websiteId, resetDate, startDate, endDate] as any,
);
}
async function clickhouseQuery(
websiteId: string,
startDate: Date,
endDate: Date,
field: string,
event: string,
withEventNames: boolean,
) {
const { rawQuery, getDateFormat, getBetweenDates } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
if (field) {
if (event) {
return rawQuery(
`select
ed.event_key as field,
ed.string_value as value,
count(ed.*) as total
from event_data as ed
join website_event as e on e.event_id = ed.website_event_id
where ed.website_id = {websiteId:UUID}
and ed.event_key = {field:String}
and ed.created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('ed.created_at', startDate, endDate)}
and e.event_name = {event:String}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
`,
{ websiteId, field, event },
);
}
return rawQuery(
`select
`
select
event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and event_key = {field:String}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
`,
{ websiteId, field },
);
}
if (withEventNames) {
return rawQuery(
`select
ed.event_key as field,
ed.data_type as type,
count(ed.*) as total,
e.event_name as event
from event_data as ed
join website_event as e on e.event_id = ed.website_event_id
where ed.website_id = {websiteId:UUID}
and ed.created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('ed.created_at', startDate, endDate)}
group by e.event_name, ed.event_key, ed.data_type
order by 3 desc, 2 asc, 1 asc
from event_data
where website_id = {{websiteId::uuid}}
and event_key = {{field}}
and created_at between {{startDate}} and {{endDate}}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
[websiteId, resetDate, startDate, endDate] as any,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
return rawQuery(
`select
event_key as field,
data_type as type,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
`
select
event_key as field,
data_type as type,
count(*) as total
from event_data
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId },
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
async function clickhouseQuery(websiteId: string, startDate: Date, endDate: Date, field: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
if (field) {
return rawQuery(
`
select
event_key as field,
string_value as value,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and event_key = {field:String}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
{ websiteId, field, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
return rawQuery(
`
select
event_key as field,
data_type as type,
count(*) as total
from event_data
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
group by event_key, data_type
order by 3 desc, 2 asc, 1 asc
limit 100
`,
{ websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}

View file

@ -1,28 +1,26 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';
export function getEventDataUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websiteIds: string[], startDate: Date, endDate: Date) {
throw new Error('Not Implemented');
}
function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) {
const { rawQuery } = clickhouse;
return rawQuery(
`select
website_id as websiteId,
count(*) as count
`
select
website_id as websiteId,
count(*) as count
from event_data
where created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_id in {websiteIds:Array(UUID)}
group by website_id`,
and website_id in {websiteIds:Array(UUID)}
group by website_id
`,
{
websiteIds,
startDate,

View file

@ -1,6 +1,6 @@
import { Prisma } from '@prisma/client';
import { DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { uuid } from 'next-basics';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { flattenJSON } from 'lib/dynamicData';
import kafka from 'lib/kafka';
@ -31,7 +31,7 @@ async function relationalQuery(data: {
const jsonKeys = flattenJSON(eventData);
//id, websiteEventId, eventStringValue
// id, websiteEventId, eventStringValue
const flattendData = jsonKeys.map(a => ({
id: uuid(),
websiteEventId: eventId,

View file

@ -2,8 +2,9 @@ import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { WebsiteEventMetric } from 'lib/types';
import { DEFAULT_CREATED_AT, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/query';
import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getEventMetrics(
...args: [
@ -45,26 +46,31 @@ async function relationalQuery(
};
},
) {
const { toUuid, rawQuery, getDateQuery, getFilterQuery } = prisma;
const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [websiteId, resetDate, startDate, endDate];
const filterQuery = getFilterQuery(filters, params);
const filterQuery = getFilterQuery(filters);
return rawQuery(
`select
`
select
event_name x,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
from website_event
where website_id = $1${toUuid()}
and created_at >= $2
and created_at between $3 and $4
and event_type = ${EVENT_TYPE.customEvent}
where website_id = {{websiteId::uuid}}
and created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1, 2
order by 2`,
params,
order by 2
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
},
);
}
@ -87,24 +93,30 @@ async function clickhouseQuery(
};
},
) {
const { rawQuery, getDateQuery, getDateFormat, getBetweenDates, getFilterQuery } = clickhouse;
const { rawQuery, getDateQuery, getFilterQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
const filterQuery = getFilterQuery(filters);
return rawQuery(
`select
`
select
event_name x,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
from website_event
where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.customEvent}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${getFilterQuery(filters, params)}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by x, t
order by t`,
params,
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.customEvent,
},
);
}

View file

@ -1,28 +1,26 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import { CLICKHOUSE, PRISMA, runQuery, notImplemented } from 'lib/db';
export function getEventUsage(...args: [websiteIds: string[], startDate: Date, endDate: Date]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[PRISMA]: notImplemented,
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websiteIds: string[], startDate: Date, endDate: Date) {
throw new Error('Not Implemented');
}
function clickhouseQuery(websiteIds: string[], startDate: Date, endDate: Date) {
const { rawQuery } = clickhouse;
return rawQuery(
`select
website_id as websiteId,
count(*) as count
`
select
website_id as websiteId,
count(*) as count
from website_event
where created_at between {startDate:DateTime64} and {endDate:DateTime64}
and website_id in {websiteIds:Array(UUID)}
group by website_id`,
where website_id in {websiteIds:Array(UUID)}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by website_id
`,
{
websiteIds,
startDate,

View file

@ -2,30 +2,31 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
export function getEvents(...args: [websiteId: string, startAt: Date, eventType: number]) {
export function getEvents(...args: [websiteId: string, startDate: Date, eventType: number]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websiteId: string, startAt: Date, eventType: number) {
function relationalQuery(websiteId: string, startDate: Date, eventType: number) {
return prisma.client.websiteEvent.findMany({
where: {
websiteId,
eventType,
createdAt: {
gte: startAt,
gte: startDate,
},
},
});
}
function clickhouseQuery(websiteId: string, startAt: Date, eventType: number) {
function clickhouseQuery(websiteId: string, startDate: Date, eventType: number) {
const { rawQuery } = clickhouse;
return rawQuery(
`select
`
select
event_id as id,
website_id as websiteId,
session_id as sessionId,
@ -35,12 +36,13 @@ function clickhouseQuery(websiteId: string, startAt: Date, eventType: number) {
referrer_domain as referrerDomain,
event_name as eventName
from website_event
where event_type = {eventType:UInt32}
and website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`,
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime}
and event_type = {eventType:UInt32}
`,
{
websiteId,
startAt,
startDate,
eventType,
},
);

View file

@ -2,7 +2,7 @@ import { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import { uuid } from 'lib/crypto';
import { uuid } from 'next-basics';
import { saveEventData } from 'queries/analytics/eventData/saveEventData';
export async function saveEvent(args: {

View file

@ -1,120 +0,0 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { DEFAULT_CREATED_AT, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/query';
export async function getPageviewStats(
...args: [
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
count?: string;
filters: object;
sessionKey?: string;
},
]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
count?: string;
filters: object;
sessionKey?: string;
},
) {
const {
startDate,
endDate,
timezone = 'utc',
unit = 'day',
count = '*',
filters = {},
sessionKey = 'session_id',
} = criteria;
const { toUuid, getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
return rawQuery(
`select ${getDateQuery('website_event.created_at', unit, timezone)} x,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from website_event
${joinSession}
where website_event.website_id = $1${toUuid()}
and website_event.created_at >= $2
and website_event.created_at between $3 and $4
and event_type = ${EVENT_TYPE.pageView}
${filterQuery}
group by 1`,
params,
);
}
async function clickhouseQuery(
websiteId: string,
criteria: {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
count?: string;
filters: object;
sessionKey?: string;
},
) {
const {
startDate,
endDate,
timezone = 'UTC',
unit = 'day',
count = '*',
filters = {},
} = criteria;
const {
parseFilters,
getDateFormat,
rawQuery,
getDateStringQuery,
getDateQuery,
getBetweenDates,
} = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
const { filterQuery } = parseFilters(filters, params);
return rawQuery(
`select
${getDateStringQuery('g.t', unit)} as x,
g.y as y
from
(select
${getDateQuery('created_at', unit, timezone)} t,
count(${count !== '*' ? 'distinct session_id' : count}) y
from website_event
where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.pageView}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${filterQuery}
group by t) g
order by t`,
params,
);
}

View file

@ -1,8 +1,9 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { DEFAULT_CREATED_AT, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/query';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getPageviewMetrics(
...args: [
@ -31,39 +32,40 @@ async function relationalQuery(
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters, toUuid } = prisma;
const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [
const params: any = {
websiteId,
resetDate,
startDate,
startDate: maxDate(startDate, website.resetAt),
endDate,
column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
];
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
};
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = 'and (website_event.referrer_domain != $6 or website_event.referrer_domain is null)';
params.push(website.domain);
excludeDomain =
'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)';
params.domain = website.domain;
}
const { filterQuery, joinSession } = parseFilters(filters, params);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
`select ${column} x, count(*) y
`
select ${column} x, count(*) y
from website_event
${joinSession}
where website_event.website_id = $1${toUuid()}
and website_event.created_at >= $2
and website_event.created_at between $3 and $4
and event_type = $5
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${excludeDomain}
${filterQuery}
group by 1
order by 2 desc
limit 100`,
limit 100
`,
params,
);
}
@ -78,11 +80,12 @@ async function clickhouseQuery(
},
) {
const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, getDateFormat, parseFilters, getBetweenDates } = clickhouse;
const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = {
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: undefined,
};
@ -97,17 +100,18 @@ async function clickhouseQuery(
const { filterQuery } = parseFilters(filters, params);
return rawQuery(
`select ${column} x, count(*) y
`
select ${column} x, count(*) y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${excludeDomain}
${filterQuery}
group by x
order by y desc
limit 100`,
limit 100
`,
params,
);
}

View file

@ -0,0 +1,103 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export interface PageviewStatsCriteria {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
count?: string;
filters: object;
sessionKey?: string;
}
export async function getPageviewStats(
...args: [websiteId: string, criteria: PageviewStatsCriteria]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const {
startDate,
endDate,
timezone = 'utc',
unit = 'day',
count = '*',
filters = {},
sessionKey = 'session_id',
} = criteria;
const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
`
select
${getDateQuery('website_event.created_at', unit, timezone)} x,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}
async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const {
startDate,
endDate,
timezone = 'UTC',
unit = 'day',
count = '*',
filters = {},
} = criteria;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
return rawQuery(
`
select
${getDateStringQuery('g.t', unit)} as x,
g.y as y
from (
select
${getDateQuery('created_at', unit, timezone)} as t,
count(${count !== '*' ? 'distinct session_id' : count}) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}

View file

@ -2,7 +2,7 @@ import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
export async function getPageviewFunnel(
export async function getFunnel(
...args: [
websiteId: string,
criteria: {
@ -49,8 +49,8 @@ async function relationalQuery(
`WITH level1 AS (
select distinct session_id, created_at
from website_event
where website_id = $1${toUuid()}
and created_at between $2 and $3
where website_id = {{websiteId}}${toUuid()}
and created_at between {{startDate}} and {{endDate}}
and url_path = $4)
${levelQuery}
${sumQuery}
@ -81,11 +81,13 @@ async function clickhouseQuery(
}[]
> {
const { windowMinutes, startDate, endDate, urls } = criteria;
const { rawQuery, getBetweenDates, getFunnelQuery } = clickhouse;
const { rawQuery, getFunnelQuery } = clickhouse;
const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getFunnelQuery(urls, windowMinutes);
const params = {
websiteId,
startDate,
endDate,
...urlParams,
};
@ -96,7 +98,7 @@ async function clickhouseQuery(
from umami.website_event
where url_path in (${urlFilterQuery})
and website_id = {websiteId:UUID}
and ${getBetweenDates('created_at', startDate, endDate)}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
), level1 AS (
select *
from level0

View file

@ -0,0 +1,42 @@
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
export interface GetInsightsCriteria {
startDate: Date;
endDate: Date;
fields: string[];
filters: string[];
groups: string[];
}
export async function getInsights(...args: [websiteId: string, criteria: GetInsightsCriteria]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(
websiteId: string,
criteria: GetInsightsCriteria,
): Promise<
{
x: string;
y: number;
}[]
> {
return null;
}
async function clickhouseQuery(
websiteId: string,
criteria: GetInsightsCriteria,
): Promise<
{
x: string;
y: number;
}[]
> {
return null;
}

View file

@ -1,8 +1,9 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { DEFAULT_CREATED_AT, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/query';
import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getSessionMetrics(
...args: [
@ -21,11 +22,9 @@ async function relationalQuery(
criteria: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const { startDate, endDate, column, filters = {} } = criteria;
const { toUuid, parseFilters, rawQuery } = prisma;
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
const { parseFilters, rawQuery } = prisma;
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
`select ${column} x, count(*) y
@ -36,15 +35,14 @@ async function relationalQuery(
join website
on website_event.website_id = website.website_id
${joinSession}
where website.website_id = $1${toUuid()}
and website_event.created_at >= $2
and website_event.created_at between $3 and $4
where website.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
)
group by 1
order by 2 desc
limit 100`,
params,
{ ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate },
);
}
@ -53,23 +51,29 @@ async function clickhouseQuery(
data: { startDate: Date; endDate: Date; column: string; filters: object },
) {
const { startDate, endDate, column, filters = {} } = data;
const { getDateFormat, parseFilters, getBetweenDates, rawQuery } = clickhouse;
const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
const { filterQuery } = parseFilters(filters, params);
const { filterQuery } = parseFilters(filters);
return rawQuery(
`select ${column} x, count(distinct session_id) y
`
select
${column} x, count(distinct session_id) y
from website_event as x
where website_id = {websiteId:UUID}
and event_type = ${EVENT_TYPE.pageView}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
order by y desc
limit 100`,
params,
limit 100
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}

View file

@ -9,22 +9,23 @@ export async function getSessions(...args: [websiteId: string, startAt: Date]) {
});
}
async function relationalQuery(websiteId: string, startAt: Date) {
async function relationalQuery(websiteId: string, startDate: Date) {
return prisma.client.session.findMany({
where: {
websiteId,
createdAt: {
gte: startAt,
gte: startDate,
},
},
});
}
async function clickhouseQuery(websiteId: string, startAt: Date) {
async function clickhouseQuery(websiteId: string, startDate: Date) {
const { rawQuery } = clickhouse;
return rawQuery(
`select distinct
`
select distinct
session_id as id,
website_id as websiteId,
created_at as createdAt,
@ -41,10 +42,11 @@ async function clickhouseQuery(websiteId: string, startAt: Date) {
city
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`,
and created_at >= {startDate:DateTime}
`,
{
websiteId,
startAt,
startDate,
},
);
}

View file

@ -1,5 +1,5 @@
import { DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { uuid } from 'next-basics';
import { flattenJSON } from 'lib/dynamicData';
import prisma from 'lib/prisma';
import { DynamicData } from 'lib/types';

View file

@ -11,31 +11,32 @@ export async function getActiveVisitors(...args: [websiteId: string]) {
}
async function relationalQuery(websiteId: string) {
const { toUuid, rawQuery } = prisma;
const date = subMinutes(new Date(), 5);
const params: any = [websiteId, date];
const { rawQuery } = prisma;
return rawQuery(
`select count(distinct session_id) x
`
select count(distinct session_id) x
from website_event
join website
join website
on website_event.website_id = website.website_id
where website.website_id = $1${toUuid()}
and website_event.created_at >= $2`,
params,
where website.website_id = {{websiteId::uuid}}
and website_event.created_at >= {{startAt}}
`,
{ websiteId, startAt: subMinutes(new Date(), 5) },
);
}
async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse;
const params = { websiteId, startAt: subMinutes(new Date(), 5) };
return rawQuery(
`select count(distinct session_id) x
`
select
count(distinct session_id) x
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startAt:DateTime('UTC')}`,
params,
and created_at >= {startAt:DateTime}
`,
{ websiteId, startAt: subMinutes(new Date(), 5) },
);
}

View file

@ -1,6 +1,5 @@
import { md5 } from 'lib/crypto';
import { getSessions } from '../session/getSessions';
import { getEvents } from '../event/getEvents';
import { md5 } from 'next-basics';
import { getSessions, getEvents } from 'queries';
import { EVENT_TYPE } from 'lib/constants';
export async function getRealtimeData(websiteId, time) {
@ -20,7 +19,7 @@ export async function getRealtimeData(websiteId, time) {
};
return {
pageviews: decorate('pageview', pageviews),
pageviews: decorate('pageviews', pageviews),
sessions: decorate('session', sessions),
events: decorate('event', events),
timestamp: Date.now(),

View file

@ -0,0 +1,51 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { loadWebsite } from 'lib/load';
import { DEFAULT_RESET_DATE } from 'lib/constants';
import { maxDate } from 'lib/date';
export async function getWebsiteDateRange(...args: [websiteId: string]) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string) {
const { rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const result = await rawQuery(
`
select
min(created_at) as mindate,
max(created_at) as maxdate
from website_event
where website_id = {{websiteId::uuid}}
and created_at >= {{startDate}}
`,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) },
);
return result[0] ?? null;
}
async function clickhouseQuery(websiteId: string) {
const { rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const result = await rawQuery(
`
select
min(created_at) as mindate,
max(created_at) as maxdate
from website_event
where website_id = {websiteId:UUID}
and created_at >= {startDate:DateTime}
`,
{ websiteId, startDate: maxDate(new Date(DEFAULT_RESET_DATE), new Date(website.resetAt)) },
);
return result[0] ?? null;
}

View file

@ -1,8 +1,9 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { DEFAULT_CREATED_AT, EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/query';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export async function getWebsiteStats(
...args: [
@ -21,34 +22,41 @@ async function relationalQuery(
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
const { toUuid, getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
const { getDateQuery, getTimestampIntervalQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params: any = [websiteId, resetDate, startDate, endDate];
const { filterQuery, joinSession } = parseFilters(filters, params);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
`select sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(t.time) as "totaltime"
from (
select website_event.session_id,
${getDateQuery('website_event.created_at', 'hour')},
count(*) c,
${getTimestampInterval('website_event.created_at')} as "time"
from website_event
join website
on website_event.website_id = website.website_id
${joinSession}
where event_type = ${EVENT_TYPE.pageView}
and website.website_id = $1${toUuid()}
and website_event.created_at >= $2
and website_event.created_at between $3 and $4
${filterQuery}
group by 1, 2
) t`,
params,
`
select
sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(t.time) as "totaltime"
from (
select
website_event.session_id,
${getDateQuery('website_event.created_at', 'hour')},
count(*) as c,
${getTimestampIntervalQuery('website_event.created_at')} as "time"
from website_event
join website
on website_event.website_id = website.website_id
${joinSession}
where event_type = {{eventType}}
and website.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery}
group by 1, 2
) as t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}
@ -57,32 +65,38 @@ async function clickhouseQuery(
criteria: { startDate: Date; endDate: Date; filters: object },
) {
const { startDate, endDate, filters = {} } = criteria;
const { rawQuery, getDateFormat, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId);
const resetDate = new Date(website?.resetAt || DEFAULT_CREATED_AT);
const params = { websiteId };
const { filterQuery } = parseFilters(filters, params);
const { filterQuery } = parseFilters(filters);
return rawQuery(
`select
sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
from (
select session_id,
${getDateQuery('created_at', 'day')} time_series,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where event_type = ${EVENT_TYPE.pageView}
and website_id = {websiteId:UUID}
and created_at >= ${getDateFormat(resetDate)}
and ${getBetweenDates('created_at', startDate, endDate)}
${filterQuery}
group by session_id, time_series
) t;`,
params,
`
select
sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
from (
select
session_id,
${getDateQuery('created_at', 'day')} time_series,
count(*) c,
min(created_at) min_time,
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, time_series
) as t;
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}

View file

@ -3,19 +3,23 @@ export * from './admin/teamUser';
export * from './admin/user';
export * from './admin/report';
export * from './admin/website';
export * from './analytics/event/getEventMetrics';
export * from './analytics/event/getEventUsage';
export * from './analytics/event/getEvents';
export * from './analytics/events/getEventMetrics';
export * from './analytics/events/getEventUsage';
export * from './analytics/events/getEvents';
export * from './analytics/eventData/getEventDataEvents';
export * from './analytics/eventData/getEventDataFields';
export * from './analytics/eventData/getEventDataUsage';
export * from './analytics/event/saveEvent';
export * from './analytics/pageview/getPageviewFunnel';
export * from './analytics/pageview/getPageviewMetrics';
export * from './analytics/pageview/getPageviewStats';
export * from './analytics/session/createSession';
export * from './analytics/session/getSession';
export * from './analytics/session/getSessionMetrics';
export * from './analytics/session/getSessions';
export * from './analytics/events/saveEvent';
export * from './analytics/reports/getFunnel';
export * from './analytics/reports/getInsights';
export * from './analytics/pageviews/getPageviewMetrics';
export * from './analytics/pageviews/getPageviewStats';
export * from './analytics/sessions/createSession';
export * from './analytics/sessions/getSession';
export * from './analytics/sessions/getSessionMetrics';
export * from './analytics/sessions/getSessions';
export * from './analytics/sessions/saveSessionData';
export * from './analytics/stats/getActiveVisitors';
export * from './analytics/stats/getRealtimeData';
export * from './analytics/stats/getWebsiteDateRange';
export * from './analytics/stats/getWebsiteStats';