watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-13 17:15:37 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Database refactoring.

Browse source
This commit is contained in:
Mike Cao 2022-08-27 21:38:35 -07:00
parent bb184dc2cc
commit 467c7f289f
37 changed files with 566 additions and 591 deletions
Download patch file Download diff file Expand all files Collapse all files

29
queries/analytics/pageview/getPageviewMetrics.js
View file

@ -1,16 +1,16 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { parseFilters, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewMetrics(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { rawQuery, parseFilters } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
`
select ${column} x, count(*) y
`select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id=$1
@ -30,15 +29,15 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
order by 2 desc
`,
order by 2 desc`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
column,
filters,
@ -46,19 +45,17 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt
'session_uuid',
);
return clickhouse.rawQuery(
`
select ${column} x, count(*) y
return rawQuery(
`select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id= $1
and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}
and ${getBetweenDates(table + '.created_at', start_at, end_at)}
${pageviewQuery}
${joinSession && sessionQuery}
${eventQuery}
group by x
order by y desc
`,
order by y desc`,
params,
);
}

15
queries/analytics/pageview/getPageviewParams.js
View file

@ -1,14 +1,15 @@
import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational';
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import prisma from 'lib/prisma';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewParams(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
const { parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@ -18,8 +19,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
`
select url x,
`select url x,
count(*) y
from ${table}
${joinSession}
@ -30,8 +30,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
order by 2 desc
`,
order by 2 desc`,
params,
);
}

31
queries/analytics/pageview/getPageviewStats.js
View file

@ -1,11 +1,10 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { getDateQuery, parseFilters, rawQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewStats(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@ -20,6 +19,7 @@ async function relationalQuery(
filters = {},
sessionKey = 'session_id',
) {
const { getDateQuery, parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -29,8 +29,7 @@ async function relationalQuery(
);
return rawQuery(
`
select ${getDateQuery('pageview.created_at', unit, timezone)} t,
`select ${getDateQuery('pageview.created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
@ -38,8 +37,7 @@ async function relationalQuery(
and pageview.created_at between $2 and $3
${pageviewQuery}
${sessionQuery}
group by 1
`,
group by 1`,
params,
);
}
@ -54,6 +52,7 @@ async function clickhouseQuery(
filters = {},
sessionKey = 'session_uuid',
) {
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@ -63,24 +62,22 @@ async function clickhouseQuery(
sessionKey,
);
return clickhouse.rawQuery(
`
select
${clickhouse.getDateStringQuery('g.t', unit)} as t,
return rawQuery(
`select
${getDateStringQuery('g.t', unit)} as t,
g.y as y
from
(select
${clickhouse.getDateQuery('created_at', unit, timezone)} t,
${getDateQuery('created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
where pageview.website_id= $1
and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by t) g
order by t
`,
order by t`,
params,
);
}

37
queries/analytics/pageview/getPageviews.js
View file

@ -1,36 +1,32 @@
import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
import { prisma, runQuery } from 'lib/relational';
import { runAnalyticsQuery } from 'lib/db';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviews(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websites, start_at) {
return runQuery(
prisma.pageview.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
created_at: {
gte: start_at,
return prisma.client.pageview.findMany({
where: {
website: {
website_id: {
in: websites,
},
},
}),
);
created_at: {
gte: start_at,
},
},
});
}
async function clickhouseQuery(websites, start_at) {
return clickhouse.rawQuery(
`
select
`select
view_id,
website_id,
session_id,
@ -38,7 +34,6 @@ async function clickhouseQuery(websites, start_at) {
url
from pageview
where website_id in (${websites.join[',']}
and created_at >= ${clickhouse.getDateFormat(start_at)})
`,
and created_at >= ${clickhouse.getDateFormat(start_at)})`,
);
}

44
queries/analytics/pageview/savePageView.js
View file

@ -1,54 +1,52 @@
import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runAnalyticsQuery } from 'lib/db';
import kafka from 'lib/kafka';
import { prisma, runQuery } from 'lib/relational';
import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
import { URL_LENGTH } from 'lib/constants';
export async function savePageView(...args) {
return runAnalyticsQuery({
[RELATIONAL]: () => relationalQuery(...args),
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
async function relationalQuery(website_id, { session_id, url, referrer }) {
return runQuery(
prisma.pageview.create({
data: {
website_id,
session_id,
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
},
}),
);
return prisma.client.pageview.create({
data: {
website_id,
session_id,
url: url?.substring(0, URL_LENGTH),
referrer: referrer?.substring(0, URL_LENGTH),
},
});
}
async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
const params = [
website_id,
session_uuid,
url?.substr(0, URL_LENGTH),
referrer?.substr(0, URL_LENGTH),
url?.substring(0, URL_LENGTH),
referrer?.substring(0, URL_LENGTH),
];
return clickhouse.rawQuery(
`
insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
`insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
const { getDateFormat, sendMessage } = kafka;
const params = {
website_id: website_id,
session_uuid: session_uuid,
created_at: kafka.getDateFormat(new Date()),
url: url?.substr(0, URL_LENGTH),
referrer: referrer?.substr(0, URL_LENGTH),
created_at: getDateFormat(new Date()),
url: url?.substring(0, URL_LENGTH),
referrer: referrer?.substring(0, URL_LENGTH),
};
await kafka.sendMessage(params, 'pageview');
await sendMessage(params, 'pageview');
}