update dashboard queries to use mv

This commit is contained in:
Francis Cao 2024-07-08 11:18:30 -07:00
parent 5289c277fb
commit bfd5c5f150
5 changed files with 44 additions and 12 deletions

View file

@ -64,7 +64,7 @@ export default async (
await useAuth(req, res); await useAuth(req, res);
await useValidate(schema, req, res); await useValidate(schema, req, res);
const { websiteId, type, limit, offset, search } = req.query; const { websiteId, type, limit, offset, search, unit } = req.query;
if (req.method === 'GET') { if (req.method === 'GET') {
if (!(await canViewWebsite(req.auth, websiteId))) { if (!(await canViewWebsite(req.auth, websiteId))) {
@ -89,7 +89,7 @@ export default async (
} }
if (SESSION_COLUMNS.includes(type)) { if (SESSION_COLUMNS.includes(type)) {
const data = await getSessionMetrics(websiteId, type, filters, limit, offset); const data = await getSessionMetrics(websiteId, type, filters, limit, offset, unit as string);
if (type === 'language') { if (type === 'language') {
const combined = {}; const combined = {};
@ -111,7 +111,14 @@ export default async (
} }
if (EVENT_COLUMNS.includes(type)) { if (EVENT_COLUMNS.includes(type)) {
const data = await getPageviewMetrics(websiteId, type, filters, limit, offset); const data = await getPageviewMetrics(
websiteId,
type,
filters,
limit,
offset,
unit as string,
);
return ok(res, data); return ok(res, data);
} }

View file

@ -1,3 +1,4 @@
/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants'; import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
@ -5,7 +6,14 @@ import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types'; import { QueryFilters } from 'lib/types';
export async function getPageviewMetrics( export async function getPageviewMetrics(
...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number] ...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number,
offset?: number,
unit?: string,
]
) { ) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -19,6 +27,7 @@ async function relationalQuery(
filters: QueryFilters, filters: QueryFilters,
limit: number = 500, limit: number = 500,
offset: number = 0, offset: number = 0,
unit: string,
) { ) {
const column = FILTER_COLUMNS[type] || type; const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = prisma; const { rawQuery, parseFilters } = prisma;
@ -79,6 +88,7 @@ async function clickhouseQuery(
filters: QueryFilters, filters: QueryFilters,
limit: number = 500, limit: number = 500,
offset: number = 0, offset: number = 0,
unit: string,
): Promise<{ x: string; y: number }[]> { ): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type; const column = FILTER_COLUMNS[type] || type;
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
@ -108,10 +118,12 @@ async function clickhouseQuery(
and x.target_created_at = website_event.created_at`; and x.target_created_at = website_event.created_at`;
} }
const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily';
return rawQuery( return rawQuery(
` `
select ${column} x, count(*) y select ${column} x, countMerge(views) y
from website_event from ${table} website_event
${entryExitQuery} ${entryExitQuery}
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}

View file

@ -46,6 +46,7 @@ async function clickhouseQuery(
...filters, ...filters,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
}); });
const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily';
return rawQuery( return rawQuery(
` `
@ -55,8 +56,8 @@ async function clickhouseQuery(
from ( from (
select select
${getDateQuery('created_at', unit, timezone)} as t, ${getDateQuery('created_at', unit, timezone)} as t,
count(*) as y countMerge(views) as y
from website_event from ${table} website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}

View file

@ -1,3 +1,4 @@
/* eslint-disable no-unused-vars, @typescript-eslint/no-unused-vars */
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
@ -5,7 +6,14 @@ import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { QueryFilters } from 'lib/types'; import { QueryFilters } from 'lib/types';
export async function getSessionMetrics( export async function getSessionMetrics(
...args: [websiteId: string, type: string, filters: QueryFilters, limit?: number, offset?: number] ...args: [
websiteId: string,
type: string,
filters: QueryFilters,
limit?: number,
offset?: number,
unit?: string,
]
) { ) {
return runQuery({ return runQuery({
[PRISMA]: () => relationalQuery(...args), [PRISMA]: () => relationalQuery(...args),
@ -19,6 +27,7 @@ async function relationalQuery(
filters: QueryFilters, filters: QueryFilters,
limit: number = 500, limit: number = 500,
offset: number = 0, offset: number = 0,
unit: string,
) { ) {
const column = FILTER_COLUMNS[type] || type; const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = prisma; const { parseFilters, rawQuery } = prisma;
@ -62,6 +71,7 @@ async function clickhouseQuery(
filters: QueryFilters, filters: QueryFilters,
limit: number = 500, limit: number = 500,
offset: number = 0, offset: number = 0,
unit: string,
): Promise<{ x: string; y: number }[]> { ): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type; const column = FILTER_COLUMNS[type] || type;
const { parseFilters, rawQuery } = clickhouse; const { parseFilters, rawQuery } = clickhouse;
@ -70,6 +80,7 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
}); });
const includeCountry = column === 'city' || column === 'subdivision1'; const includeCountry = column === 'city' || column === 'subdivision1';
const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily';
return rawQuery( return rawQuery(
` `
@ -77,7 +88,7 @@ async function clickhouseQuery(
${column} x, ${column} x,
uniq(session_id) y uniq(session_id) y
${includeCountry ? ', country' : ''} ${includeCountry ? ', country' : ''}
from website_event from ${table} website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}

View file

@ -46,6 +46,7 @@ async function clickhouseQuery(
...filters, ...filters,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
}); });
const table = unit === 'hour' ? 'website_event_metric_hourly' : 'website_event_metric_daily';
return rawQuery( return rawQuery(
` `
@ -55,8 +56,8 @@ async function clickhouseQuery(
from ( from (
select select
${getDateQuery('created_at', unit, timezone)} as t, ${getDateQuery('created_at', unit, timezone)} as t,
count(distinct session_id) as y uniq(session_id) as y
from website_event from ${table} website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64} and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}