diff --git a/src/app/(main)/websites/[websiteId]/WebsiteHeader.tsx b/src/app/(main)/websites/[websiteId]/WebsiteHeader.tsx index 0cbaeb44..bbd6460a 100644 --- a/src/app/(main)/websites/[websiteId]/WebsiteHeader.tsx +++ b/src/app/(main)/websites/[websiteId]/WebsiteHeader.tsx @@ -46,9 +46,14 @@ export function WebsiteHeader({ path: '/reports', }, { - label: formatMessage(labels.eventData), + label: formatMessage(labels.sessions), + icon: , + path: '/sessions', + }, + { + label: formatMessage(labels.events), icon: , - path: '/event-data', + path: '/events', }, ]; diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataMetricsBar.module.css b/src/app/(main)/websites/[websiteId]/events/EventDataMetricsBar.module.css similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/EventDataMetricsBar.module.css rename to src/app/(main)/websites/[websiteId]/events/EventDataMetricsBar.module.css diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataMetricsBar.tsx b/src/app/(main)/websites/[websiteId]/events/EventDataMetricsBar.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/EventDataMetricsBar.tsx rename to src/app/(main)/websites/[websiteId]/events/EventDataMetricsBar.tsx diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataPage.tsx b/src/app/(main)/websites/[websiteId]/events/EventDataPage.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/EventDataPage.tsx rename to src/app/(main)/websites/[websiteId]/events/EventDataPage.tsx diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataTable.tsx b/src/app/(main)/websites/[websiteId]/events/EventDataTable.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/EventDataTable.tsx rename to src/app/(main)/websites/[websiteId]/events/EventDataTable.tsx diff --git a/src/app/(main)/websites/[websiteId]/event-data/EventDataValueTable.tsx b/src/app/(main)/websites/[websiteId]/events/EventDataValueTable.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/EventDataValueTable.tsx rename to src/app/(main)/websites/[websiteId]/events/EventDataValueTable.tsx diff --git a/src/app/(main)/websites/[websiteId]/event-data/WebsiteEventData.module.css b/src/app/(main)/websites/[websiteId]/events/WebsiteEventData.module.css similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/WebsiteEventData.module.css rename to src/app/(main)/websites/[websiteId]/events/WebsiteEventData.module.css diff --git a/src/app/(main)/websites/[websiteId]/event-data/WebsiteEventData.tsx b/src/app/(main)/websites/[websiteId]/events/WebsiteEventData.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/WebsiteEventData.tsx rename to src/app/(main)/websites/[websiteId]/events/WebsiteEventData.tsx diff --git a/src/app/(main)/websites/[websiteId]/event-data/page.tsx b/src/app/(main)/websites/[websiteId]/events/page.tsx similarity index 100% rename from src/app/(main)/websites/[websiteId]/event-data/page.tsx rename to src/app/(main)/websites/[websiteId]/events/page.tsx diff --git a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx index cbdeb1ac..700e83ae 100644 --- a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx +++ b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx @@ -144,7 +144,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) { const { events, visitors } = data; let logs = [ - ...events.map(e => ({ __type: e.eventName ? TYPE_EVENT : TYPE_PAGEVIEW, ...e })), + ...events.map(e => ({ + __type: e.eventName ? TYPE_EVENT : TYPE_PAGEVIEW, + ...e, + })), ...visitors.map(v => ({ __type: TYPE_SESSION, ...v })), ].sort(thenby.firstBy('timestamp', -1)); diff --git a/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx b/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx new file mode 100644 index 00000000..9e9f97e9 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx @@ -0,0 +1,25 @@ +import { useSessions } from 'components/hooks'; +import SessionsTable from './SessionsTable'; +import DataTable from 'components/common/DataTable'; +import { ReactNode } from 'react'; + +export default function SessionsDataTable({ + websiteId, + children, +}: { + websiteId?: string; + teamId?: string; + children?: ReactNode; +}) { + const queryResult = useSessions(websiteId); + + if (queryResult?.result?.data?.length === 0) { + return children; + } + + return ( + + {({ data }) => } + + ); +} diff --git a/src/app/(main)/websites/[websiteId]/sessions/SessionsPage.tsx b/src/app/(main)/websites/[websiteId]/sessions/SessionsPage.tsx new file mode 100644 index 00000000..e95145a7 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/sessions/SessionsPage.tsx @@ -0,0 +1,14 @@ +'use client'; +import WebsiteHeader from '../WebsiteHeader'; +import SessionsDataTable from './SessionsDataTable'; + +export function SessionsPage({ websiteId }) { + return ( + <> + + + + ); +} + +export default SessionsPage; diff --git a/src/app/(main)/websites/[websiteId]/sessions/SessionsTable.tsx b/src/app/(main)/websites/[websiteId]/sessions/SessionsTable.tsx new file mode 100644 index 00000000..41c0fef3 --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/sessions/SessionsTable.tsx @@ -0,0 +1,21 @@ +import { GridColumn, GridTable, useBreakpoint } from 'react-basics'; +import { useMessages } from 'components/hooks'; + +export function SessionsTable({ data = [] }: { data: any[]; showDomain?: boolean }) { + const { formatMessage, labels } = useMessages(); + const breakpoint = useBreakpoint(); + + return ( + + + + + + + + + + ); +} + +export default SessionsTable; diff --git a/src/app/(main)/websites/[websiteId]/sessions/page.tsx b/src/app/(main)/websites/[websiteId]/sessions/page.tsx new file mode 100644 index 00000000..771f682d --- /dev/null +++ b/src/app/(main)/websites/[websiteId]/sessions/page.tsx @@ -0,0 +1,10 @@ +import SessionsPage from './SessionsPage'; +import { Metadata } from 'next'; + +export default function ({ params: { websiteId } }) { + return ; +} + +export const metadata: Metadata = { + title: 'Sessions', +}; diff --git a/src/app/api/scripts/telemetry/route.ts b/src/app/api/scripts/telemetry/route.ts new file mode 100644 index 00000000..ecd83fcb --- /dev/null +++ b/src/app/api/scripts/telemetry/route.ts @@ -0,0 +1,28 @@ +import { CURRENT_VERSION, TELEMETRY_PIXEL } from 'lib/constants'; + +export async function GET() { + if ( + process.env.NODE_ENV !== 'production' && + process.env.DISABLE_TELEMETRY && + process.env.PRIVATE_MODE + ) { + const script = ` + (()=>{const i=document.createElement('img'); + i.setAttribute('src','${TELEMETRY_PIXEL}?v=${CURRENT_VERSION}'); + i.setAttribute('style','width:0;height:0;position:absolute;pointer-events:none;'); + document.body.appendChild(i);})(); + `; + + return new Response(script.replace(/\s\s+/g, ''), { + headers: { + 'content-type': 'text/javascript', + }, + }); + } + + return new Response('/* telemetry disabled */', { + headers: { + 'content-type': 'text/javascript', + }, + }); +} diff --git a/src/components/hooks/index.ts b/src/components/hooks/index.ts index df4fbd88..86abdd84 100644 --- a/src/components/hooks/index.ts +++ b/src/components/hooks/index.ts @@ -5,6 +5,7 @@ export * from './queries/useLogin'; export * from './queries/useRealtime'; export * from './queries/useReport'; export * from './queries/useReports'; +export * from './queries/useSessions'; export * from './queries/useShareToken'; export * from './queries/useTeam'; export * from './queries/useTeams'; diff --git a/src/components/hooks/queries/useSessions.ts b/src/components/hooks/queries/useSessions.ts new file mode 100644 index 00000000..c54c3acd --- /dev/null +++ b/src/components/hooks/queries/useSessions.ts @@ -0,0 +1,20 @@ +import { useApi } from './useApi'; +import { useFilterQuery } from './useFilterQuery'; +import useModified from '../useModified'; + +export function useSessions(websiteId: string, params?: { [key: string]: string | number }) { + const { get } = useApi(); + const { modified } = useModified(`websites`); + + return useFilterQuery({ + queryKey: ['sessions', { websiteId, modified, ...params }], + queryFn: (data: any) => { + return get(`/websites/${websiteId}/sessions`, { + ...data, + ...params, + }); + }, + }); +} + +export default useSessions; diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index cd057cd7..e716d649 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -2,8 +2,8 @@ import { ClickHouseClient, createClient } from '@clickhouse/client'; import dateFormat from 'dateformat'; import debug from 'debug'; import { CLICKHOUSE } from 'lib/db'; -import { QueryFilters, QueryOptions } from './types'; -import { OPERATORS } from './constants'; +import { PageParams, QueryFilters, QueryOptions } from './types'; +import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants'; import { fetchWebsite } from './load'; import { maxDate } from './date'; import { filtersToArray } from './params'; @@ -47,11 +47,11 @@ function getClient() { return client; } -function getDateStringQuery(data: any, unit: string | number) { +function getDateStringSQL(data: any, unit: string | number) { return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`; } -function getDateQuery(field: string, unit: string, timezone?: string) { +function getDateSQL(field: string, unit: string, timezone?: string) { if (timezone) { return `date_trunc('${unit}', ${field}, '${timezone}')`; } @@ -95,6 +95,20 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) return query.join('\n'); } +function getDateQuery(filters: QueryFilters = {}) { + const { startDate, endDate } = filters; + + if (startDate) { + if (endDate) { + return `and created_at between {startDate:DateTime64} and {endDate:DateTime64}`; + } else { + return `and created_at >= {startDate:DateTime64}`; + } + } + + return ''; +} + function getFilterParams(filters: QueryFilters = {}) { return filtersToArray(filters).reduce((obj, { name, value }) => { if (name && value !== undefined) { @@ -110,6 +124,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio return { filterQuery: getFilterQuery(filters, options), + dateQuery: getDateQuery(filters), params: { ...getFilterParams(filters), websiteId, @@ -119,6 +134,32 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio }; } +async function pagedQuery( + query: string, + queryParams: { [key: string]: any }, + pageParams: PageParams = {}, +) { + const { page = 1, pageSize, orderBy, sortDescending = false } = pageParams; + const size = +pageSize || DEFAULT_PAGE_SIZE; + const offset = +size * (page - 1); + const direction = sortDescending ? 'desc' : 'asc'; + + const statements = [ + orderBy && `order by ${orderBy} ${direction}`, + +size > 0 && `limit ${+size} offset ${offset}`, + ] + .filter(n => n) + .join('\n'); + + const count = await rawQuery(`select count(*) as num from (${query}) t`, queryParams).then( + res => res[0].num, + ); + + const data = await rawQuery(`${query}${statements}`, queryParams); + + return { data, count, page: +page, pageSize: size, orderBy }; +} + async function rawQuery( query: string, params: Record = {}, @@ -170,11 +211,12 @@ export default { client: clickhouse, log, connect, - getDateStringQuery, - getDateQuery, + getDateStringSQL, + getDateSQL, getDateFormat, getFilterQuery, parseFilters, + pagedQuery, findUnique, findFirst, rawQuery, diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index 6250f2e5..28835414 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -60,7 +60,7 @@ function getCastColumnQuery(field: string, type: string): string { } } -function getDateQuery(field: string, unit: string, timezone?: string): string { +function getDateSQL(field: string, unit: string, timezone?: string): string { const db = getDatabaseType(); if (db === POSTGRESQL) { @@ -81,7 +81,19 @@ function getDateQuery(field: string, unit: string, timezone?: string): string { } } -function getTimestampDiffQuery(field1: string, field2: string): string { +export function getTimestampSQL(field: string) { + const db = getDatabaseType(); + + if (db === POSTGRESQL) { + return `floor(extract(epoch from ${field}))`; + } + + if (db === MYSQL) { + return `UNIX_TIMESTAMP(${field})`; + } +} + +function getTimestampDiffSQL(field1: string, field2: string): string { const db = getDatabaseType(); if (db === POSTGRESQL) { @@ -93,7 +105,7 @@ function getTimestampDiffQuery(field1: string, field2: string): string { } } -function getSearchQuery(column: string): string { +function getSearchSQL(column: string): string { const db = getDatabaseType(); const like = db === POSTGRESQL ? 'ilike' : 'like'; @@ -137,6 +149,20 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): return query.join('\n'); } +function getDateQuery(filters: QueryFilters = {}) { + const { startDate, endDate } = filters; + + if (startDate) { + if (endDate) { + return `and website_event.created_at between {{startDate}} and {{endDate}}`; + } else { + return `and website_event.created_at >= {{startDate}}`; + } + } + + return ''; +} + function getFilterParams(filters: QueryFilters = {}) { return filtersToArray(filters).reduce((obj, { name, operator, value }) => { obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator) @@ -161,6 +187,7 @@ async function parseFilters( ? `inner join session on website_event.session_id = session.session_id` : '', filterQuery: getFilterQuery(filters, options), + dateQuery: getDateQuery(filters), params: { ...getFilterParams(filters), websiteId, @@ -191,8 +218,8 @@ async function rawQuery(sql: string, data: object): Promise { return prisma.rawQuery(query, params); } -async function pagedQuery(model: string, criteria: T, filters: PageParams) { - const { page = 1, pageSize, orderBy, sortDescending = false } = filters || {}; +async function pagedQuery(model: string, criteria: T, pageParams: PageParams) { + const { page = 1, pageSize, orderBy, sortDescending = false } = pageParams || {}; const size = +pageSize || DEFAULT_PAGE_SIZE; const data = await prisma.client[model].findMany({ @@ -256,11 +283,11 @@ export default { getAddIntervalQuery, getCastColumnQuery, getDayDiffQuery, - getDateQuery, + getDateSQL, getFilterQuery, getSearchParameters, - getTimestampDiffQuery, - getSearchQuery, + getTimestampDiffSQL, + getSearchSQL, getQueryMode, pagedQuery, parseFilters, diff --git a/src/pages/api/scripts/telemetry.ts b/src/pages/api/scripts/telemetry.ts deleted file mode 100644 index a8a8872e..00000000 --- a/src/pages/api/scripts/telemetry.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { ok } from 'next-basics'; -import { CURRENT_VERSION, TELEMETRY_PIXEL } from 'lib/constants'; -import { NextApiRequest, NextApiResponse } from 'next'; - -export default function handler(req: NextApiRequest, res: NextApiResponse) { - if (process.env.NODE_ENV === 'production') { - res.setHeader('content-type', 'text/javascript'); - - if (process.env.DISABLE_TELEMETRY || process.env.PRIVATE_MODE) { - return res.send('/* telemetry disabled */'); - } - - const script = ` - (()=>{const i=document.createElement('img'); - i.setAttribute('src','${TELEMETRY_PIXEL}?v=${CURRENT_VERSION}'); - i.setAttribute('style','width:0;height:0;position:absolute;pointer-events:none;'); - document.body.appendChild(i);})(); - `; - - return res.send(script.replace(/\s\s+/g, '')); - } - - return ok(res); -} diff --git a/src/pages/api/websites/[websiteId]/sessions.ts b/src/pages/api/websites/[websiteId]/sessions.ts new file mode 100644 index 00000000..21cacb1c --- /dev/null +++ b/src/pages/api/websites/[websiteId]/sessions.ts @@ -0,0 +1,42 @@ +import * as yup from 'yup'; +import { canViewWebsite } from 'lib/auth'; +import { useAuth, useCors, useValidate } from 'lib/middleware'; +import { NextApiRequestQueryBody, PageParams } from 'lib/types'; +import { NextApiResponse } from 'next'; +import { methodNotAllowed, ok, unauthorized } from 'next-basics'; +import { pageInfo } from 'lib/schema'; +import { getSessions } from 'queries'; + +export interface ReportsRequestQuery extends PageParams { + websiteId: string; +} + +const schema = { + GET: yup.object().shape({ + websiteId: yup.string().uuid().required(), + ...pageInfo, + }), +}; + +export default async ( + req: NextApiRequestQueryBody, + res: NextApiResponse, +) => { + await useCors(req, res); + await useAuth(req, res); + await useValidate(schema, req, res); + + const { websiteId } = req.query; + + if (req.method === 'GET') { + if (!(await canViewWebsite(req.auth, websiteId))) { + return unauthorized(res); + } + + const data = await getSessions(websiteId, {}, req.query); + + return ok(res, data); + } + + return methodNotAllowed(res); +}; diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts index 32cccd3e..8efbf769 100644 --- a/src/queries/analytics/events/getEventMetrics.ts +++ b/src/queries/analytics/events/getEventMetrics.ts @@ -15,7 +15,7 @@ export async function getEventMetrics( async function relationalQuery(websiteId: string, filters: QueryFilters) { const { timezone = 'utc', unit = 'day' } = filters; - const { rawQuery, getDateQuery, parseFilters } = prisma; + const { rawQuery, getDateSQL, parseFilters } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.customEvent, @@ -25,7 +25,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ` select event_name x, - ${getDateQuery('website_event.created_at', unit, timezone)} t, + ${getDateSQL('website_event.created_at', unit, timezone)} t, count(*) y from website_event ${joinSession} @@ -45,7 +45,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; t: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; - const { rawQuery, getDateQuery, parseFilters } = clickhouse; + const { rawQuery, getDateSQL, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.customEvent, @@ -55,7 +55,7 @@ async function clickhouseQuery( ` select event_name x, - ${getDateQuery('created_at', unit, timezone)} t, + ${getDateSQL('created_at', unit, timezone)} t, count(*) y from website_event where website_id = {websiteId:UUID} diff --git a/src/queries/analytics/events/getEvents.ts b/src/queries/analytics/events/getEvents.ts index c333242e..a00f6848 100644 --- a/src/queries/analytics/events/getEvents.ts +++ b/src/queries/analytics/events/getEvents.ts @@ -1,45 +1,33 @@ import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; -import { QueryFilters } from 'lib/types'; +import { PageParams, QueryFilters } from 'lib/types'; -export function getEvents(...args: [websiteId: string, filters: QueryFilters]) { +export function getEvents( + ...args: [websiteId: string, filters: QueryFilters, pageParams?: PageParams] +) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -function relationalQuery(websiteId: string, filters: QueryFilters) { - const { startDate } = filters; +async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) { + const { pagedQuery } = prisma; - return prisma.client.websiteEvent - .findMany({ - where: { - websiteId, - createdAt: { - gte: startDate, - }, - }, - orderBy: { - createdAt: 'desc', - }, - }) - .then(a => { - return Object.values(a).map(a => { - return { - ...a, - timestamp: new Date(a.createdAt).getTime() / 1000, - }; - }); - }); + const where = { + ...filters, + id: websiteId, + }; + + return pagedQuery('website_event', { where }, pageParams); } -function clickhouseQuery(websiteId: string, filters: QueryFilters) { - const { rawQuery } = clickhouse; - const { startDate } = filters; +async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) { + const { pagedQuery, parseFilters } = clickhouse; + const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters); - return rawQuery( + return pagedQuery( ` select event_id as id, @@ -48,16 +36,20 @@ function clickhouseQuery(websiteId: string, filters: QueryFilters) { created_at as createdAt, toUnixTimestamp(created_at) as timestamp, url_path as urlPath, + url_query as urlQuery, + referrer_path as referrerPath, + referrer_query as referrerQuery, referrer_domain as referrerDomain, + page_title as pageTitle, + event_type as eventType, event_name as eventName from website_event where website_id = {websiteId:UUID} - and created_at >= {startDate:DateTime64} + ${dateQuery} + ${filterQuery} order by created_at desc `, - { - websiteId, - startDate, - }, + params, + pageParams, ); } diff --git a/src/queries/analytics/getRealtimeData.ts b/src/queries/analytics/getRealtimeData.ts index b42fbc50..afc7ff8f 100644 --- a/src/queries/analytics/getRealtimeData.ts +++ b/src/queries/analytics/getRealtimeData.ts @@ -19,15 +19,15 @@ export async function getRealtimeData( const { startDate, timezone } = criteria; const filters = { startDate, endDate: new Date(), unit: 'minute', timezone }; const [events, sessions, pageviews, sessionviews] = await Promise.all([ - getEvents(websiteId, { startDate }), - getSessions(websiteId, { startDate }), + getEvents(websiteId, { startDate }, { pageSize: 10000 }), + getSessions(websiteId, { startDate }, { pageSize: 10000 }), getPageviewStats(websiteId, filters), getSessionStats(websiteId, filters), ]); const uniques = new Set(); - const sessionStats = sessions.reduce( + const sessionStats = sessions.data.reduce( (obj: { visitors: any; countries: any }, session: { id: any; country: any }) => { const { countries, visitors } = obj; const { id, country } = session; @@ -49,7 +49,7 @@ export async function getRealtimeData( }, ); - const eventStats = events.reduce( + const eventStats = events.data.reduce( ( obj: { urls: any; referrers: any; events: any }, event: { urlPath: any; referrerDomain: any }, @@ -81,9 +81,9 @@ export async function getRealtimeData( visitors: sessionviews, }, totals: { - views: events.filter(e => !e.eventName).length, + views: events.data.filter(e => !e.eventName).length, visitors: uniques.size, - events: events.filter(e => e.eventName).length, + events: events.data.filter(e => e.eventName).length, countries: Object.keys(sessionStats.countries).length, }, timestamp: Date.now(), diff --git a/src/queries/analytics/getValues.ts b/src/queries/analytics/getValues.ts index 7cd34994..8b1afb3f 100644 --- a/src/queries/analytics/getValues.ts +++ b/src/queries/analytics/getValues.ts @@ -18,11 +18,11 @@ async function relationalQuery( endDate: Date, search: string, ) { - const { rawQuery, getSearchQuery } = prisma; + const { rawQuery, getSearchSQL } = prisma; let searchQuery = ''; if (search) { - searchQuery = getSearchQuery(column); + searchQuery = getSearchSQL(column); } return rawQuery( diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts index 2f3c82e8..84ceaf1c 100644 --- a/src/queries/analytics/getWebsiteStats.ts +++ b/src/queries/analytics/getWebsiteStats.ts @@ -21,7 +21,7 @@ async function relationalQuery( ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { - const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma; + const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -34,7 +34,7 @@ async function relationalQuery( count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", sum(case when t.c = 1 then 1 else 0 end) as "bounces", - sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as "totaltime" + sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime" from ( select website_event.session_id, diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts index 67ccb04a..b3ae633a 100644 --- a/src/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts @@ -42,15 +42,18 @@ async function relationalQuery( const aggregrate = type === 'entry' ? 'min' : 'max'; entryExitQuery = ` - JOIN (select visit_id, - ${aggregrate}(created_at) target_created_at - from website_event - where website_event.website_id = {{websiteId::uuid}} - and website_event.created_at between {{startDate}} and {{endDate}} - and event_type = {{eventType}} - group by visit_id) x - ON x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at`; + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; } return rawQuery( @@ -97,15 +100,18 @@ async function clickhouseQuery( const aggregrate = type === 'entry' ? 'min' : 'max'; entryExitQuery = ` - JOIN (select visit_id, - ${aggregrate}(created_at) target_created_at - from website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - group by visit_id) x - ON x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at`; + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; } return rawQuery( diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts index a37a1566..65bc8625 100644 --- a/src/queries/analytics/pageviews/getPageviewStats.ts +++ b/src/queries/analytics/pageviews/getPageviewStats.ts @@ -13,7 +13,7 @@ export async function getPageviewStats(...args: [websiteId: string, filters: Que async function relationalQuery(websiteId: string, filters: QueryFilters) { const { timezone = 'utc', unit = 'day' } = filters; - const { getDateQuery, parseFilters, rawQuery } = prisma; + const { getDateSQL, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -22,7 +22,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` select - ${getDateQuery('website_event.created_at', unit, timezone)} x, + ${getDateSQL('website_event.created_at', unit, timezone)} x, count(*) y from website_event ${joinSession} @@ -41,7 +41,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; - const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; + const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -50,11 +50,11 @@ async function clickhouseQuery( return rawQuery( ` select - ${getDateStringQuery('g.t', unit)} as x, + ${getDateStringSQL('g.t', unit)} as x, g.y as y from ( select - ${getDateQuery('created_at', unit, timezone)} as t, + ${getDateSQL('created_at', unit, timezone)} as t, count(*) as y from website_event where website_id = {websiteId:UUID} diff --git a/src/queries/analytics/reports/getInsights.ts b/src/queries/analytics/reports/getInsights.ts index c1a4f1f1..8e6e3289 100644 --- a/src/queries/analytics/reports/getInsights.ts +++ b/src/queries/analytics/reports/getInsights.ts @@ -23,7 +23,7 @@ async function relationalQuery( y: number; }[] > { - const { getTimestampDiffQuery, parseFilters, rawQuery } = prisma; + const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters( websiteId, { @@ -42,7 +42,7 @@ async function relationalQuery( count(distinct t.session_id) as "visitors", count(distinct t.visit_id) as "visits", sum(case when t.c = 1 then 1 else 0 end) as "bounces", - sum(${getTimestampDiffQuery('t.min_time', 't.max_time')}) as "totaltime", + sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime", ${parseFieldsByName(fields)} from ( select diff --git a/src/queries/analytics/reports/getRetention.ts b/src/queries/analytics/reports/getRetention.ts index de495cc4..24aa2e3a 100644 --- a/src/queries/analytics/reports/getRetention.ts +++ b/src/queries/analytics/reports/getRetention.ts @@ -35,14 +35,14 @@ async function relationalQuery( }[] > { const { startDate, endDate, timezone = 'UTC' } = filters; - const { getDateQuery, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma; + const { getDateSQL, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma; const unit = 'day'; return rawQuery( ` WITH cohort_items AS ( select session_id, - ${getDateQuery('created_at', unit, timezone)} as cohort_date + ${getDateSQL('created_at', unit, timezone)} as cohort_date from session where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} @@ -50,10 +50,7 @@ async function relationalQuery( user_activities AS ( select distinct w.session_id, - ${getDayDiffQuery( - getDateQuery('created_at', unit, timezone), - 'c.cohort_date', - )} as day_number + ${getDayDiffQuery(getDateSQL('created_at', unit, timezone), 'c.cohort_date')} as day_number from website_event w join cohort_items c on w.session_id = c.session_id @@ -115,14 +112,14 @@ async function clickhouseQuery( }[] > { const { startDate, endDate, timezone = 'UTC' } = filters; - const { getDateQuery, getDateStringQuery, rawQuery } = clickhouse; + const { getDateSQL, getDateStringSQL, rawQuery } = clickhouse; const unit = 'day'; return rawQuery( ` WITH cohort_items AS ( select - min(${getDateQuery('created_at', unit, timezone)}) as cohort_date, + min(${getDateSQL('created_at', unit, timezone)}) as cohort_date, session_id from website_event where website_id = {websiteId:UUID} @@ -132,7 +129,7 @@ async function clickhouseQuery( user_activities AS ( select distinct w.session_id, - (${getDateQuery('created_at', unit, timezone)} - c.cohort_date) / 86400 as day_number + (${getDateSQL('created_at', unit, timezone)} - c.cohort_date) / 86400 as day_number from website_event w join cohort_items c on w.session_id = c.session_id @@ -157,7 +154,7 @@ async function clickhouseQuery( group by 1, 2 ) select - ${getDateStringQuery('c.cohort_date', unit)} as date, + ${getDateStringSQL('c.cohort_date', unit)} as date, c.day_number as day, s.visitors as visitors, c.visitors returnVisitors, diff --git a/src/queries/analytics/reports/getRevenue.ts b/src/queries/analytics/reports/getRevenue.ts index 6b151bb7..e4857a43 100644 --- a/src/queries/analytics/reports/getRevenue.ts +++ b/src/queries/analytics/reports/getRevenue.ts @@ -46,12 +46,12 @@ async function relationalQuery( timezone = 'UTC', unit = 'day', } = criteria; - const { getDateQuery, rawQuery } = prisma; + const { getDateSQL, rawQuery } = prisma; const chartRes = await rawQuery( ` select - ${getDateQuery('website_event.created_at', unit, timezone)} time, + ${getDateSQL('website_event.created_at', unit, timezone)} time, sum(case when data_key = {{revenueProperty}} then number_value else 0 end) sum, avg(case when data_key = {{revenueProperty}} then number_value else 0 end) avg, count(case when data_key = {{revenueProperty}} then 1 else 0 end) count, @@ -110,7 +110,7 @@ async function clickhouseQuery( timezone = 'UTC', unit = 'day', } = criteria; - const { getDateStringQuery, getDateQuery, rawQuery } = clickhouse; + const { getDateStringSQL, getDateSQL, rawQuery } = clickhouse; const chartRes = await rawQuery<{ time: string; @@ -121,14 +121,14 @@ async function clickhouseQuery( }>( ` select - ${getDateStringQuery('g.time', unit)} as time, + ${getDateStringSQL('g.time', unit)} as time, g.sum as sum, g.avg as avg, g.count as count, g.uniqueCount as uniqueCount from ( select - ${getDateQuery('created_at', unit, timezone)} as time, + ${getDateSQL('created_at', unit, timezone)} as time, sumIf(number_value, data_key = {revenueProperty:String}) as sum, avgIf(number_value, data_key = {revenueProperty:String}) as avg, countIf(data_key = {revenueProperty:String}) as count, diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts index e3af7ba6..54c46c35 100644 --- a/src/queries/analytics/sessions/getSessionStats.ts +++ b/src/queries/analytics/sessions/getSessionStats.ts @@ -13,7 +13,7 @@ export async function getSessionStats(...args: [websiteId: string, filters: Quer async function relationalQuery(websiteId: string, filters: QueryFilters) { const { timezone = 'utc', unit = 'day' } = filters; - const { getDateQuery, parseFilters, rawQuery } = prisma; + const { getDateSQL, parseFilters, rawQuery } = prisma; const { filterQuery, joinSession, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -22,7 +22,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` select - ${getDateQuery('website_event.created_at', unit, timezone)} x, + ${getDateSQL('website_event.created_at', unit, timezone)} x, count(distinct website_event.session_id) y from website_event ${joinSession} @@ -41,7 +41,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { timezone = 'UTC', unit = 'day' } = filters; - const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; + const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, eventType: EVENT_TYPE.pageView, @@ -50,11 +50,11 @@ async function clickhouseQuery( return rawQuery( ` select - ${getDateStringQuery('g.t', unit)} as x, + ${getDateStringSQL('g.t', unit)} as x, g.y as y from ( select - ${getDateQuery('created_at', unit, timezone)} as t, + ${getDateSQL('created_at', unit, timezone)} as t, count(distinct session_id) as y from website_event where website_id = {websiteId:UUID} diff --git a/src/queries/analytics/sessions/getSessions.ts b/src/queries/analytics/sessions/getSessions.ts index a11edd39..47471d98 100644 --- a/src/queries/analytics/sessions/getSessions.ts +++ b/src/queries/analytics/sessions/getSessions.ts @@ -1,45 +1,33 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db'; -import { QueryFilters } from 'lib/types'; +import { PageParams, QueryFilters } from 'lib/types'; -export async function getSessions(...args: [websiteId: string, filters: QueryFilters]) { +export async function getSessions( + ...args: [websiteId: string, filters?: QueryFilters, pageParams?: PageParams] +) { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), }); } -async function relationalQuery(websiteId: string, filters: QueryFilters) { - const { startDate } = filters; +async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams: PageParams) { + const { pagedQuery } = prisma; - return prisma.client.session - .findMany({ - where: { - websiteId, - createdAt: { - gte: startDate, - }, - }, - orderBy: { - createdAt: 'desc', - }, - }) - .then(a => { - return Object.values(a).map(a => { - return { - ...a, - timestamp: new Date(a.createdAt).getTime() / 1000, - }; - }); - }); + const where = { + ...filters, + id: websiteId, + }; + + return pagedQuery('session', { where }, pageParams); } -async function clickhouseQuery(websiteId: string, filters: QueryFilters) { - const { rawQuery } = clickhouse; - const { startDate } = filters; +async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) { + const { pagedQuery, parseFilters } = clickhouse; + const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters); - return rawQuery( + return pagedQuery( ` select session_id as id, @@ -58,12 +46,11 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { city from website_event where website_id = {websiteId:UUID} - and created_at >= {startDate:DateTime64} + ${dateQuery} + ${filterQuery} order by created_at desc `, - { - websiteId, - startDate, - }, + params, + pageParams, ); } diff --git a/src/queries/index.ts b/src/queries/index.ts index 8cef080a..796adea6 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -1,8 +1,8 @@ -export * from './admin/report'; -export * from './admin/team'; -export * from './admin/teamUser'; -export * from './admin/user'; -export * from './admin/website'; +export * from 'queries/prisma/report'; +export * from 'queries/prisma/team'; +export * from 'queries/prisma/teamUser'; +export * from 'queries/prisma/user'; +export * from 'queries/prisma/website'; export * from './analytics/events/getEventMetrics'; export * from './analytics/events/getEventUsage'; export * from './analytics/events/getEvents'; diff --git a/src/queries/admin/report.ts b/src/queries/prisma/report.ts similarity index 93% rename from src/queries/admin/report.ts rename to src/queries/prisma/report.ts index dc05a1d5..a0e6364c 100644 --- a/src/queries/admin/report.ts +++ b/src/queries/prisma/report.ts @@ -17,9 +17,9 @@ export async function getReport(reportId: string): Promise { export async function getReports( criteria: ReportFindManyArgs, - filters: PageParams = {}, + pageParams: PageParams = {}, ): Promise> { - const { query } = filters; + const { query } = pageParams; const where: Prisma.ReportWhereInput = { ...criteria.where, @@ -45,7 +45,7 @@ export async function getReports( ]), }; - return prisma.pagedQuery('report', { ...criteria, where }, filters); + return prisma.pagedQuery('report', { ...criteria, where }, pageParams); } export async function getUserReports( diff --git a/src/queries/admin/team.ts b/src/queries/prisma/team.ts similarity index 100% rename from src/queries/admin/team.ts rename to src/queries/prisma/team.ts diff --git a/src/queries/admin/teamUser.ts b/src/queries/prisma/teamUser.ts similarity index 100% rename from src/queries/admin/teamUser.ts rename to src/queries/prisma/teamUser.ts diff --git a/src/queries/admin/user.ts b/src/queries/prisma/user.ts similarity index 98% rename from src/queries/admin/user.ts rename to src/queries/prisma/user.ts index 9e085112..9b471787 100644 --- a/src/queries/admin/user.ts +++ b/src/queries/prisma/user.ts @@ -49,9 +49,9 @@ export async function getUserByUsername(username: string, options: GetUserOption export async function getUsers( criteria: UserFindManyArgs, - filters?: PageParams, + pageParams?: PageParams, ): Promise> { - const { query } = filters; + const { query } = pageParams; const where: Prisma.UserWhereInput = { ...criteria.where, @@ -68,7 +68,7 @@ export async function getUsers( { orderBy: 'createdAt', sortDescending: true, - ...filters, + ...pageParams, }, ); } diff --git a/src/queries/admin/website.ts b/src/queries/prisma/website.ts similarity index 97% rename from src/queries/admin/website.ts rename to src/queries/prisma/website.ts index eb07f779..0814a137 100644 --- a/src/queries/admin/website.ts +++ b/src/queries/prisma/website.ts @@ -27,9 +27,9 @@ export async function getSharedWebsite(shareId: string) { export async function getWebsites( criteria: WebsiteFindManyArgs, - filters: PageParams, + pageParams: PageParams, ): Promise> { - const { query } = filters; + const { query } = pageParams; const where: Prisma.WebsiteWhereInput = { ...criteria.where, @@ -42,7 +42,7 @@ export async function getWebsites( deletedAt: null, }; - return prisma.pagedQuery('website', { ...criteria, where }, filters); + return prisma.pagedQuery('website', { ...criteria, where }, pageParams); } export async function getAllWebsites(userId: string) {