diff --git a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx index acd72ab9..e6788ae9 100644 --- a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx +++ b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx @@ -156,6 +156,7 @@ export function WebsiteExpandedView({ itemCount={25} allowFilter={true} allowSearch={true} + expanded={true} /> diff --git a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts new file mode 100644 index 00000000..a327bd0b --- /dev/null +++ b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts @@ -0,0 +1,71 @@ +import { canViewWebsite } from '@/lib/auth'; +import { EVENT_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { badRequest, json, unauthorized } from '@/lib/response'; +import { dateRangeParams, filterParams, searchParams } from '@/lib/schema'; +import { + getChannelExpandedMetrics, + getEventExpandedMetrics, + getPageviewExpandedMetrics, + getSessionExpandedMetrics, +} from '@/queries'; +import { z } from 'zod'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + type: z.string(), + limit: z.coerce.number().optional(), + offset: z.coerce.number().optional(), + ...dateRangeParams, + ...searchParams, + ...filterParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const { type, limit, offset, search } = query; + const filters = await getQueryFilters(query, websiteId); + + if (search) { + filters[type] = `c.${search}`; + } + + if (SESSION_COLUMNS.includes(type)) { + const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters); + + return json(data); + } + + if (EVENT_COLUMNS.includes(type)) { + let data; + + if (type === 'event') { + data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters); + } else { + data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters); + } + + return json(data); + } + + if (type === 'channel') { + const data = await getChannelExpandedMetrics(websiteId, { limit, offset }, filters); + + return json(data); + } + + return badRequest(); +} diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 49592a0e..bc295d79 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -46,22 +46,6 @@ export async function GET( if (SESSION_COLUMNS.includes(type)) { const data = await getSessionMetrics(websiteId, { type, limit, offset }, filters); - if (type === 'language') { - const combined = {}; - - for (const { x, y } of data) { - const key = String(x).toLowerCase().split('-')[0]; - - if (combined[key] === undefined) { - combined[key] = { x: key, y }; - } else { - combined[key].y += y; - } - } - - return json(Object.values(combined)); - } - return json(data); } diff --git a/src/components/hooks/index.ts b/src/components/hooks/index.ts index 3ad21b87..ddd66d57 100644 --- a/src/components/hooks/index.ts +++ b/src/components/hooks/index.ts @@ -32,6 +32,7 @@ export * from './queries/useWebsitesQuery'; export * from './queries/useWebsiteEventsQuery'; export * from './queries/useWebsiteEventsSeriesQuery'; export * from './queries/useWebsiteMetricsQuery'; +export * from './queries/useWebsiteExpandedMetricsQuery'; export * from './queries/useWebsiteValuesQuery'; export * from './useApi'; export * from './useConfig'; diff --git a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts new file mode 100644 index 00000000..9718dfe1 --- /dev/null +++ b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts @@ -0,0 +1,45 @@ +import { keepPreviousData } from '@tanstack/react-query'; +import { useApi } from '../useApi'; +import { useFilterParameters } from '../useFilterParameters'; +import { useDateParameters } from '../useDateParameters'; +import { ReactQueryOptions } from '@/lib/types'; + +export type WebsiteExpandedMetricsData = { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +}[]; + +export function useWebsiteExpandedMetricsQuery( + websiteId: string, + params: { type: string; limit?: number; search?: string }, + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const date = useDateParameters(websiteId); + const filters = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:metrics:expanded', + { + websiteId, + ...date, + ...filters, + ...params, + }, + ], + queryFn: async () => + get(`/websites/${websiteId}/metrics/expanded`, { + ...date, + ...filters, + ...params, + }), + enabled: !!websiteId, + placeholderData: keepPreviousData, + ...options, + }); +} diff --git a/src/components/metrics/ListExpandedTable.module.css b/src/components/metrics/ListExpandedTable.module.css new file mode 100644 index 00000000..4bd287c4 --- /dev/null +++ b/src/components/metrics/ListExpandedTable.module.css @@ -0,0 +1,7 @@ +.truncate { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + width: 300px; + display: block; +} diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsx new file mode 100644 index 00000000..dc27b17f --- /dev/null +++ b/src/components/metrics/ListExpandedTable.tsx @@ -0,0 +1,57 @@ +import { useMessages } from '@/components/hooks'; +import { formatShortTime } from '@/lib/format'; +import { DataColumn, DataTable } from '@umami/react-zen'; +import { ReactNode } from 'react'; +import styles from './ListExpandedTable.module.css'; + +export interface ListExpandedTableProps { + data?: any[]; + title?: string; + type?: string; + renderLabel?: (row: any, index: number) => ReactNode; +} + +export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) { + const { formatMessage, labels } = useMessages(); + + return ( + + + {row => + renderLabel + ? renderLabel({ x: row?.['name'], country: row?.['country'] }, Number(row.id)) + : (row?.['name'] ?? formatMessage(labels.unknown)) + } + + + {row => row?.['visitors']?.toLocaleString()} + + + {row => row?.['visits']?.toLocaleString()} + + + {row => row?.['pageviews']?.toLocaleString()} + + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; + return Math.round(+n) + '%'; + }} + + ) : ( + <> + )} + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (row?.['totaltime'] / row?.['visits']) * 100; + return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; + }} + + ) : ( + <> + )} + + ); +} diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index c229b00c..fb97f8db 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -1,13 +1,20 @@ -import { ReactNode, useMemo, useState } from 'react'; -import { Icon, Text, SearchField, Row, Column } from '@umami/react-zen'; import { LinkButton } from '@/components/common/LinkButton'; +import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { + useFormat, + useMessages, + useNavigation, + useWebsiteExpandedMetricsQuery, + useWebsiteMetricsQuery, +} from '@/components/hooks'; +import { Arrow } from '@/components/icons'; +import { DownloadButton } from '@/components/input/DownloadButton'; import { DEFAULT_ANIMATION_DURATION } from '@/lib/constants'; import { percentFilter } from '@/lib/filters'; -import { useNavigation, useWebsiteMetricsQuery, useMessages, useFormat } from '@/components/hooks'; -import { Arrow } from '@/components/icons'; +import { Column, Icon, Row, SearchField, Text } from '@umami/react-zen'; +import { ReactNode, useMemo, useState } from 'react'; +import { ListExpandedTable, ListExpandedTableProps } from './ListExpandedTable'; import { ListTable, ListTableProps } from './ListTable'; -import { LoadingPanel } from '@/components/common/LoadingPanel'; -import { DownloadButton } from '@/components/input/DownloadButton'; export interface MetricsTableProps extends ListTableProps { websiteId: string; @@ -21,6 +28,7 @@ export interface MetricsTableProps extends ListTableProps { showMore?: boolean; params?: { [key: string]: any }; allowDownload?: boolean; + expanded?: boolean; children?: ReactNode; } @@ -35,6 +43,7 @@ export function MetricsTable({ showMore = true, params, allowDownload = true, + expanded = false, children, ...props }: MetricsTableProps) { @@ -43,7 +52,20 @@ export function MetricsTable({ const { updateParams } = useNavigation(); const { formatMessage, labels } = useMessages(); - const { data, isLoading, isFetching, error } = useWebsiteMetricsQuery( + const expandedQuery = useWebsiteExpandedMetricsQuery( + websiteId, + { + type, + search: searchFormattedValues ? undefined : search, + ...params, + }, + { + retryDelay: delay || DEFAULT_ANIMATION_DURATION, + enabled: expanded, + }, + ); + + const query = useWebsiteMetricsQuery( websiteId, { type, @@ -53,9 +75,12 @@ export function MetricsTable({ }, { retryDelay: delay || DEFAULT_ANIMATION_DURATION, + enabled: !expanded, }, ); + const { data, isLoading, isFetching, error } = expanded ? expandedQuery : query; + const filteredData = useMemo(() => { if (data) { let items = data as any[]; @@ -85,6 +110,8 @@ export function MetricsTable({ return []; }, [data, dataFilter, search, limit, formatValue, type]); + const downloadData = expanded ? data : filteredData; + return ( @@ -92,10 +119,15 @@ export function MetricsTable({ {allowSearch && } {children} - {allowDownload && } + {allowDownload && } - {data && } + {data && + (expanded ? ( + + ) : ( + + ))} {showMore && data && !error && limit && ( diff --git a/src/components/metrics/QueryParametersTable.tsx b/src/components/metrics/QueryParametersTable.tsx index e98f9134..0d3ac268 100644 --- a/src/components/metrics/QueryParametersTable.tsx +++ b/src/components/metrics/QueryParametersTable.tsx @@ -58,6 +58,7 @@ export function QueryParametersTable({ dataFilter={filters[filter]} renderLabel={renderLabel} delay={0} + expanded={false} > {allowFilter && } diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 233a78bf..fb5e1227 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -90,7 +90,11 @@ function mapFilter(column: string, operator: string, name: string, type: string function getFilterQuery(filters: Record, options: QueryOptions = {}) { const query = filtersObjectToArray(filters, options).reduce((arr, { name, column, operator }) => { if (column) { - arr.push(`and ${mapFilter(column, operator, name)}`); + if (name === 'eventType') { + arr.push(`and ${mapFilter(column, operator, name, 'UInt32')}`); + } else { + arr.push(`and ${mapFilter(column, operator, name)}`); + } if (name === 'referrer') { arr.push(`and referrer_domain != hostname`); diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 1564458d..16bb71af 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -35,7 +35,7 @@ export const EVENT_COLUMNS = [ 'query', 'event', 'tag', - 'host', + 'hostname', ]; export const SESSION_COLUMNS = [ diff --git a/src/queries/index.ts b/src/queries/index.ts index fba7e548..707dc874 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues'; export * from '@/queries/sql/events/getEventDataStats'; export * from '@/queries/sql/events/getEventDataUsage'; export * from '@/queries/sql/events/getEventMetrics'; +export * from '@/queries/sql/events/getEventExpandedMetrics'; export * from '@/queries/sql/events/getEventStats'; export * from '@/queries/sql/events/getWebsiteEvents'; export * from '@/queries/sql/events/getEventUsage'; @@ -21,6 +22,7 @@ export * from '@/queries/sql/reports/getRetention'; export * from '@/queries/sql/reports/getBreakdown'; export * from '@/queries/sql/reports/getUTM'; export * from '@/queries/sql/pageviews/getPageviewMetrics'; +export * from '@/queries/sql/pageviews/getPageviewExpandedMetrics'; export * from '@/queries/sql/pageviews/getPageviewStats'; export * from '@/queries/sql/sessions/createSession'; export * from '@/queries/sql/sessions/getWebsiteSession'; @@ -28,6 +30,7 @@ export * from '@/queries/sql/sessions/getSessionData'; export * from '@/queries/sql/sessions/getSessionDataProperties'; export * from '@/queries/sql/sessions/getSessionDataValues'; export * from '@/queries/sql/sessions/getSessionMetrics'; +export * from '@/queries/sql/sessions/getSessionExpandedMetrics'; export * from '@/queries/sql/sessions/getWebsiteSessions'; export * from '@/queries/sql/sessions/getWebsiteSessionStats'; export * from '@/queries/sql/sessions/getWebsiteSessionsWeekly'; @@ -36,6 +39,7 @@ export * from '@/queries/sql/sessions/getSessionStats'; export * from '@/queries/sql/sessions/saveSessionData'; export * from '@/queries/sql/getActiveVisitors'; export * from '@/queries/sql/getChannelMetrics'; +export * from '@/queries/sql/getChannelExpandedMetrics'; export * from '@/queries/sql/getRealtimeActivity'; export * from '@/queries/sql/getRealtimeData'; export * from '@/queries/sql/getValues'; diff --git a/src/queries/sql/events/getEventExpandedMetrics.ts b/src/queries/sql/events/getEventExpandedMetrics.ts new file mode 100644 index 00000000..235822b8 --- /dev/null +++ b/src/queries/sql/events/getEventExpandedMetrics.ts @@ -0,0 +1,113 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface EventExpandedMetricParameters { + type: string; + limit?: string; + offset?: string; +} + +export interface EventExpandedMetricData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getEventExpandedMetrics( + ...args: [websiteId: string, parameters: EventExpandedMetricParameters, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +) { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + return rawQuery( + ` + select ${column} x, + count(*) as y + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/events/getEventMetrics.ts b/src/queries/sql/events/getEventMetrics.ts index 73af2278..3d148edf 100644 --- a/src/queries/sql/events/getEventMetrics.ts +++ b/src/queries/sql/events/getEventMetrics.ts @@ -83,7 +83,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x order by y desc diff --git a/src/queries/sql/events/getEventStats.ts b/src/queries/sql/events/getEventStats.ts index ad6b155d..8d26dcfc 100644 --- a/src/queries/sql/events/getEventStats.ts +++ b/src/queries/sql/events/getEventStats.ts @@ -72,7 +72,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x, t order by t diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.ts new file mode 100644 index 00000000..d8fce3d2 --- /dev/null +++ b/src/queries/sql/getChannelExpandedMetrics.ts @@ -0,0 +1,162 @@ +import clickhouse from '@/lib/clickhouse'; +import { + EMAIL_DOMAINS, + EVENT_TYPE, + PAID_AD_PARAMS, + SEARCH_DOMAINS, + SHOPPING_DOMAINS, + SOCIAL_DOMAINS, + VIDEO_DOMAINS, +} from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface ChannelExpandedMetricsParameters { + limit?: number | string; + offset?: number | string; +} + +export interface ChannelExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getChannelExpandedMetrics( + ...args: [websiteId: string, parameters: ChannelExpandedMetricsParameters, filters?: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { rawQuery, parseFilters } = prisma; + const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + WITH channels as ( + select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' + when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') + when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email' + when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS x, + count(distinct session_id) y + from website_event + ${cohortQuery} + where website_id = {{websiteId::uuid}} + and event_type = {{eventType}} + ${dateQuery} + ${filterQuery} + group by 1, 2 + order by y desc) + + select x, sum(y) y + from channels + where x != '' + group by x + order by y desc; + `, + { ...queryParams, ...parameters }, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { limit = 500, offset = 0 } = parameters; + const { rawQuery, parseFilters } = clickhouse; + const { queryParams, filterQuery, cohortQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when multiSearchAny(url_query, [${toClickHouseStringArray( + PAID_AD_PARAMS, + )}]) != 0 then 'paidAds' + when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SEARCH_DOMAINS, + )}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SOCIAL_DOMAINS, + )}]) != 0 then concat(prefix, 'Social') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + EMAIL_DOMAINS, + )}]) != 0 or position(utm_medium, 'mail') > 0 then 'email' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SHOPPING_DOMAINS, + )}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + VIDEO_DOMAINS, + )}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by prefix, name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} + +function toClickHouseStringArray(arr: string[]): string { + return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', '); +} + +function toPostgresPositionClause(column: string, arr: string[]) { + return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n '); +} diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index 9054b18b..9e6ba82c 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -29,17 +29,35 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` - select - referrer_domain as domain, - url_query as query, - count(distinct session_id) as visitors - from website_event - ${cohortQuery} - where website_id = {{websiteId::uuid}} - ${filterQuery} + WITH channels as ( + select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' + when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') + when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email' + when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS x, + count(distinct session_id) y + from website_event + ${cohortQuery} + where website_id = {{websiteId::uuid}} + and event_type = {{eventType}} ${dateQuery} - group by 1, 2 - order by visitors desc + ${filterQuery} + group by 1, 2 + order by y desc) + + select x, sum(y) y + from channels + where x != '' + group by x + order by y desc; `, queryParams, ); @@ -87,7 +105,6 @@ async function clickhouseQuery( from website_event ${cohortQuery} where website_id = {websiteId:UUID} - and event_type = {eventType:UInt32} ${dateQuery} ${filterQuery} group by 1, 2 @@ -106,3 +123,7 @@ async function clickhouseQuery( function toClickHouseStringArray(arr: string[]): string { return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', '); } + +function toPostgresPositionClause(column: string, arr: string[]) { + return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n '); +} diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts index 24242ad6..4fa15a3c 100644 --- a/src/queries/sql/getWebsiteStats.ts +++ b/src/queries/sql/getWebsiteStats.ts @@ -94,7 +94,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; @@ -117,7 +116,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts new file mode 100644 index 00000000..ebe31d7b --- /dev/null +++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts @@ -0,0 +1,166 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface PageviewExpandedMetricsParameters { + type: string; + limit?: number | string; + offset?: number | string; +} + +export interface PageviewExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getPageviewExpandedMetrics( + ...args: [websiteId: string, parameters: PageviewExpandedMetricsParameters, filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + let entryExitQuery = ''; + let excludeDomain = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and website_event.referrer_domain != website_event.hostname + and website_event.referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; + } + + return rawQuery( + ` + select ${column} x, + count(distinct website_event.session_id) as y + from website_event + ${joinSessionQuery} + ${cohortQuery} + ${entryExitQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${excludeDomain} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); + + let excludeDomain = ''; + let entryExitQuery = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + JOIN (select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id) x + ON x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at`; + } + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + ${entryExitQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${excludeDomain} + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index fd485685..4ebd4cd9 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -136,7 +136,6 @@ async function clickhouseQuery( ${entryExitQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} group by x @@ -174,7 +173,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} ${groupByQuery}) as g diff --git a/src/queries/sql/pageviews/getPageviewStats.ts b/src/queries/sql/pageviews/getPageviewStats.ts index 704dea1e..e9e763e8 100644 --- a/src/queries/sql/pageviews/getPageviewStats.ts +++ b/src/queries/sql/pageviews/getPageviewStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -85,7 +84,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g diff --git a/src/queries/sql/reports/getAttribution.ts b/src/queries/sql/reports/getAttribution.ts index df452343..0bfbd58b 100644 --- a/src/queries/sql/reports/getAttribution.ts +++ b/src/queries/sql/reports/getAttribution.ts @@ -477,7 +477,6 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and ${column} = {step:String} - and event_type = {eventType:UInt32} ${filterQuery} `, queryParams, diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.ts index a39c49ef..7cae67a1 100644 --- a/src/queries/sql/reports/getBreakdown.ts +++ b/src/queries/sql/reports/getBreakdown.ts @@ -115,7 +115,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by ${parseFieldsByName(fields)}, session_id, visit_id diff --git a/src/queries/sql/sessions/getSessionDataValues.ts b/src/queries/sql/sessions/getSessionDataValues.ts index 682bfe4f..e4d04f1a 100644 --- a/src/queries/sql/sessions/getSessionDataValues.ts +++ b/src/queries/sql/sessions/getSessionDataValues.ts @@ -28,9 +28,9 @@ async function relationalQuery( else string_value end as "value", count(distinct session_data.session_id) as "total" - from website_event e + from website_event ${cohortQuery} - join session_data d + join session_data on session_data.session_id = website_event.session_id where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} @@ -58,9 +58,9 @@ async function clickhouseQuery( data_type = 4, toString(date_trunc('hour', date_value)), string_value) as "value", uniq(session_data.session_id) as "total" - from website_event e + from website_event ${cohortQuery} - join session_data d final + join session_data final on session_data.session_id = website_event.session_id where website_event.website_id = {websiteId:UUID} and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64} diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts new file mode 100644 index 00000000..7c6d98f6 --- /dev/null +++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts @@ -0,0 +1,133 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface SessionExpandedMetricsParameters { + type: string; + limit?: number | string; + offset?: number | string; +} + +export interface SessionExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getSessionExpandedMetrics( + ...args: [websiteId: string, parameters: SessionExpandedMetricsParameters, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: SessionExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + let column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = prisma; + const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }, + { + joinSession: SESSION_COLUMNS.includes(type), + }, + ); + const includeCountry = column === 'city' || column === 'region'; + + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + + return rawQuery( + ` + select + ${column} x, + count(distinct website_event.session_id) y + ${includeCountry ? ', country' : ''} + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = {{eventType}} + ${filterQuery} + group by 1 + ${includeCountry ? ', 3' : ''} + order by 2 desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: SessionExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + let column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + const includeCountry = column === 'city' || column === 'region'; + + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + + return rawQuery( + ` + select + name, + ${includeCountry ? 'country,' : ''} + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + ${includeCountry ? 'country,' : ''} + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by name, session_id, visit_id + ${includeCountry ? ', country' : ''} + ) as t + group by name + ${includeCountry ? ', country' : ''} + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts index 55962310..725c7379 100644 --- a/src/queries/sql/sessions/getSessionMetrics.ts +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -25,7 +25,7 @@ async function relationalQuery( filters: QueryFilters, ) { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -39,6 +39,10 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select @@ -68,7 +72,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -77,6 +81,10 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + let sql = ''; if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { @@ -89,7 +97,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} @@ -107,7 +114,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} diff --git a/src/queries/sql/sessions/getSessionStats.ts b/src/queries/sql/sessions/getSessionStats.ts index 258de290..97a8755e 100644 --- a/src/queries/sql/sessions/getSessionStats.ts +++ b/src/queries/sql/sessions/getSessionStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -84,7 +83,6 @@ async function clickhouseQuery( from website_event_stats_hourly as website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g