diff --git a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts index 395971b6..a327bd0b 100644 --- a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts @@ -4,9 +4,9 @@ import { getQueryFilters, parseRequest } from '@/lib/request'; import { badRequest, json, unauthorized } from '@/lib/response'; import { dateRangeParams, filterParams, searchParams } from '@/lib/schema'; import { - getChannelMetrics, - getEventMetrics, - getPageviewMetrics, + getChannelExpandedMetrics, + getEventExpandedMetrics, + getPageviewExpandedMetrics, getSessionExpandedMetrics, } from '@/queries'; import { z } from 'zod'; @@ -46,22 +46,6 @@ export async function GET( if (SESSION_COLUMNS.includes(type)) { const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters); - // if (type === 'language') { - // const combined = {}; - - // for (const { x, y } of data) { - // const key = String(x).toLowerCase().split('-')[0]; - - // if (combined[key] === undefined) { - // combined[key] = { x: key, y }; - // } else { - // combined[key].y += y; - // } - // } - - // return json(Object.values(combined)); - // } - return json(data); } @@ -69,16 +53,16 @@ export async function GET( let data; if (type === 'event') { - data = await getEventMetrics(websiteId, { type, limit, offset }, filters); + data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters); } else { - data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters); + data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters); } return json(data); } if (type === 'channel') { - const data = await getChannelMetrics(websiteId, filters); + const data = await getChannelExpandedMetrics(websiteId, { limit, offset }, filters); return json(data); } diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 49592a0e..bc295d79 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -46,22 +46,6 @@ export async function GET( if (SESSION_COLUMNS.includes(type)) { const data = await getSessionMetrics(websiteId, { type, limit, offset }, filters); - if (type === 'language') { - const combined = {}; - - for (const { x, y } of data) { - const key = String(x).toLowerCase().split('-')[0]; - - if (combined[key] === undefined) { - combined[key] = { x: key, y }; - } else { - combined[key].y += y; - } - } - - return json(Object.values(combined)); - } - return json(data); } diff --git a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts index 0443fd5f..9718dfe1 100644 --- a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts +++ b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts @@ -5,7 +5,7 @@ import { useDateParameters } from '../useDateParameters'; import { ReactQueryOptions } from '@/lib/types'; export type WebsiteExpandedMetricsData = { - label: string; + name: string; pageviews: number; visitors: number; visits: number; diff --git a/src/components/metrics/ListExpandedTable.module.css b/src/components/metrics/ListExpandedTable.module.css new file mode 100644 index 00000000..4bd287c4 --- /dev/null +++ b/src/components/metrics/ListExpandedTable.module.css @@ -0,0 +1,7 @@ +.truncate { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + width: 300px; + display: block; +} diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsx index fb3efdb5..dc27b17f 100644 --- a/src/components/metrics/ListExpandedTable.tsx +++ b/src/components/metrics/ListExpandedTable.tsx @@ -2,23 +2,25 @@ import { useMessages } from '@/components/hooks'; import { formatShortTime } from '@/lib/format'; import { DataColumn, DataTable } from '@umami/react-zen'; import { ReactNode } from 'react'; +import styles from './ListExpandedTable.module.css'; export interface ListExpandedTableProps { data?: any[]; title?: string; + type?: string; renderLabel?: (row: any, index: number) => ReactNode; } -export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) { +export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) { const { formatMessage, labels } = useMessages(); return ( - + {row => renderLabel - ? renderLabel({ x: row?.label, country: row?.['country'] }, Number(row.id)) - : (row.label ?? formatMessage(labels.unknown)) + ? renderLabel({ x: row?.['name'], country: row?.['country'] }, Number(row.id)) + : (row?.['name'] ?? formatMessage(labels.unknown)) } @@ -30,18 +32,26 @@ export function ListExpandedTable({ data = [], title, renderLabel }: ListExpande {row => row?.['pageviews']?.toLocaleString()} - - {row => { - const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; - return Math.round(+n) + '%'; - }} - - - {row => { - const n = (row?.['totaltime'] / row?.['visits']) * 100; - return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; - }} - + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; + return Math.round(+n) + '%'; + }} + + ) : ( + <> + )} + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (row?.['totaltime'] / row?.['visits']) * 100; + return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; + }} + + ) : ( + <> + )} ); } diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index 760a0da6..fb97f8db 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -56,7 +56,6 @@ export function MetricsTable({ websiteId, { type, - limit: 30, search: searchFormattedValues ? undefined : search, ...params, }, @@ -111,6 +110,8 @@ export function MetricsTable({ return []; }, [data, dataFilter, search, limit, formatValue, type]); + const downloadData = expanded ? data : filteredData; + return ( @@ -118,12 +119,12 @@ export function MetricsTable({ {allowSearch && } {children} - {allowDownload && } + {allowDownload && } {data && (expanded ? ( - + ) : ( ))} diff --git a/src/components/metrics/QueryParametersTable.tsx b/src/components/metrics/QueryParametersTable.tsx index e6d85495..c33c23cf 100644 --- a/src/components/metrics/QueryParametersTable.tsx +++ b/src/components/metrics/QueryParametersTable.tsx @@ -58,6 +58,7 @@ export function QueryParametersTable({ dataFilter={filters[filter]} renderLabel={renderLabel} delay={0} + expanded={false} > {allowFilter && } diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index bb184d6b..bcb02d0d 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -90,7 +90,11 @@ function mapFilter(column: string, operator: string, name: string, type: string function getFilterQuery(filters: Record, options: QueryOptions = {}) { const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { if (column) { - arr.push(`and ${mapFilter(column, operator, name)}`); + if (name === 'eventType') { + arr.push(`and ${mapFilter(column, operator, name, 'UInt32')}`); + } else { + arr.push(`and ${mapFilter(column, operator, name)}`); + } if (name === 'referrer') { arr.push(`and referrer_domain != hostname`); diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 1564458d..16bb71af 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -35,7 +35,7 @@ export const EVENT_COLUMNS = [ 'query', 'event', 'tag', - 'host', + 'hostname', ]; export const SESSION_COLUMNS = [ diff --git a/src/queries/index.ts b/src/queries/index.ts index 52ca6513..707dc874 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues'; export * from '@/queries/sql/events/getEventDataStats'; export * from '@/queries/sql/events/getEventDataUsage'; export * from '@/queries/sql/events/getEventMetrics'; +export * from '@/queries/sql/events/getEventExpandedMetrics'; export * from '@/queries/sql/events/getEventStats'; export * from '@/queries/sql/events/getWebsiteEvents'; export * from '@/queries/sql/events/getEventUsage'; @@ -21,6 +22,7 @@ export * from '@/queries/sql/reports/getRetention'; export * from '@/queries/sql/reports/getBreakdown'; export * from '@/queries/sql/reports/getUTM'; export * from '@/queries/sql/pageviews/getPageviewMetrics'; +export * from '@/queries/sql/pageviews/getPageviewExpandedMetrics'; export * from '@/queries/sql/pageviews/getPageviewStats'; export * from '@/queries/sql/sessions/createSession'; export * from '@/queries/sql/sessions/getWebsiteSession'; @@ -37,6 +39,7 @@ export * from '@/queries/sql/sessions/getSessionStats'; export * from '@/queries/sql/sessions/saveSessionData'; export * from '@/queries/sql/getActiveVisitors'; export * from '@/queries/sql/getChannelMetrics'; +export * from '@/queries/sql/getChannelExpandedMetrics'; export * from '@/queries/sql/getRealtimeActivity'; export * from '@/queries/sql/getRealtimeData'; export * from '@/queries/sql/getValues'; diff --git a/src/queries/sql/events/getEventExpandedMetrics.ts b/src/queries/sql/events/getEventExpandedMetrics.ts new file mode 100644 index 00000000..235822b8 --- /dev/null +++ b/src/queries/sql/events/getEventExpandedMetrics.ts @@ -0,0 +1,113 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface EventExpandedMetricParameters { + type: string; + limit?: string; + offset?: string; +} + +export interface EventExpandedMetricData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getEventExpandedMetrics( + ...args: [websiteId: string, parameters: EventExpandedMetricParameters, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +) { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + return rawQuery( + ` + select ${column} x, + count(*) as y + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/events/getEventMetrics.ts b/src/queries/sql/events/getEventMetrics.ts index 73af2278..3d148edf 100644 --- a/src/queries/sql/events/getEventMetrics.ts +++ b/src/queries/sql/events/getEventMetrics.ts @@ -83,7 +83,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x order by y desc diff --git a/src/queries/sql/events/getEventStats.ts b/src/queries/sql/events/getEventStats.ts index ad6b155d..8d26dcfc 100644 --- a/src/queries/sql/events/getEventStats.ts +++ b/src/queries/sql/events/getEventStats.ts @@ -72,7 +72,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x, t order by t diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.ts new file mode 100644 index 00000000..d8fce3d2 --- /dev/null +++ b/src/queries/sql/getChannelExpandedMetrics.ts @@ -0,0 +1,162 @@ +import clickhouse from '@/lib/clickhouse'; +import { + EMAIL_DOMAINS, + EVENT_TYPE, + PAID_AD_PARAMS, + SEARCH_DOMAINS, + SHOPPING_DOMAINS, + SOCIAL_DOMAINS, + VIDEO_DOMAINS, +} from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface ChannelExpandedMetricsParameters { + limit?: number | string; + offset?: number | string; +} + +export interface ChannelExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getChannelExpandedMetrics( + ...args: [websiteId: string, parameters: ChannelExpandedMetricsParameters, filters?: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { rawQuery, parseFilters } = prisma; + const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + WITH channels as ( + select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' + when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') + when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email' + when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS x, + count(distinct session_id) y + from website_event + ${cohortQuery} + where website_id = {{websiteId::uuid}} + and event_type = {{eventType}} + ${dateQuery} + ${filterQuery} + group by 1, 2 + order by y desc) + + select x, sum(y) y + from channels + where x != '' + group by x + order by y desc; + `, + { ...queryParams, ...parameters }, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { limit = 500, offset = 0 } = parameters; + const { rawQuery, parseFilters } = clickhouse; + const { queryParams, filterQuery, cohortQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when multiSearchAny(url_query, [${toClickHouseStringArray( + PAID_AD_PARAMS, + )}]) != 0 then 'paidAds' + when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SEARCH_DOMAINS, + )}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SOCIAL_DOMAINS, + )}]) != 0 then concat(prefix, 'Social') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + EMAIL_DOMAINS, + )}]) != 0 or position(utm_medium, 'mail') > 0 then 'email' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SHOPPING_DOMAINS, + )}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + VIDEO_DOMAINS, + )}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by prefix, name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} + +function toClickHouseStringArray(arr: string[]): string { + return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', '); +} + +function toPostgresPositionClause(column: string, arr: string[]) { + return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n '); +} diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index c240cc86..9e6ba82c 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -105,7 +105,6 @@ async function clickhouseQuery( from website_event ${cohortQuery} where website_id = {websiteId:UUID} - and event_type = {eventType:UInt32} ${dateQuery} ${filterQuery} group by 1, 2 diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts index 24242ad6..4fa15a3c 100644 --- a/src/queries/sql/getWebsiteStats.ts +++ b/src/queries/sql/getWebsiteStats.ts @@ -94,7 +94,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; @@ -117,7 +116,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts new file mode 100644 index 00000000..ebe31d7b --- /dev/null +++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts @@ -0,0 +1,166 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface PageviewExpandedMetricsParameters { + type: string; + limit?: number | string; + offset?: number | string; +} + +export interface PageviewExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getPageviewExpandedMetrics( + ...args: [websiteId: string, parameters: PageviewExpandedMetricsParameters, filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + let entryExitQuery = ''; + let excludeDomain = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and website_event.referrer_domain != website_event.hostname + and website_event.referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; + } + + return rawQuery( + ` + select ${column} x, + count(distinct website_event.session_id) as y + from website_event + ${joinSessionQuery} + ${cohortQuery} + ${entryExitQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${excludeDomain} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); + + let excludeDomain = ''; + let entryExitQuery = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + JOIN (select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id) x + ON x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at`; + } + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + ${entryExitQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${excludeDomain} + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index fd485685..4ebd4cd9 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -136,7 +136,6 @@ async function clickhouseQuery( ${entryExitQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} group by x @@ -174,7 +173,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} ${groupByQuery}) as g diff --git a/src/queries/sql/pageviews/getPageviewStats.ts b/src/queries/sql/pageviews/getPageviewStats.ts index 704dea1e..e9e763e8 100644 --- a/src/queries/sql/pageviews/getPageviewStats.ts +++ b/src/queries/sql/pageviews/getPageviewStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -85,7 +84,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g diff --git a/src/queries/sql/reports/getAttribution.ts b/src/queries/sql/reports/getAttribution.ts index df452343..0bfbd58b 100644 --- a/src/queries/sql/reports/getAttribution.ts +++ b/src/queries/sql/reports/getAttribution.ts @@ -477,7 +477,6 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and ${column} = {step:String} - and event_type = {eventType:UInt32} ${filterQuery} `, queryParams, diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.ts index a39c49ef..7cae67a1 100644 --- a/src/queries/sql/reports/getBreakdown.ts +++ b/src/queries/sql/reports/getBreakdown.ts @@ -115,7 +115,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by ${parseFieldsByName(fields)}, session_id, visit_id diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts index ba0841dc..7c6d98f6 100644 --- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts +++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts @@ -11,7 +11,7 @@ export interface SessionExpandedMetricsParameters { } export interface SessionExpandedMetricsData { - label: string; + name: string; pageviews: number; visitors: number; visits: number; @@ -34,7 +34,7 @@ async function relationalQuery( filters: QueryFilters, ): Promise { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -48,6 +48,10 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select @@ -77,7 +81,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -86,10 +90,14 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select - label, + name, ${includeCountry ? 'country,' : ''} sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", @@ -98,7 +106,7 @@ async function clickhouseQuery( sum(max_time-min_time) as "totaltime" from ( select - ${column} label, + ${column} name, ${includeCountry ? 'country,' : ''} session_id, visit_id, @@ -109,13 +117,12 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - and label != '' + and name != '' ${filterQuery} - group by label, session_id, visit_id + group by name, session_id, visit_id ${includeCountry ? ', country' : ''} ) as t - group by label + group by name ${includeCountry ? ', country' : ''} order by visitors desc, visits desc limit ${limit} diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts index 55962310..725c7379 100644 --- a/src/queries/sql/sessions/getSessionMetrics.ts +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -25,7 +25,7 @@ async function relationalQuery( filters: QueryFilters, ) { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -39,6 +39,10 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select @@ -68,7 +72,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -77,6 +81,10 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + let sql = ''; if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { @@ -89,7 +97,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} @@ -107,7 +114,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} diff --git a/src/queries/sql/sessions/getSessionStats.ts b/src/queries/sql/sessions/getSessionStats.ts index 258de290..97a8755e 100644 --- a/src/queries/sql/sessions/getSessionStats.ts +++ b/src/queries/sql/sessions/getSessionStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -84,7 +83,6 @@ async function clickhouseQuery( from website_event_stats_hourly as website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g