From c8b4ee8ca55a64ed0634cc1cbfdab0b3c33f6a08 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 31 Jul 2025 09:45:19 -0700 Subject: [PATCH 1/3] update psql getChannelMetrics --- src/queries/sql/getChannelMetrics.ts | 42 +++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index 9054b18b..c240cc86 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -29,17 +29,35 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` - select - referrer_domain as domain, - url_query as query, - count(distinct session_id) as visitors - from website_event - ${cohortQuery} - where website_id = {{websiteId::uuid}} - ${filterQuery} + WITH channels as ( + select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' + when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') + when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email' + when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS x, + count(distinct session_id) y + from website_event + ${cohortQuery} + where website_id = {{websiteId::uuid}} + and event_type = {{eventType}} ${dateQuery} - group by 1, 2 - order by visitors desc + ${filterQuery} + group by 1, 2 + order by y desc) + + select x, sum(y) y + from channels + where x != '' + group by x + order by y desc; `, queryParams, ); @@ -106,3 +124,7 @@ async function clickhouseQuery( function toClickHouseStringArray(arr: string[]): string { return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', '); } + +function toPostgresPositionClause(column: string, arr: string[]) { + return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n '); +} From c1cad16cb9453d3e2de95a2c9c29494e8e6c1dbd Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 4 Aug 2025 18:27:31 -0700 Subject: [PATCH 2/3] implement sessions metric expanded queries --- src/app/(main)/App.tsx | 2 +- .../[websiteId]/WebsiteExpandedView.tsx | 1 + .../[websiteId]/metrics/expanded/route.ts | 87 ++++++++++++ src/components/hooks/index.ts | 1 + .../queries/useWebsiteExpandedMetricsQuery.ts | 45 +++++++ src/components/metrics/ListExpandedTable.tsx | 47 +++++++ src/components/metrics/MetricsTable.tsx | 47 +++++-- src/queries/index.ts | 1 + .../sql/sessions/getSessionDataValues.ts | 8 +- .../sql/sessions/getSessionExpandedMetrics.ts | 126 ++++++++++++++++++ 10 files changed, 352 insertions(+), 13 deletions(-) create mode 100644 src/app/api/websites/[websiteId]/metrics/expanded/route.ts create mode 100644 src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts create mode 100644 src/components/metrics/ListExpandedTable.tsx create mode 100644 src/queries/sql/sessions/getSessionExpandedMetrics.ts diff --git a/src/app/(main)/App.tsx b/src/app/(main)/App.tsx index e3602d5a..ee453941 100644 --- a/src/app/(main)/App.tsx +++ b/src/app/(main)/App.tsx @@ -30,7 +30,7 @@ export function App({ children }) { - + {children} diff --git a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx index acd72ab9..e6788ae9 100644 --- a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx +++ b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx @@ -156,6 +156,7 @@ export function WebsiteExpandedView({ itemCount={25} allowFilter={true} allowSearch={true} + expanded={true} /> diff --git a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts new file mode 100644 index 00000000..395971b6 --- /dev/null +++ b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts @@ -0,0 +1,87 @@ +import { canViewWebsite } from '@/lib/auth'; +import { EVENT_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { badRequest, json, unauthorized } from '@/lib/response'; +import { dateRangeParams, filterParams, searchParams } from '@/lib/schema'; +import { + getChannelMetrics, + getEventMetrics, + getPageviewMetrics, + getSessionExpandedMetrics, +} from '@/queries'; +import { z } from 'zod'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + type: z.string(), + limit: z.coerce.number().optional(), + offset: z.coerce.number().optional(), + ...dateRangeParams, + ...searchParams, + ...filterParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const { type, limit, offset, search } = query; + const filters = await getQueryFilters(query, websiteId); + + if (search) { + filters[type] = `c.${search}`; + } + + if (SESSION_COLUMNS.includes(type)) { + const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters); + + // if (type === 'language') { + // const combined = {}; + + // for (const { x, y } of data) { + // const key = String(x).toLowerCase().split('-')[0]; + + // if (combined[key] === undefined) { + // combined[key] = { x: key, y }; + // } else { + // combined[key].y += y; + // } + // } + + // return json(Object.values(combined)); + // } + + return json(data); + } + + if (EVENT_COLUMNS.includes(type)) { + let data; + + if (type === 'event') { + data = await getEventMetrics(websiteId, { type, limit, offset }, filters); + } else { + data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters); + } + + return json(data); + } + + if (type === 'channel') { + const data = await getChannelMetrics(websiteId, filters); + + return json(data); + } + + return badRequest(); +} diff --git a/src/components/hooks/index.ts b/src/components/hooks/index.ts index 5c0d8d6b..e62479fb 100644 --- a/src/components/hooks/index.ts +++ b/src/components/hooks/index.ts @@ -31,6 +31,7 @@ export * from './queries/useWebsitesQuery'; export * from './queries/useWebsiteEventsQuery'; export * from './queries/useWebsiteEventsSeriesQuery'; export * from './queries/useWebsiteMetricsQuery'; +export * from './queries/useWebsiteExpandedMetricsQuery'; export * from './queries/useWebsiteValuesQuery'; export * from './useApi'; export * from './useConfig'; diff --git a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts new file mode 100644 index 00000000..0443fd5f --- /dev/null +++ b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts @@ -0,0 +1,45 @@ +import { keepPreviousData } from '@tanstack/react-query'; +import { useApi } from '../useApi'; +import { useFilterParameters } from '../useFilterParameters'; +import { useDateParameters } from '../useDateParameters'; +import { ReactQueryOptions } from '@/lib/types'; + +export type WebsiteExpandedMetricsData = { + label: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +}[]; + +export function useWebsiteExpandedMetricsQuery( + websiteId: string, + params: { type: string; limit?: number; search?: string }, + options?: ReactQueryOptions, +) { + const { get, useQuery } = useApi(); + const date = useDateParameters(websiteId); + const filters = useFilterParameters(); + + return useQuery({ + queryKey: [ + 'websites:metrics:expanded', + { + websiteId, + ...date, + ...filters, + ...params, + }, + ], + queryFn: async () => + get(`/websites/${websiteId}/metrics/expanded`, { + ...date, + ...filters, + ...params, + }), + enabled: !!websiteId, + placeholderData: keepPreviousData, + ...options, + }); +} diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsx new file mode 100644 index 00000000..fb3efdb5 --- /dev/null +++ b/src/components/metrics/ListExpandedTable.tsx @@ -0,0 +1,47 @@ +import { useMessages } from '@/components/hooks'; +import { formatShortTime } from '@/lib/format'; +import { DataColumn, DataTable } from '@umami/react-zen'; +import { ReactNode } from 'react'; + +export interface ListExpandedTableProps { + data?: any[]; + title?: string; + renderLabel?: (row: any, index: number) => ReactNode; +} + +export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) { + const { formatMessage, labels } = useMessages(); + + return ( + + + {row => + renderLabel + ? renderLabel({ x: row?.label, country: row?.['country'] }, Number(row.id)) + : (row.label ?? formatMessage(labels.unknown)) + } + + + {row => row?.['visitors']?.toLocaleString()} + + + {row => row?.['visits']?.toLocaleString()} + + + {row => row?.['pageviews']?.toLocaleString()} + + + {row => { + const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; + return Math.round(+n) + '%'; + }} + + + {row => { + const n = (row?.['totaltime'] / row?.['visits']) * 100; + return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; + }} + + + ); +} diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index c229b00c..760a0da6 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -1,13 +1,20 @@ -import { ReactNode, useMemo, useState } from 'react'; -import { Icon, Text, SearchField, Row, Column } from '@umami/react-zen'; import { LinkButton } from '@/components/common/LinkButton'; +import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { + useFormat, + useMessages, + useNavigation, + useWebsiteExpandedMetricsQuery, + useWebsiteMetricsQuery, +} from '@/components/hooks'; +import { Arrow } from '@/components/icons'; +import { DownloadButton } from '@/components/input/DownloadButton'; import { DEFAULT_ANIMATION_DURATION } from '@/lib/constants'; import { percentFilter } from '@/lib/filters'; -import { useNavigation, useWebsiteMetricsQuery, useMessages, useFormat } from '@/components/hooks'; -import { Arrow } from '@/components/icons'; +import { Column, Icon, Row, SearchField, Text } from '@umami/react-zen'; +import { ReactNode, useMemo, useState } from 'react'; +import { ListExpandedTable, ListExpandedTableProps } from './ListExpandedTable'; import { ListTable, ListTableProps } from './ListTable'; -import { LoadingPanel } from '@/components/common/LoadingPanel'; -import { DownloadButton } from '@/components/input/DownloadButton'; export interface MetricsTableProps extends ListTableProps { websiteId: string; @@ -21,6 +28,7 @@ export interface MetricsTableProps extends ListTableProps { showMore?: boolean; params?: { [key: string]: any }; allowDownload?: boolean; + expanded?: boolean; children?: ReactNode; } @@ -35,6 +43,7 @@ export function MetricsTable({ showMore = true, params, allowDownload = true, + expanded = false, children, ...props }: MetricsTableProps) { @@ -43,7 +52,21 @@ export function MetricsTable({ const { updateParams } = useNavigation(); const { formatMessage, labels } = useMessages(); - const { data, isLoading, isFetching, error } = useWebsiteMetricsQuery( + const expandedQuery = useWebsiteExpandedMetricsQuery( + websiteId, + { + type, + limit: 30, + search: searchFormattedValues ? undefined : search, + ...params, + }, + { + retryDelay: delay || DEFAULT_ANIMATION_DURATION, + enabled: expanded, + }, + ); + + const query = useWebsiteMetricsQuery( websiteId, { type, @@ -53,9 +76,12 @@ export function MetricsTable({ }, { retryDelay: delay || DEFAULT_ANIMATION_DURATION, + enabled: !expanded, }, ); + const { data, isLoading, isFetching, error } = expanded ? expandedQuery : query; + const filteredData = useMemo(() => { if (data) { let items = data as any[]; @@ -95,7 +121,12 @@ export function MetricsTable({ {allowDownload && } - {data && } + {data && + (expanded ? ( + + ) : ( + + ))} {showMore && data && !error && limit && ( diff --git a/src/queries/index.ts b/src/queries/index.ts index fba7e548..52ca6513 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -28,6 +28,7 @@ export * from '@/queries/sql/sessions/getSessionData'; export * from '@/queries/sql/sessions/getSessionDataProperties'; export * from '@/queries/sql/sessions/getSessionDataValues'; export * from '@/queries/sql/sessions/getSessionMetrics'; +export * from '@/queries/sql/sessions/getSessionExpandedMetrics'; export * from '@/queries/sql/sessions/getWebsiteSessions'; export * from '@/queries/sql/sessions/getWebsiteSessionStats'; export * from '@/queries/sql/sessions/getWebsiteSessionsWeekly'; diff --git a/src/queries/sql/sessions/getSessionDataValues.ts b/src/queries/sql/sessions/getSessionDataValues.ts index 682bfe4f..e4d04f1a 100644 --- a/src/queries/sql/sessions/getSessionDataValues.ts +++ b/src/queries/sql/sessions/getSessionDataValues.ts @@ -28,9 +28,9 @@ async function relationalQuery( else string_value end as "value", count(distinct session_data.session_id) as "total" - from website_event e + from website_event ${cohortQuery} - join session_data d + join session_data on session_data.session_id = website_event.session_id where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} @@ -58,9 +58,9 @@ async function clickhouseQuery( data_type = 4, toString(date_trunc('hour', date_value)), string_value) as "value", uniq(session_data.session_id) as "total" - from website_event e + from website_event ${cohortQuery} - join session_data d final + join session_data final on session_data.session_id = website_event.session_id where website_event.website_id = {websiteId:UUID} and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64} diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts new file mode 100644 index 00000000..ba0841dc --- /dev/null +++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts @@ -0,0 +1,126 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface SessionExpandedMetricsParameters { + type: string; + limit?: number | string; + offset?: number | string; +} + +export interface SessionExpandedMetricsData { + label: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getSessionExpandedMetrics( + ...args: [websiteId: string, parameters: SessionExpandedMetricsParameters, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: SessionExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = prisma; + const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }, + { + joinSession: SESSION_COLUMNS.includes(type), + }, + ); + const includeCountry = column === 'city' || column === 'region'; + + return rawQuery( + ` + select + ${column} x, + count(distinct website_event.session_id) y + ${includeCountry ? ', country' : ''} + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = {{eventType}} + ${filterQuery} + group by 1 + ${includeCountry ? ', 3' : ''} + order by 2 desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: SessionExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + const includeCountry = column === 'city' || column === 'region'; + + return rawQuery( + ` + select + label, + ${includeCountry ? 'country,' : ''} + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} label, + ${includeCountry ? 'country,' : ''} + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + and label != '' + ${filterQuery} + group by label, session_id, visit_id + ${includeCountry ? ', country' : ''} + ) as t + group by label + ${includeCountry ? ', country' : ''} + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} From 0a0c1f27c6b7ec9ad5c9933a0d06cfdb0cbfc427 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 5 Aug 2025 00:37:44 -0700 Subject: [PATCH 3/3] implement pageviews, events, and channels queries --- .../[websiteId]/metrics/expanded/route.ts | 28 +-- .../api/websites/[websiteId]/metrics/route.ts | 16 -- .../queries/useWebsiteExpandedMetricsQuery.ts | 2 +- .../metrics/ListExpandedTable.module.css | 7 + src/components/metrics/ListExpandedTable.tsx | 42 +++-- src/components/metrics/MetricsTable.tsx | 7 +- .../metrics/QueryParametersTable.tsx | 1 + src/lib/clickhouse.ts | 6 +- src/lib/constants.ts | 2 +- src/queries/index.ts | 3 + .../sql/events/getEventExpandedMetrics.ts | 113 ++++++++++++ src/queries/sql/events/getEventMetrics.ts | 1 - src/queries/sql/events/getEventStats.ts | 1 - src/queries/sql/getChannelExpandedMetrics.ts | 162 +++++++++++++++++ src/queries/sql/getChannelMetrics.ts | 1 - src/queries/sql/getWebsiteStats.ts | 2 - .../pageviews/getPageviewExpandedMetrics.ts | 166 ++++++++++++++++++ .../sql/pageviews/getPageviewMetrics.ts | 2 - src/queries/sql/pageviews/getPageviewStats.ts | 2 - src/queries/sql/reports/getAttribution.ts | 1 - src/queries/sql/reports/getBreakdown.ts | 1 - .../sql/sessions/getSessionExpandedMetrics.ts | 25 ++- src/queries/sql/sessions/getSessionMetrics.ts | 14 +- src/queries/sql/sessions/getSessionStats.ts | 2 - 24 files changed, 521 insertions(+), 86 deletions(-) create mode 100644 src/components/metrics/ListExpandedTable.module.css create mode 100644 src/queries/sql/events/getEventExpandedMetrics.ts create mode 100644 src/queries/sql/getChannelExpandedMetrics.ts create mode 100644 src/queries/sql/pageviews/getPageviewExpandedMetrics.ts diff --git a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts index 395971b6..a327bd0b 100644 --- a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts @@ -4,9 +4,9 @@ import { getQueryFilters, parseRequest } from '@/lib/request'; import { badRequest, json, unauthorized } from '@/lib/response'; import { dateRangeParams, filterParams, searchParams } from '@/lib/schema'; import { - getChannelMetrics, - getEventMetrics, - getPageviewMetrics, + getChannelExpandedMetrics, + getEventExpandedMetrics, + getPageviewExpandedMetrics, getSessionExpandedMetrics, } from '@/queries'; import { z } from 'zod'; @@ -46,22 +46,6 @@ export async function GET( if (SESSION_COLUMNS.includes(type)) { const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters); - // if (type === 'language') { - // const combined = {}; - - // for (const { x, y } of data) { - // const key = String(x).toLowerCase().split('-')[0]; - - // if (combined[key] === undefined) { - // combined[key] = { x: key, y }; - // } else { - // combined[key].y += y; - // } - // } - - // return json(Object.values(combined)); - // } - return json(data); } @@ -69,16 +53,16 @@ export async function GET( let data; if (type === 'event') { - data = await getEventMetrics(websiteId, { type, limit, offset }, filters); + data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters); } else { - data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters); + data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters); } return json(data); } if (type === 'channel') { - const data = await getChannelMetrics(websiteId, filters); + const data = await getChannelExpandedMetrics(websiteId, { limit, offset }, filters); return json(data); } diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 49592a0e..bc295d79 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -46,22 +46,6 @@ export async function GET( if (SESSION_COLUMNS.includes(type)) { const data = await getSessionMetrics(websiteId, { type, limit, offset }, filters); - if (type === 'language') { - const combined = {}; - - for (const { x, y } of data) { - const key = String(x).toLowerCase().split('-')[0]; - - if (combined[key] === undefined) { - combined[key] = { x: key, y }; - } else { - combined[key].y += y; - } - } - - return json(Object.values(combined)); - } - return json(data); } diff --git a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts index 0443fd5f..9718dfe1 100644 --- a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts +++ b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts @@ -5,7 +5,7 @@ import { useDateParameters } from '../useDateParameters'; import { ReactQueryOptions } from '@/lib/types'; export type WebsiteExpandedMetricsData = { - label: string; + name: string; pageviews: number; visitors: number; visits: number; diff --git a/src/components/metrics/ListExpandedTable.module.css b/src/components/metrics/ListExpandedTable.module.css new file mode 100644 index 00000000..4bd287c4 --- /dev/null +++ b/src/components/metrics/ListExpandedTable.module.css @@ -0,0 +1,7 @@ +.truncate { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + width: 300px; + display: block; +} diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsx index fb3efdb5..dc27b17f 100644 --- a/src/components/metrics/ListExpandedTable.tsx +++ b/src/components/metrics/ListExpandedTable.tsx @@ -2,23 +2,25 @@ import { useMessages } from '@/components/hooks'; import { formatShortTime } from '@/lib/format'; import { DataColumn, DataTable } from '@umami/react-zen'; import { ReactNode } from 'react'; +import styles from './ListExpandedTable.module.css'; export interface ListExpandedTableProps { data?: any[]; title?: string; + type?: string; renderLabel?: (row: any, index: number) => ReactNode; } -export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) { +export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) { const { formatMessage, labels } = useMessages(); return ( - + {row => renderLabel - ? renderLabel({ x: row?.label, country: row?.['country'] }, Number(row.id)) - : (row.label ?? formatMessage(labels.unknown)) + ? renderLabel({ x: row?.['name'], country: row?.['country'] }, Number(row.id)) + : (row?.['name'] ?? formatMessage(labels.unknown)) } @@ -30,18 +32,26 @@ export function ListExpandedTable({ data = [], title, renderLabel }: ListExpande {row => row?.['pageviews']?.toLocaleString()} - - {row => { - const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; - return Math.round(+n) + '%'; - }} - - - {row => { - const n = (row?.['totaltime'] / row?.['visits']) * 100; - return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; - }} - + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; + return Math.round(+n) + '%'; + }} + + ) : ( + <> + )} + {type !== 'exit' && type !== 'entry' ? ( + + {row => { + const n = (row?.['totaltime'] / row?.['visits']) * 100; + return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; + }} + + ) : ( + <> + )} ); } diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index 760a0da6..fb97f8db 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -56,7 +56,6 @@ export function MetricsTable({ websiteId, { type, - limit: 30, search: searchFormattedValues ? undefined : search, ...params, }, @@ -111,6 +110,8 @@ export function MetricsTable({ return []; }, [data, dataFilter, search, limit, formatValue, type]); + const downloadData = expanded ? data : filteredData; + return ( @@ -118,12 +119,12 @@ export function MetricsTable({ {allowSearch && } {children} - {allowDownload && } + {allowDownload && } {data && (expanded ? ( - + ) : ( ))} diff --git a/src/components/metrics/QueryParametersTable.tsx b/src/components/metrics/QueryParametersTable.tsx index e6d85495..c33c23cf 100644 --- a/src/components/metrics/QueryParametersTable.tsx +++ b/src/components/metrics/QueryParametersTable.tsx @@ -58,6 +58,7 @@ export function QueryParametersTable({ dataFilter={filters[filter]} renderLabel={renderLabel} delay={0} + expanded={false} > {allowFilter && } diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index bb184d6b..bcb02d0d 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -90,7 +90,11 @@ function mapFilter(column: string, operator: string, name: string, type: string function getFilterQuery(filters: Record, options: QueryOptions = {}) { const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => { if (column) { - arr.push(`and ${mapFilter(column, operator, name)}`); + if (name === 'eventType') { + arr.push(`and ${mapFilter(column, operator, name, 'UInt32')}`); + } else { + arr.push(`and ${mapFilter(column, operator, name)}`); + } if (name === 'referrer') { arr.push(`and referrer_domain != hostname`); diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 1564458d..16bb71af 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -35,7 +35,7 @@ export const EVENT_COLUMNS = [ 'query', 'event', 'tag', - 'host', + 'hostname', ]; export const SESSION_COLUMNS = [ diff --git a/src/queries/index.ts b/src/queries/index.ts index 52ca6513..707dc874 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues'; export * from '@/queries/sql/events/getEventDataStats'; export * from '@/queries/sql/events/getEventDataUsage'; export * from '@/queries/sql/events/getEventMetrics'; +export * from '@/queries/sql/events/getEventExpandedMetrics'; export * from '@/queries/sql/events/getEventStats'; export * from '@/queries/sql/events/getWebsiteEvents'; export * from '@/queries/sql/events/getEventUsage'; @@ -21,6 +22,7 @@ export * from '@/queries/sql/reports/getRetention'; export * from '@/queries/sql/reports/getBreakdown'; export * from '@/queries/sql/reports/getUTM'; export * from '@/queries/sql/pageviews/getPageviewMetrics'; +export * from '@/queries/sql/pageviews/getPageviewExpandedMetrics'; export * from '@/queries/sql/pageviews/getPageviewStats'; export * from '@/queries/sql/sessions/createSession'; export * from '@/queries/sql/sessions/getWebsiteSession'; @@ -37,6 +39,7 @@ export * from '@/queries/sql/sessions/getSessionStats'; export * from '@/queries/sql/sessions/saveSessionData'; export * from '@/queries/sql/getActiveVisitors'; export * from '@/queries/sql/getChannelMetrics'; +export * from '@/queries/sql/getChannelExpandedMetrics'; export * from '@/queries/sql/getRealtimeActivity'; export * from '@/queries/sql/getRealtimeData'; export * from '@/queries/sql/getValues'; diff --git a/src/queries/sql/events/getEventExpandedMetrics.ts b/src/queries/sql/events/getEventExpandedMetrics.ts new file mode 100644 index 00000000..235822b8 --- /dev/null +++ b/src/queries/sql/events/getEventExpandedMetrics.ts @@ -0,0 +1,113 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface EventExpandedMetricParameters { + type: string; + limit?: string; + offset?: string; +} + +export interface EventExpandedMetricData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getEventExpandedMetrics( + ...args: [websiteId: string, parameters: EventExpandedMetricParameters, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +) { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + return rawQuery( + ` + select ${column} x, + count(*) as y + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: EventExpandedMetricParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/events/getEventMetrics.ts b/src/queries/sql/events/getEventMetrics.ts index 73af2278..3d148edf 100644 --- a/src/queries/sql/events/getEventMetrics.ts +++ b/src/queries/sql/events/getEventMetrics.ts @@ -83,7 +83,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x order by y desc diff --git a/src/queries/sql/events/getEventStats.ts b/src/queries/sql/events/getEventStats.ts index ad6b155d..8d26dcfc 100644 --- a/src/queries/sql/events/getEventStats.ts +++ b/src/queries/sql/events/getEventStats.ts @@ -72,7 +72,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x, t order by t diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.ts new file mode 100644 index 00000000..d8fce3d2 --- /dev/null +++ b/src/queries/sql/getChannelExpandedMetrics.ts @@ -0,0 +1,162 @@ +import clickhouse from '@/lib/clickhouse'; +import { + EMAIL_DOMAINS, + EVENT_TYPE, + PAID_AD_PARAMS, + SEARCH_DOMAINS, + SHOPPING_DOMAINS, + SOCIAL_DOMAINS, + VIDEO_DOMAINS, +} from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface ChannelExpandedMetricsParameters { + limit?: number | string; + offset?: number | string; +} + +export interface ChannelExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getChannelExpandedMetrics( + ...args: [websiteId: string, parameters: ChannelExpandedMetricsParameters, filters?: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { rawQuery, parseFilters } = prisma; + const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + WITH channels as ( + select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds' + when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social') + when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email' + when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS x, + count(distinct session_id) y + from website_event + ${cohortQuery} + where website_id = {{websiteId::uuid}} + and event_type = {{eventType}} + ${dateQuery} + ${filterQuery} + group by 1, 2 + order by y desc) + + select x, sum(y) y + from channels + where x != '' + group by x + order by y desc; + `, + { ...queryParams, ...parameters }, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: ChannelExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { limit = 500, offset = 0 } = parameters; + const { rawQuery, parseFilters } = clickhouse; + const { queryParams, filterQuery, cohortQuery } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix, + case + when referrer_domain = '' and url_query = '' then 'direct' + when multiSearchAny(url_query, [${toClickHouseStringArray( + PAID_AD_PARAMS, + )}]) != 0 then 'paidAds' + when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral' + when position(utm_medium, 'affiliate') > 0 then 'affiliate' + when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SEARCH_DOMAINS, + )}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SOCIAL_DOMAINS, + )}]) != 0 then concat(prefix, 'Social') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + EMAIL_DOMAINS, + )}]) != 0 or position(utm_medium, 'mail') > 0 then 'email' + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + SHOPPING_DOMAINS, + )}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping') + when multiSearchAny(referrer_domain, [${toClickHouseStringArray( + VIDEO_DOMAINS, + )}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video') + else '' end AS name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${filterQuery} + group by prefix, name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} + +function toClickHouseStringArray(arr: string[]): string { + return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', '); +} + +function toPostgresPositionClause(column: string, arr: string[]) { + return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n '); +} diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index c240cc86..9e6ba82c 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -105,7 +105,6 @@ async function clickhouseQuery( from website_event ${cohortQuery} where website_id = {websiteId:UUID} - and event_type = {eventType:UInt32} ${dateQuery} ${filterQuery} group by 1, 2 diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts index 24242ad6..4fa15a3c 100644 --- a/src/queries/sql/getWebsiteStats.ts +++ b/src/queries/sql/getWebsiteStats.ts @@ -94,7 +94,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; @@ -117,7 +116,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by session_id, visit_id ) as t; diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts new file mode 100644 index 00000000..ebe31d7b --- /dev/null +++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts @@ -0,0 +1,166 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface PageviewExpandedMetricsParameters { + type: string; + limit?: number | string; + offset?: number | string; +} + +export interface PageviewExpandedMetricsData { + name: string; + pageviews: number; + visitors: number; + visits: number; + bounces: number; + totaltime: number; +} + +export async function getPageviewExpandedMetrics( + ...args: [websiteId: string, parameters: PageviewExpandedMetricsParameters, filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); + + let entryExitQuery = ''; + let excludeDomain = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and website_event.referrer_domain != website_event.hostname + and website_event.referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; + } + + return rawQuery( + ` + select ${column} x, + count(distinct website_event.session_id) as y + from website_event + ${joinSessionQuery} + ${cohortQuery} + ${entryExitQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${excludeDomain} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + parameters: PageviewExpandedMetricsParameters, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); + + let excludeDomain = ''; + let entryExitQuery = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + JOIN (select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id) x + ON x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at`; + } + + return rawQuery( + ` + select + name, + sum(t.c) as "pageviews", + uniq(t.session_id) as "visitors", + uniq(t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime" + from ( + select + ${column} name, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + ${cohortQuery} + ${entryExitQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and name != '' + ${excludeDomain} + ${filterQuery} + group by name, session_id, visit_id + ) as t + group by name + order by visitors desc, visits desc + limit ${limit} + offset ${offset} + `, + { ...queryParams, ...parameters }, + ); +} diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index fd485685..4ebd4cd9 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -136,7 +136,6 @@ async function clickhouseQuery( ${entryExitQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} group by x @@ -174,7 +173,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} ${groupByQuery}) as g diff --git a/src/queries/sql/pageviews/getPageviewStats.ts b/src/queries/sql/pageviews/getPageviewStats.ts index 704dea1e..e9e763e8 100644 --- a/src/queries/sql/pageviews/getPageviewStats.ts +++ b/src/queries/sql/pageviews/getPageviewStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -85,7 +84,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g diff --git a/src/queries/sql/reports/getAttribution.ts b/src/queries/sql/reports/getAttribution.ts index df452343..0bfbd58b 100644 --- a/src/queries/sql/reports/getAttribution.ts +++ b/src/queries/sql/reports/getAttribution.ts @@ -477,7 +477,6 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and ${column} = {step:String} - and event_type = {eventType:UInt32} ${filterQuery} `, queryParams, diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.ts index a39c49ef..7cae67a1 100644 --- a/src/queries/sql/reports/getBreakdown.ts +++ b/src/queries/sql/reports/getBreakdown.ts @@ -115,7 +115,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by ${parseFieldsByName(fields)}, session_id, visit_id diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts index ba0841dc..7c6d98f6 100644 --- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts +++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts @@ -11,7 +11,7 @@ export interface SessionExpandedMetricsParameters { } export interface SessionExpandedMetricsData { - label: string; + name: string; pageviews: number; visitors: number; visits: number; @@ -34,7 +34,7 @@ async function relationalQuery( filters: QueryFilters, ): Promise { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -48,6 +48,10 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select @@ -77,7 +81,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -86,10 +90,14 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select - label, + name, ${includeCountry ? 'country,' : ''} sum(t.c) as "pageviews", uniq(t.session_id) as "visitors", @@ -98,7 +106,7 @@ async function clickhouseQuery( sum(max_time-min_time) as "totaltime" from ( select - ${column} label, + ${column} name, ${includeCountry ? 'country,' : ''} session_id, visit_id, @@ -109,13 +117,12 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - and label != '' + and name != '' ${filterQuery} - group by label, session_id, visit_id + group by name, session_id, visit_id ${includeCountry ? ', country' : ''} ) as t - group by label + group by name ${includeCountry ? ', country' : ''} order by visitors desc, visits desc limit ${limit} diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts index 55962310..725c7379 100644 --- a/src/queries/sql/sessions/getSessionMetrics.ts +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -25,7 +25,7 @@ async function relationalQuery( filters: QueryFilters, ) { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -39,6 +39,10 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + return rawQuery( ` select @@ -68,7 +72,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { parseFilters, rawQuery } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -77,6 +81,10 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + if (type === 'language') { + column = `lower(left(${type}, 2))`; + } + let sql = ''; if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { @@ -89,7 +97,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} @@ -107,7 +114,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by x ${includeCountry ? ', country' : ''} diff --git a/src/queries/sql/sessions/getSessionStats.ts b/src/queries/sql/sessions/getSessionStats.ts index 258de290..97a8755e 100644 --- a/src/queries/sql/sessions/getSessionStats.ts +++ b/src/queries/sql/sessions/getSessionStats.ts @@ -66,7 +66,6 @@ async function clickhouseQuery( ${cohortQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g @@ -84,7 +83,6 @@ async function clickhouseQuery( from website_event_stats_hourly as website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} ${filterQuery} group by t ) as g