From 4b3b9f1ee1898a872b593fdf3cd1dba1dc55ef73 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 12 Feb 2025 09:51:45 -0800 Subject: [PATCH 01/11] change referrers table from views to visitors --- src/components/metrics/ReferrersTable.tsx | 2 +- src/queries/sql/pageviews/getPageviewMetrics.ts | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/components/metrics/ReferrersTable.tsx b/src/components/metrics/ReferrersTable.tsx index 142f361b..db40a617 100644 --- a/src/components/metrics/ReferrersTable.tsx +++ b/src/components/metrics/ReferrersTable.tsx @@ -87,7 +87,7 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) { {...props} title={formatMessage(labels.referrers)} type="referrer" - metric={formatMessage(labels.views)} + metric={formatMessage(labels.visitors)} dataFilter={view === 'grouped' ? groupedFilter : undefined} renderLabel={renderLink} > diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index f7604298..344a5e94 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -62,7 +62,8 @@ async function relationalQuery( return rawQuery( ` - select ${column} x, count(*) y + select ${column} x, + ${column === 'referrer_domain' ? 'count(distinct session_id)' : 'count(*)'} as y from website_event ${joinSession} ${entryExitQuery} @@ -119,7 +120,8 @@ async function clickhouseQuery( } sql = ` - select ${column} x, count(*) y + select ${column} x, + ${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y from website_event ${entryExitQuery} where website_id = {websiteId:UUID} @@ -133,13 +135,13 @@ async function clickhouseQuery( `; } else { let groupByQuery = ''; + let columnQuery = `arrayJoin(${column})`; if (column === 'referrer_domain') { excludeDomain = `and t != hostname and hostname != ''`; + columnQuery = `session_id s, arrayJoin(${column})`; } - let columnQuery = `arrayJoin(${column})`; - if (type === 'entry') { columnQuery = `visit_id x, argMinMerge(entry_url)`; } @@ -154,7 +156,7 @@ async function clickhouseQuery( sql = ` select g.t as x, - count(*) as y + ${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y from ( select ${columnQuery} as t from website_event_stats_hourly website_event From d1b01370efec92d25fe16a445e160e583b051d3a Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 12 Feb 2025 10:25:38 -0800 Subject: [PATCH 02/11] fix getsessiondata props --- src/queries/sql/sessions/getSessionDataProperties.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/queries/sql/sessions/getSessionDataProperties.ts b/src/queries/sql/sessions/getSessionDataProperties.ts index bfa6a269..ed914518 100644 --- a/src/queries/sql/sessions/getSessionDataProperties.ts +++ b/src/queries/sql/sessions/getSessionDataProperties.ts @@ -25,13 +25,12 @@ async function relationalQuery( ` select data_key as "propertyName", - count(*) as "total" + count(distinct d.session_id) as "total" from website_event e - left join session_data d + join session_data d on d.session_id = e.session_id where e.website_id = {{websiteId:uuid}} and e.created_at between {{startDate}} and {{endDate}} - and d.data_key is not null ${filterQuery} group by 1 order by 2 desc @@ -54,9 +53,9 @@ async function clickhouseQuery( ` select data_key as propertyName, - count(*) as total + count(distinct d.session_id) as total from website_event e - left join session_data d + join session_data d final on d.session_id = e.session_id where e.website_id = {websiteId:UUID} and e.created_at between {startDate:DateTime64} and {endDate:DateTime64} From 1438231898b84f1edf7c922cbd7543581569ad84 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 12 Feb 2025 11:21:38 -0800 Subject: [PATCH 03/11] fix session values route --- .../[websiteId]/session-data/values/route.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/app/api/websites/[websiteId]/session-data/values/route.ts b/src/app/api/websites/[websiteId]/session-data/values/route.ts index 93e91775..d950da34 100644 --- a/src/app/api/websites/[websiteId]/session-data/values/route.ts +++ b/src/app/api/websites/[websiteId]/session-data/values/route.ts @@ -1,8 +1,8 @@ -import { z } from 'zod'; -import { parseRequest } from '@/lib/request'; -import { unauthorized, json } from '@/lib/response'; import { canViewWebsite } from '@/lib/auth'; -import { getEventDataEvents } from '@/queries/sql/events/getEventDataEvents'; +import { parseRequest } from '@/lib/request'; +import { json, unauthorized } from '@/lib/response'; +import { getSessionDataValues } from '@/queries'; +import { z } from 'zod'; export async function GET( request: Request, @@ -20,7 +20,7 @@ export async function GET( return error(); } - const { startAt, endAt, event } = query; + const { startAt, endAt, propertyName } = query; const { websiteId } = await params; if (!(await canViewWebsite(auth, websiteId))) { @@ -30,10 +30,10 @@ export async function GET( const startDate = new Date(+startAt); const endDate = new Date(+endAt); - const data = await getEventDataEvents(websiteId, { + const data = await getSessionDataValues(websiteId, { startDate, endDate, - event, + propertyName, }); return json(data); From 33662f9f5e7302ec7ad5a2e5a9f4f93f265d2f7c Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 12 Feb 2025 11:26:40 -0800 Subject: [PATCH 04/11] join on website event for date ranges --- .../sql/sessions/getSessionDataValues.ts | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/queries/sql/sessions/getSessionDataValues.ts b/src/queries/sql/sessions/getSessionDataValues.ts index 3281521a..8cd6a4ab 100644 --- a/src/queries/sql/sessions/getSessionDataValues.ts +++ b/src/queries/sql/sessions/getSessionDataValues.ts @@ -27,11 +27,13 @@ async function relationalQuery( when data_type = 4 then ${getDateSQL('date_value', 'hour')} else string_value end as "value", - count(*) as "total" - from session_data - where website_id = {{websiteId::uuid}} - and created_at between {{startDate}} and {{endDate}} - and data_key = {{propertyName}} + count(distinct d.session_id) as "total" + from website_event e + join session_data d + on d.session_id = e.session_id + where e.website_id = {{websiteId::uuid}} + and e.created_at between {{startDate}} and {{endDate}} + and d.data_key = {{propertyName}} ${filterQuery} group by value order by 2 desc @@ -54,11 +56,13 @@ async function clickhouseQuery( multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), data_type = 4, toString(date_trunc('hour', date_value)), string_value) as "value", - count(*) as "total" - from session_data final - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and data_key = {propertyName:String} + uniq(d.session_id) as "total" + from website_event e + join session_data d final + on d.session_id = e.session_id + where e.website_id = {websiteId:UUID} + and e.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and d.data_key = {propertyName:String} ${filterQuery} group by value order by 2 desc From ab08fcb00fda6603017bdca2ec269ff0138c04ec Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 12 Feb 2025 13:20:54 -0800 Subject: [PATCH 05/11] fix pageview metrics / channel metrics --- src/queries/sql/getChannelMetrics.ts | 2 +- .../sql/pageviews/getPageviewMetrics.ts | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index a2223870..9141290e 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -21,7 +21,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { referrer_query as query, count(distinct session_id) as visitors from website_event - where website_id = {websiteId:UUID} + where website_id = {{websiteId::uuid}} ${filterQuery} ${dateQuery} group by 1, 2 diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index 344a5e94..19a9b467 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -32,15 +32,17 @@ async function relationalQuery( websiteId, { ...filters, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, }, - { joinSession: SESSION_COLUMNS.includes(type) }, + { joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' }, ); let entryExitQuery = ''; let excludeDomain = ''; + if (column === 'referrer_domain') { - excludeDomain = `and website_event.referrer_domain != website_event.hostname - and website_event.referrer_domain is not null`; + excludeDomain = `and website_event.referrer_domain != session.hostname + and website_event.referrer_domain != ''`; } if (type === 'entry' || type === 'exit') { @@ -53,6 +55,7 @@ async function relationalQuery( from website_event where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} group by visit_id ) x on x.visit_id = website_event.visit_id @@ -63,7 +66,7 @@ async function relationalQuery( return rawQuery( ` select ${column} x, - ${column === 'referrer_domain' ? 'count(distinct session_id)' : 'count(*)'} as y + ${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y from website_event ${joinSession} ${entryExitQuery} @@ -102,7 +105,7 @@ async function clickhouseQuery( let entryExitQuery = ''; if (column === 'referrer_domain') { - excludeDomain = `and referrer_domain != hostname and hostname != ''`; + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; } if (type === 'entry' || type === 'exit') { @@ -114,6 +117,7 @@ async function clickhouseQuery( from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} group by visit_id) x ON x.visit_id = website_event.visit_id and x.target_created_at = website_event.created_at`; @@ -126,6 +130,7 @@ async function clickhouseQuery( ${entryExitQuery} where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} group by x @@ -138,7 +143,7 @@ async function clickhouseQuery( let columnQuery = `arrayJoin(${column})`; if (column === 'referrer_domain') { - excludeDomain = `and t != hostname and hostname != ''`; + excludeDomain = `and t != hostname and t != ''`; columnQuery = `session_id s, arrayJoin(${column})`; } @@ -162,6 +167,7 @@ async function clickhouseQuery( from website_event_stats_hourly website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} ${excludeDomain} ${filterQuery} ${groupByQuery}) as g From 53bcd0515e9af0972be5bfddb694aeb658cd46c6 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 13 Feb 2025 10:10:33 -0800 Subject: [PATCH 06/11] Fix type error for getChannels --- .../api/websites/[websiteId]/metrics/route.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 70ed9f90..c0958739 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -131,35 +131,35 @@ function getChannels(data: { domain: string; query: string; visitors: number }[] for (const { domain, query, visitors } of data) { if (!domain && !query) { - channels.direct += visitors; + channels.direct += Number(visitors); } const prefix = /utm_medium=(.*cp.*|ppc|retargeting|paid.*)/.test(query) ? 'paid' : 'organic'; if (SEARCH_DOMAINS.some(match(domain)) || /utm_medium=organic/.test(query)) { - channels[`${prefix}Search`] += visitors; + channels[`${prefix}Search`] += Number(visitors); } else if ( SOCIAL_DOMAINS.some(match(domain)) || /utm_medium=(social|social-network|social-media|sm|social network|social media)/.test(query) ) { - channels[`${prefix}Social`] += visitors; + channels[`${prefix}Social`] += Number(visitors); } else if (EMAIL_DOMAINS.some(match(domain)) || /utm_medium=(.*e[-_ ]?mail.*)/.test(query)) { - channels.email += visitors; + channels.email += Number(visitors); } else if ( SHOPPING_DOMAINS.some(match(domain)) || /utm_campaign=(.*(([^a-df-z]|^)shop|shopping).*)/.test(query) ) { - channels[`${prefix}Shopping`] += visitors; + channels[`${prefix}Shopping`] += Number(visitors); } else if (VIDEO_DOMAINS.some(match(domain)) || /utm_medium=(.*video.*)/.test(query)) { - channels[`${prefix}Video`] += visitors; + channels[`${prefix}Video`] += Number(visitors); } else if (PAID_AD_PARAMS.some(match(query))) { - channels.paidAds += visitors; + channels.paidAds += Number(visitors); } else if (/utm_medium=(referral|app|link)/.test(query)) { - channels.referral += visitors; + channels.referral += Number(visitors); } else if (/utm_medium=affiliate/.test(query)) { - channels.affiliate += visitors; + channels.affiliate += Number(visitors); } else if (/utm_(source|medium)=sms/.test(query)) { - channels.sms += visitors; + channels.sms += Number(visitors); } } From 97f21d6f23cce7692e881af0ed0158a4a1808f3f Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 13 Feb 2025 11:20:50 -0800 Subject: [PATCH 07/11] fix syntax error on session properties query --- src/queries/sql/sessions/getSessionDataProperties.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queries/sql/sessions/getSessionDataProperties.ts b/src/queries/sql/sessions/getSessionDataProperties.ts index ed914518..20fb11d5 100644 --- a/src/queries/sql/sessions/getSessionDataProperties.ts +++ b/src/queries/sql/sessions/getSessionDataProperties.ts @@ -29,7 +29,7 @@ async function relationalQuery( from website_event e join session_data d on d.session_id = e.session_id - where e.website_id = {{websiteId:uuid}} + where e.website_id = {{websiteId::uuid}} and e.created_at between {{startDate}} and {{endDate}} ${filterQuery} group by 1 From 089b4fee69a451392b75c887cd094d8f982881fd Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 13 Feb 2025 11:29:22 -0800 Subject: [PATCH 08/11] fix zod validation for revenue report --- src/app/api/reports/revenue/route.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/api/reports/revenue/route.ts b/src/app/api/reports/revenue/route.ts index f8f4041f..13a34f38 100644 --- a/src/app/api/reports/revenue/route.ts +++ b/src/app/api/reports/revenue/route.ts @@ -29,6 +29,7 @@ export async function GET(request: Request) { export async function POST(request: Request) { const schema = z.object({ + currency: z.string(), ...reportParms, timezone: timezoneParam, }); From 575c50cdfeb8f5a01f4c49cfdc115aa5898db4fa Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 13 Feb 2025 13:21:15 -0800 Subject: [PATCH 09/11] fix websiteselect search --- src/components/input/WebsiteSelect.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/components/input/WebsiteSelect.tsx b/src/components/input/WebsiteSelect.tsx index ed78efc8..0a112e89 100644 --- a/src/components/input/WebsiteSelect.tsx +++ b/src/components/input/WebsiteSelect.tsx @@ -14,12 +14,12 @@ export function WebsiteSelect({ onSelect?: (key: any) => void; }) { const { formatMessage, labels, messages } = useMessages(); - const [query, setQuery] = useState(''); + const [search, setSearch] = useState(''); const [selectedId, setSelectedId] = useState(websiteId); const { data: website } = useWebsite(selectedId as string); - const queryResult = useWebsites({ teamId }, { query, pageSize: 5 }); + const queryResult = useWebsites({ teamId }, { search }); const renderValue = () => { return website?.name; @@ -35,7 +35,7 @@ export function WebsiteSelect({ }; const handleSearch = (value: string) => { - setQuery(value); + setSearch(value); }; return ( From 3e5866832ffffeb4408b17fe9e31eee922ed500f Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 13 Feb 2025 13:24:06 -0800 Subject: [PATCH 10/11] fix search bug --- src/components/input/WebsiteSelect.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/input/WebsiteSelect.tsx b/src/components/input/WebsiteSelect.tsx index 0a112e89..8a7e4ac0 100644 --- a/src/components/input/WebsiteSelect.tsx +++ b/src/components/input/WebsiteSelect.tsx @@ -19,7 +19,7 @@ export function WebsiteSelect({ const { data: website } = useWebsite(selectedId as string); - const queryResult = useWebsites({ teamId }, { search }); + const queryResult = useWebsites({ teamId }, { search, pageSize: 5 }); const renderValue = () => { return website?.name; From 35dc6ef83bc69095cdc14a3ea98520e22b65cda9 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 13 Feb 2025 19:29:41 -0800 Subject: [PATCH 11/11] Fixed error with contains filters. --- .../[reportId]/FieldFilterEditForm.tsx | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx index d171c780..c1f95e80 100644 --- a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx +++ b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx @@ -1,13 +1,7 @@ -import { - useFilters, - useFormat, - useLocale, - useMessages, - useWebsiteValues, -} from '@/components/hooks'; +import { useMemo, useState } from 'react'; +import { useFilters, useFormat, useMessages, useWebsiteValues } from '@/components/hooks'; import { OPERATORS } from '@/lib/constants'; import { isEqualsOperator } from '@/lib/params'; -import { useMemo, useState } from 'react'; import { Button, Dropdown, @@ -61,7 +55,6 @@ export default function FieldFilterEditForm({ const [selected, setSelected] = useState(isEquals ? value : ''); const { filters } = useFilters(); const { formatValue } = useFormat(); - const { locale } = useLocale(); const isDisabled = !operator || (isEquals && !selected) || (!isEquals && !value); const { data: values = [], @@ -86,29 +79,17 @@ export default function FieldFilterEditForm({ }; const formattedValues = useMemo(() => { - if (!values) { - return {}; - } - const formatted = {}; - const format = (val: string) => { - formatted[val] = formatValue(val, name); - return formatted[val]; - }; + return values.reduce((obj: { [x: string]: string }, { value }: { value: string }) => { + obj[value] = formatValue(value, name); - if (values?.length !== 1) { - const { compare } = new Intl.Collator(locale, { numeric: true }); - values.sort((a, b) => compare(formatted[a] ?? format(a), formatted[b] ?? format(b))); - } else { - format(values[0]); - } - - return formatted; - }, [formatValue, locale, name, values]); + return obj; + }, {}); + }, [formatValue, name, values]); const filteredValues = useMemo(() => { return value ? values.filter((n: string | number) => - formattedValues[n].toLowerCase().includes(value.toLowerCase()), + formattedValues[n]?.toLowerCase()?.includes(value.toLowerCase()), ) : values; }, [value, formattedValues]);