From 1d78cc4950a0ae80229aad1b1d0ae807146ebeb1 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 14 Aug 2023 23:07:39 -0700 Subject: [PATCH 1/7] fix UTC timezone for CH query --- queries/analytics/events/getEventMetrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/analytics/events/getEventMetrics.ts b/queries/analytics/events/getEventMetrics.ts index 09a859461..778cfee1b 100644 --- a/queries/analytics/events/getEventMetrics.ts +++ b/queries/analytics/events/getEventMetrics.ts @@ -41,7 +41,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { } async function clickhouseQuery(websiteId: string, filters: QueryFilters) { - const { timezone = 'utc', unit = 'day' } = filters; + const { timezone = 'UTC', unit = 'day' } = filters; const { rawQuery, getDateQuery, parseFilters } = clickhouse; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, From 203f4e8e03df86c4a605b247bfd6fb242bd5aade Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 15 Aug 2023 10:57:25 -0700 Subject: [PATCH 2/7] Add search Mode for mysql. --- lib/prisma.ts | 14 ++++++++++++++ queries/admin/report.ts | 14 ++++++++------ queries/admin/team.ts | 6 ++++-- queries/admin/user.ts | 4 +++- queries/admin/website.ts | 5 +++-- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index efce3f4eb..76c5a09b1 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -5,6 +5,7 @@ import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS } from './constants'; import { loadWebsite } from './load'; import { maxDate } from './date'; import { QueryFilters, QueryOptions, SearchFilter } from './types'; +import { Prisma } from '@prisma/client'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -177,6 +178,18 @@ function getPageFilters(filters: SearchFilter): [ ]; } +function getSearchMode(): { mode?: Prisma.QueryMode } { + const db = getDatabaseType(); + + if (db === POSTGRESQL) { + return { + mode: 'insensitive', + }; + } + + return {}; +} + export default { ...prisma, getAddMinutesQuery, @@ -185,5 +198,6 @@ export default { getFilterQuery, parseFilters, getPageFilters, + getSearchMode, rawQuery, }; diff --git a/queries/admin/report.ts b/queries/admin/report.ts index 3c3e3e2db..22f3c62bc 100644 --- a/queries/admin/report.ts +++ b/queries/admin/report.ts @@ -38,6 +38,8 @@ export async function getReports( filterType = REPORT_FILTER_TYPES.all, } = ReportSearchFilter; + const mode = prisma.getSearchMode(); + const where: Prisma.ReportWhereInput = { ...(userId && { userId: userId }), ...(websiteId && { websiteId: websiteId }), @@ -73,7 +75,7 @@ export async function getReports( filterType === REPORT_FILTER_TYPES.name) && { name: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }), }, @@ -82,7 +84,7 @@ export async function getReports( filterType === REPORT_FILTER_TYPES.description) && { description: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }), }, @@ -91,7 +93,7 @@ export async function getReports( filterType === REPORT_FILTER_TYPES.type) && { type: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }), }, @@ -101,7 +103,7 @@ export async function getReports( user: { username: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }, }), @@ -112,7 +114,7 @@ export async function getReports( website: { name: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }, }), @@ -123,7 +125,7 @@ export async function getReports( website: { domain: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }, }), diff --git a/queries/admin/team.ts b/queries/admin/team.ts index 71ea634a1..79735fc77 100644 --- a/queries/admin/team.ts +++ b/queries/admin/team.ts @@ -86,6 +86,8 @@ export async function getTeams( options?: { include?: Prisma.TeamInclude }, ): Promise> { const { userId, filter, filterType = TEAM_FILTER_TYPES.all } = TeamSearchFilter; + const mode = prisma.getSearchMode(); + const where: Prisma.TeamWhereInput = { ...(userId && { teamUser: { @@ -97,7 +99,7 @@ export async function getTeams( OR: [ { ...((filterType === TEAM_FILTER_TYPES.all || filterType === TEAM_FILTER_TYPES.name) && { - name: { startsWith: filter, mode: 'insensitive' }, + name: { startsWith: filter, ...mode }, }), }, { @@ -109,7 +111,7 @@ export async function getTeams( user: { username: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }, }, diff --git a/queries/admin/user.ts b/queries/admin/user.ts index 3aece6d12..dfb923f31 100644 --- a/queries/admin/user.ts +++ b/queries/admin/user.ts @@ -41,6 +41,8 @@ export async function getUsers( options?: { include?: Prisma.UserInclude }, ): Promise> { const { teamId, filter, filterType = USER_FILTER_TYPES.all } = UserSearchFilter; + const mode = prisma.getSearchMode(); + const where: Prisma.UserWhereInput = { ...(teamId && { teamUser: { @@ -57,7 +59,7 @@ export async function getUsers( filterType === USER_FILTER_TYPES.username) && { username: { startsWith: filter, - mode: 'insensitive', + ...mode, }, }), }, diff --git a/queries/admin/website.ts b/queries/admin/website.ts index 8c3535cc8..3d0c773b6 100644 --- a/queries/admin/website.ts +++ b/queries/admin/website.ts @@ -30,6 +30,7 @@ export async function getWebsites( filter, filterType = WEBSITE_FILTER_TYPES.all, } = WebsiteSearchFilter; + const mode = prisma.getSearchMode(); const where: Prisma.WebsiteWhereInput = { ...(teamId && { @@ -79,13 +80,13 @@ export async function getWebsites( { ...((filterType === WEBSITE_FILTER_TYPES.all || filterType === WEBSITE_FILTER_TYPES.name) && { - name: { startsWith: filter, mode: 'insensitive' }, + name: { startsWith: filter, ...mode }, }), }, { ...((filterType === WEBSITE_FILTER_TYPES.all || filterType === WEBSITE_FILTER_TYPES.domain) && { - domain: { startsWith: filter, mode: 'insensitive' }, + domain: { startsWith: filter, ...mode }, }), }, ], From 979ea33d7457291de341ca902d40208477cffad3 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 15 Aug 2023 10:58:55 -0700 Subject: [PATCH 3/7] add daydiff function query --- lib/prisma.ts | 13 +++++++++++++ queries/analytics/reports/getRetention.ts | 7 +++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index efce3f4eb..94fcf0a08 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -34,6 +34,18 @@ function getAddMinutesQuery(field: string, minutes: number): string { } } +function getDayDiffQuery(field1: string, field2: string): string { + const db = getDatabaseType(process.env.DATABASE_URL); + + if (db === POSTGRESQL) { + return `${field1}::date - ${field2}::date`; + } + + if (db === MYSQL) { + return `DATEDIFF(${field1}, ${field2});`; + } +} + function getDateQuery(field: string, unit: string, timezone?: string): string { const db = getDatabaseType(); @@ -180,6 +192,7 @@ function getPageFilters(filters: SearchFilter): [ export default { ...prisma, getAddMinutesQuery, + getDayDiffQuery, getDateQuery, getTimestampIntervalQuery, getFilterQuery, diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.ts index ee7e4619b..12ef1d529 100644 --- a/queries/analytics/reports/getRetention.ts +++ b/queries/analytics/reports/getRetention.ts @@ -33,7 +33,7 @@ async function relationalQuery( }[] > { const { startDate, endDate } = dateRange; - const { getDateQuery, rawQuery } = prisma; + const { getDateQuery, getDayDiffQuery, rawQuery } = prisma; const timezone = 'utc'; const unit = 'day'; @@ -49,7 +49,10 @@ async function relationalQuery( user_activities AS ( select distinct w.session_id, - (${getDateQuery('created_at', unit, timezone)}::date - c.cohort_date::date) as day_number + (${getDayDiffQuery( + getDateQuery('created_at', unit, timezone), + 'c.cohort_date', + )}) as day_number from website_event w join cohort_items c on w.session_id = c.session_id From 495b7f1f20115c4261b2c3d50809c664c2a98861 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 15 Aug 2023 12:15:27 -0700 Subject: [PATCH 4/7] fix mysql retention query --- lib/prisma.ts | 15 ++++++++++++++- queries/analytics/reports/getRetention.ts | 12 +++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index f392252e9..5a21a6ec6 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -43,7 +43,19 @@ function getDayDiffQuery(field1: string, field2: string): string { } if (db === MYSQL) { - return `DATEDIFF(${field1}, ${field2});`; + return `DATEDIFF(${field1}, ${field2})`; + } +} + +function getCastColumnQuery(field: string, type: string): string { + const db = getDatabaseType(process.env.DATABASE_URL); + + if (db === POSTGRESQL) { + return `${field}::${type}`; + } + + if (db === MYSQL) { + return `${field}`; } } @@ -206,6 +218,7 @@ export default { ...prisma, getAddMinutesQuery, getDayDiffQuery, + getCastColumnQuery, getDateQuery, getTimestampIntervalQuery, getFilterQuery, diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.ts index 12ef1d529..3abc0e7ae 100644 --- a/queries/analytics/reports/getRetention.ts +++ b/queries/analytics/reports/getRetention.ts @@ -33,7 +33,7 @@ async function relationalQuery( }[] > { const { startDate, endDate } = dateRange; - const { getDateQuery, getDayDiffQuery, rawQuery } = prisma; + const { getDateQuery, getDayDiffQuery, getCastColumnQuery, rawQuery } = prisma; const timezone = 'utc'; const unit = 'day'; @@ -49,10 +49,10 @@ async function relationalQuery( user_activities AS ( select distinct w.session_id, - (${getDayDiffQuery( + ${getDayDiffQuery( getDateQuery('created_at', unit, timezone), 'c.cohort_date', - )}) as day_number + )} as day_number from website_event w join cohort_items c on w.session_id = c.session_id @@ -81,7 +81,7 @@ async function relationalQuery( c.day_number as day, s.visitors, c.visitors as "returnVisitors", - c.visitors::float * 100 / s.visitors as percentage + ${getCastColumnQuery('c.visitors', 'float')} * 100 / s.visitors as percentage from cohort_date c join cohort_size s on c.cohort_date = s.cohort_date @@ -92,7 +92,9 @@ async function relationalQuery( startDate, endDate, }, - ); + ).then(results => { + return results.map(i => ({ ...i, percentage: Number(i.percentage) || 0 })); + }); } async function clickhouseQuery( From c3d0a1a62843ff039130b8903e8aa9b99945e1ea Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 15 Aug 2023 13:01:50 -0700 Subject: [PATCH 5/7] Fix noResultsFound. --- .../pages/settings/websites/WebsitesList.js | 24 ++--- .../pages/settings/websites/WebsitesTable.js | 98 ++++++++++--------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/components/pages/settings/websites/WebsitesList.js b/components/pages/settings/websites/WebsitesList.js index d7748f272..799b032bf 100644 --- a/components/pages/settings/websites/WebsitesList.js +++ b/components/pages/settings/websites/WebsitesList.js @@ -29,7 +29,6 @@ export function WebsitesList({ showTeam, showHeader = true, includeTeams, onlyTe { enabled: !!user }, ); const { showToast } = useToasts(); - const hasData = data && data.length !== 0; const handleSave = async () => { await refetch(); @@ -57,21 +56,14 @@ export function WebsitesList({ showTeam, showHeader = true, includeTeams, onlyTe return ( {showHeader && {addButton}} - {hasData && ( - - )} - {!hasData && ( - - {addButton} - - )} + ); } diff --git a/components/pages/settings/websites/WebsitesTable.js b/components/pages/settings/websites/WebsitesTable.js index dec1b3233..803ce1dc8 100644 --- a/components/pages/settings/websites/WebsitesTable.js +++ b/components/pages/settings/websites/WebsitesTable.js @@ -1,3 +1,4 @@ +import EmptyPlaceholder from 'components/common/EmptyPlaceholder'; import Link from 'next/link'; import { Button, Text, Icon, Icons } from 'react-basics'; import SettingsTable from 'components/common/SettingsTable'; @@ -13,10 +14,12 @@ export function WebsitesTable({ onPageSizeChange, showTeam, }) { - const { formatMessage, labels } = useMessages(); + const { formatMessage, labels, messages } = useMessages(); const { openExternal } = useConfig(); const { user } = useUser(); + const showTable = data && (filterValue || data?.data.length !== 0); + const teamColumns = [ { name: 'teamName', label: formatMessage(labels.teamName) }, { name: 'owner', label: formatMessage(labels.owner) }, @@ -30,51 +33,56 @@ export function WebsitesTable({ ]; return ( - - {row => { - const { - id, - teamWebsite, - user: { username, id: ownerId }, - } = row; - if (showTeam) { - row.teamName = teamWebsite[0]?.team.name; - row.owner = username; - } + <> + {showTable && ( + + {row => { + const { + id, + teamWebsite, + user: { username, id: ownerId }, + } = row; + if (showTeam) { + row.teamName = teamWebsite[0]?.team.name; + row.owner = username; + } - return ( - <> - {(!showTeam || ownerId === user.id) && ( - - - - )} - - - - - ); - }} - + return ( + <> + {(!showTeam || ownerId === user.id) && ( + + + + )} + + + + + ); + }} + + )} + {!showTable && } + ); } From 38445fce7a22edbce476985d4beb3ab163bc7a81 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 15 Aug 2023 13:08:18 -0700 Subject: [PATCH 6/7] Fix test console. --- components/pages/console/TestConsole.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.js index 4f167b9a7..060314fd2 100644 --- a/components/pages/console/TestConsole.js +++ b/components/pages/console/TestConsole.js @@ -72,7 +72,7 @@ export function TestConsole() { } const [websiteId] = id || []; - const website = data.find(({ id }) => websiteId === id); + const website = data?.data.find(({ id }) => websiteId === id); return ( From c3e261fc50a78f0d50119aff9d583c8cea766516 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Wed, 16 Aug 2023 08:49:22 -0700 Subject: [PATCH 7/7] Resolve issues in event data --- components/pages/event-data/EventDataValueTable.js | 1 + components/pages/websites/WebsiteEventData.js | 6 +++--- lib/types.ts | 9 ++------- pages/api/event-data/events.ts | 5 +++-- pages/api/event-data/fields.ts | 4 ++-- pages/api/event-data/stats.ts | 12 ++++++------ queries/analytics/eventData/getEventDataEvents.ts | 8 ++++---- queries/analytics/eventData/getEventDataFields.ts | 4 ++-- 8 files changed, 23 insertions(+), 26 deletions(-) diff --git a/components/pages/event-data/EventDataValueTable.js b/components/pages/event-data/EventDataValueTable.js index 3688ad094..69ed10a7a 100644 --- a/components/pages/event-data/EventDataValueTable.js +++ b/components/pages/event-data/EventDataValueTable.js @@ -36,6 +36,7 @@ export function EventDataValueTable({ data = [], event }) { {row => DATA_TYPES[row.dataType]} + {({ total }) => total.toLocaleString()} diff --git a/components/pages/websites/WebsiteEventData.js b/components/pages/websites/WebsiteEventData.js index 5e2083557..7f9a68294 100644 --- a/components/pages/websites/WebsiteEventData.js +++ b/components/pages/websites/WebsiteEventData.js @@ -5,18 +5,18 @@ import { EventDataMetricsBar } from 'components/pages/event-data/EventDataMetric import { useDateRange, useApi, usePageQuery } from 'hooks'; import styles from './WebsiteEventData.module.css'; -function useData(websiteId, eventName) { +function useData(websiteId, event) { const [dateRange] = useDateRange(websiteId); const { startDate, endDate } = dateRange; const { get, useQuery } = useApi(); const { data, error, isLoading } = useQuery( - ['event-data:events', { websiteId, startDate, endDate, eventName }], + ['event-data:events', { websiteId, startDate, endDate, event }], () => get('/event-data/events', { websiteId, startAt: +startDate, endAt: +endDate, - eventName, + event, }), { enabled: !!(websiteId && startDate && endDate) }, ); diff --git a/lib/types.ts b/lib/types.ts index 65bef8fb9..3f3ac5337 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -126,13 +126,8 @@ export interface WebsiteEventMetric { y: number; } -export interface WebsiteEventDataStats { - fieldName: string; - dataType: number; - total: number; -} - -export interface WebsiteEventDataFields { +export interface WebsiteEventData { + eventName?: string; fieldName: string; dataType: number; fieldValue?: string; diff --git a/pages/api/event-data/events.ts b/pages/api/event-data/events.ts index e83e541bf..9f8f964b3 100644 --- a/pages/api/event-data/events.ts +++ b/pages/api/event-data/events.ts @@ -5,16 +5,17 @@ import { NextApiResponse } from 'next'; import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { getEventDataEvents } from 'queries'; -export interface EventDataFieldsRequestBody { +export interface EventDataEventsRequestQuery { websiteId: string; dateRange: { startDate: string; endDate: string; }; + event?: string; } export default async ( - req: NextApiRequestQueryBody, + req: NextApiRequestQueryBody, res: NextApiResponse, ) => { await useCors(req, res); diff --git a/pages/api/event-data/fields.ts b/pages/api/event-data/fields.ts index f21bd570f..b6a731336 100644 --- a/pages/api/event-data/fields.ts +++ b/pages/api/event-data/fields.ts @@ -5,7 +5,7 @@ import { NextApiResponse } from 'next'; import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { getEventDataFields } from 'queries'; -export interface EventDataFieldsRequestBody { +export interface EventDataFieldsRequestQuery { websiteId: string; dateRange: { startDate: string; @@ -15,7 +15,7 @@ export interface EventDataFieldsRequestBody { } export default async ( - req: NextApiRequestQueryBody, + req: NextApiRequestQueryBody, res: NextApiResponse, ) => { await useCors(req, res); diff --git a/pages/api/event-data/stats.ts b/pages/api/event-data/stats.ts index 74f420c49..d1ee396bc 100644 --- a/pages/api/event-data/stats.ts +++ b/pages/api/event-data/stats.ts @@ -5,7 +5,7 @@ import { NextApiResponse } from 'next'; import { ok, methodNotAllowed, unauthorized } from 'next-basics'; import { getEventDataFields } from 'queries'; -export interface EventDataRequestBody { +export interface EventDataStatsRequestQuery { websiteId: string; dateRange: { startDate: string; @@ -15,7 +15,7 @@ export interface EventDataRequestBody { } export default async ( - req: NextApiRequestQueryBody, + req: NextApiRequestQueryBody, res: NextApiResponse, ) => { await useCors(req, res); @@ -32,18 +32,18 @@ export default async ( const endDate = new Date(+endAt); const results = await getEventDataFields(websiteId, { startDate, endDate }); - const events = new Set(); + const fields = new Set(); const data = results.reduce( (obj, row) => { - events.add(row.fieldName); + fields.add(row.fieldName); obj.records += Number(row.total); return obj; }, - { fields: results.length, records: 0 }, + { events: results.length, records: 0 }, ); - return ok(res, { ...data, events: events.size }); + return ok(res, { ...data, fields: fields.size }); } return methodNotAllowed(res); diff --git a/queries/analytics/eventData/getEventDataEvents.ts b/queries/analytics/eventData/getEventDataEvents.ts index 250841113..2c8cb0e0d 100644 --- a/queries/analytics/eventData/getEventDataEvents.ts +++ b/queries/analytics/eventData/getEventDataEvents.ts @@ -1,11 +1,11 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; +import { QueryFilters, WebsiteEventData } from 'lib/types'; export async function getEventDataEvents( ...args: [websiteId: string, filters: QueryFilters] -): Promise { +): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args), @@ -24,7 +24,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { website_event.event_name as "eventName", event_data.event_key as "fieldName", event_data.data_type as "dataType", - event_data.string_value as "value", + event_data.string_value as "fieldValue", count(*) as "total" from event_data inner join website_event @@ -71,7 +71,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { event_name as eventName, event_key as fieldName, data_type as dataType, - string_value as value, + string_value as fieldValue, count(*) as total from event_data where website_id = {websiteId:UUID} diff --git a/queries/analytics/eventData/getEventDataFields.ts b/queries/analytics/eventData/getEventDataFields.ts index f5f426e0c..ac32b188f 100644 --- a/queries/analytics/eventData/getEventDataFields.ts +++ b/queries/analytics/eventData/getEventDataFields.ts @@ -1,11 +1,11 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; -import { QueryFilters, WebsiteEventDataFields } from 'lib/types'; +import { QueryFilters, WebsiteEventData } from 'lib/types'; export async function getEventDataFields( ...args: [websiteId: string, filters: QueryFilters & { field?: string }] -): Promise { +): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), [CLICKHOUSE]: () => clickhouseQuery(...args),