From 9cde107ddfa857a65431d54be44d1c79f311d24d Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Fri, 4 Aug 2023 13:10:03 -0700 Subject: [PATCH 1/5] build out retention reports --- components/messages.js | 1 + components/pages/reports/ReportDetails.js | 2 + components/pages/reports/ReportTemplates.js | 6 + .../pages/reports/funnel/FunnelChart.js | 17 +- .../pages/reports/retention/RetentionChart.js | 74 +++++++ .../retention/RetentionChart.module.css | 3 + .../reports/retention/RetentionParameters.js | 44 ++++ .../reports/retention/RetentionReport.js | 28 +++ .../retention/RetentionReport.module.css | 10 + .../pages/reports/retention/RetentionTable.js | 19 ++ pages/api/reports/retention.ts | 55 +++++ pages/reports/retention.js | 13 ++ queries/analytics/reports/getRetention.ts | 209 ++++++++++++++++++ queries/index.js | 1 + 14 files changed, 479 insertions(+), 3 deletions(-) create mode 100644 components/pages/reports/retention/RetentionChart.js create mode 100644 components/pages/reports/retention/RetentionChart.module.css create mode 100644 components/pages/reports/retention/RetentionParameters.js create mode 100644 components/pages/reports/retention/RetentionReport.js create mode 100644 components/pages/reports/retention/RetentionReport.module.css create mode 100644 components/pages/reports/retention/RetentionTable.js create mode 100644 pages/api/reports/retention.ts create mode 100644 pages/reports/retention.js create mode 100644 queries/analytics/reports/getRetention.ts diff --git a/components/messages.js b/components/messages.js index a31e28751..68e3b3d59 100644 --- a/components/messages.js +++ b/components/messages.js @@ -161,6 +161,7 @@ export const labels = defineMessages({ overview: { id: 'labels.overview', defaultMessage: 'Overview' }, totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' }, insights: { id: 'label.insights', defaultMessage: 'Insights' }, + retention: { id: 'label.retention', defaultMessage: 'Retention' }, dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' }, }); diff --git a/components/pages/reports/ReportDetails.js b/components/pages/reports/ReportDetails.js index c41d12f6a..39cd285d7 100644 --- a/components/pages/reports/ReportDetails.js +++ b/components/pages/reports/ReportDetails.js @@ -1,9 +1,11 @@ import FunnelReport from './funnel/FunnelReport'; import EventDataReport from './event-data/EventDataReport'; +import RetentionReport from './retention/RetentionReport'; const reports = { funnel: FunnelReport, 'event-data': EventDataReport, + retention: RetentionReport, }; export default function ReportDetails({ reportId, reportType }) { diff --git a/components/pages/reports/ReportTemplates.js b/components/pages/reports/ReportTemplates.js index 60ae11e79..29c193a86 100644 --- a/components/pages/reports/ReportTemplates.js +++ b/components/pages/reports/ReportTemplates.js @@ -47,6 +47,12 @@ export function ReportTemplates() { url: '/reports/funnel', icon: , }, + { + title: formatMessage(labels.retention), + description: 'Track your websites user retention', + url: '/reports/retention', + icon: , + }, ]; return ( diff --git a/components/pages/reports/funnel/FunnelChart.js b/components/pages/reports/funnel/FunnelChart.js index 7253c3fa5..c35afe4e6 100644 --- a/components/pages/reports/funnel/FunnelChart.js +++ b/components/pages/reports/funnel/FunnelChart.js @@ -1,5 +1,5 @@ import { useCallback, useContext, useMemo } from 'react'; -import { Loading } from 'react-basics'; +import { Loading, StatusLight } from 'react-basics'; import useMessages from 'hooks/useMessages'; import useTheme from 'hooks/useTheme'; import BarChart from 'components/metrics/BarChart'; @@ -22,14 +22,25 @@ export function FunnelChart({ className, loading }) { ); const renderTooltipPopup = useCallback((setTooltipPopup, model) => { - const { opacity, dataPoints } = model.tooltip; + const { opacity, labelColors, dataPoints } = model.tooltip; if (!dataPoints?.length || !opacity) { setTooltipPopup(null); return; } - setTooltipPopup(`${formatLongNumber(dataPoints[0].raw.y)} ${formatMessage(labels.visitors)}`); + setTooltipPopup( + <> +
+ {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)} +
+
+ + {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)} + +
+ , + ); }, []); const datasets = useMemo(() => { diff --git a/components/pages/reports/retention/RetentionChart.js b/components/pages/reports/retention/RetentionChart.js new file mode 100644 index 000000000..5f7361fdb --- /dev/null +++ b/components/pages/reports/retention/RetentionChart.js @@ -0,0 +1,74 @@ +import { useCallback, useContext, useMemo } from 'react'; +import { Loading, StatusLight } from 'react-basics'; +import useMessages from 'hooks/useMessages'; +import useTheme from 'hooks/useTheme'; +import BarChart from 'components/metrics/BarChart'; +import { formatLongNumber } from 'lib/format'; +import styles from './RetentionChart.module.css'; +import { ReportContext } from '../Report'; + +export function RetentionChart({ className, loading }) { + const { report } = useContext(ReportContext); + const { formatMessage, labels } = useMessages(); + const { colors } = useTheme(); + + const { parameters, data } = report || {}; + + const renderXLabel = useCallback( + (label, index) => { + return parameters.urls[index]; + }, + [parameters], + ); + + const renderTooltipPopup = useCallback((setTooltipPopup, model) => { + const { opacity, labelColors, dataPoints } = model.tooltip; + + if (!dataPoints?.length || !opacity) { + setTooltipPopup(null); + return; + } + + setTooltipPopup( + <> +
+ {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)} +
+
+ + {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)} + +
+ , + ); + }, []); + + const datasets = useMemo(() => { + return [ + { + label: formatMessage(labels.uniqueVisitors), + data: data, + borderWidth: 1, + ...colors.chart.visitors, + }, + ]; + }, [data]); + + if (loading) { + return ; + } + + return ( + + ); +} + +export default RetentionChart; diff --git a/components/pages/reports/retention/RetentionChart.module.css b/components/pages/reports/retention/RetentionChart.module.css new file mode 100644 index 000000000..9e1690b31 --- /dev/null +++ b/components/pages/reports/retention/RetentionChart.module.css @@ -0,0 +1,3 @@ +.loading { + height: 300px; +} diff --git a/components/pages/reports/retention/RetentionParameters.js b/components/pages/reports/retention/RetentionParameters.js new file mode 100644 index 000000000..29c0eff2f --- /dev/null +++ b/components/pages/reports/retention/RetentionParameters.js @@ -0,0 +1,44 @@ +import { useContext, useRef } from 'react'; +import { useMessages } from 'hooks'; +import { Form, FormButtons, FormInput, FormRow, SubmitButton, TextField } from 'react-basics'; +import { ReportContext } from 'components/pages/reports/Report'; +import BaseParameters from '../BaseParameters'; + +export function RetentionParameters() { + const { report, runReport, isRunning } = useContext(ReportContext); + const { formatMessage, labels } = useMessages(); + const ref = useRef(null); + + const { parameters } = report || {}; + const { websiteId, dateRange } = parameters || {}; + const queryDisabled = !websiteId || !dateRange; + + const handleSubmit = (data, e) => { + e.stopPropagation(); + e.preventDefault(); + if (!queryDisabled) { + runReport(data); + } + }; + + return ( +
+ + + + + + + + + {formatMessage(labels.runQuery)} + + + + ); +} + +export default RetentionParameters; diff --git a/components/pages/reports/retention/RetentionReport.js b/components/pages/reports/retention/RetentionReport.js new file mode 100644 index 000000000..31d085f7b --- /dev/null +++ b/components/pages/reports/retention/RetentionReport.js @@ -0,0 +1,28 @@ +import RetentionChart from './RetentionChart'; +import RetentionTable from './RetentionTable'; +import RetentionParameters from './RetentionParameters'; +import Report from '../Report'; +import ReportHeader from '../ReportHeader'; +import ReportMenu from '../ReportMenu'; +import ReportBody from '../ReportBody'; +import Funnel from 'assets/funnel.svg'; + +const defaultParameters = { + type: 'Retention', + parameters: { window: 60, urls: [] }, +}; + +export default function RetentionReport({ reportId }) { + return ( + + } /> + + + + + + + + + ); +} diff --git a/components/pages/reports/retention/RetentionReport.module.css b/components/pages/reports/retention/RetentionReport.module.css new file mode 100644 index 000000000..aed66b74e --- /dev/null +++ b/components/pages/reports/retention/RetentionReport.module.css @@ -0,0 +1,10 @@ +.filters { + display: flex; + flex-direction: column; + justify-content: space-between; + border: 1px solid var(--base400); + border-radius: var(--border-radius); + line-height: 32px; + padding: 10px; + overflow: hidden; +} diff --git a/components/pages/reports/retention/RetentionTable.js b/components/pages/reports/retention/RetentionTable.js new file mode 100644 index 000000000..4ef879862 --- /dev/null +++ b/components/pages/reports/retention/RetentionTable.js @@ -0,0 +1,19 @@ +import { useContext } from 'react'; +import DataTable from 'components/metrics/DataTable'; +import { useMessages } from 'hooks'; +import { ReportContext } from '../Report'; + +export function RetentionTable() { + const { report } = useContext(ReportContext); + const { formatMessage, labels } = useMessages(); + return ( + + ); +} + +export default RetentionTable; diff --git a/pages/api/reports/retention.ts b/pages/api/reports/retention.ts new file mode 100644 index 000000000..6b8aebcc3 --- /dev/null +++ b/pages/api/reports/retention.ts @@ -0,0 +1,55 @@ +import { canViewWebsite } from 'lib/auth'; +import { useCors, useAuth } from 'lib/middleware'; +import { NextApiRequestQueryBody } from 'lib/types'; +import { NextApiResponse } from 'next'; +import { ok, methodNotAllowed, unauthorized } from 'next-basics'; +import { getRetention } from 'queries'; + +export interface RetentionRequestBody { + websiteId: string; + urls: string[]; + window: number; + dateRange: { + startDate: string; + endDate: string; + }; +} + +export interface RetentionResponse { + urls: string[]; + window: number; + startAt: number; + endAt: number; +} + +export default async ( + req: NextApiRequestQueryBody, + res: NextApiResponse, +) => { + await useCors(req, res); + await useAuth(req, res); + + if (req.method === 'POST') { + const { + websiteId, + urls, + window, + dateRange: { startDate, endDate }, + } = req.body; + + if (!(await canViewWebsite(req.auth, websiteId))) { + return unauthorized(res); + } + + const data = await getRetention(websiteId, { + startDate: new Date(startDate), + endDate: new Date(endDate), + urls, + windowMinutes: +window, + }); + + return ok(res, data); + } + + return methodNotAllowed(res); +}; diff --git a/pages/reports/retention.js b/pages/reports/retention.js new file mode 100644 index 000000000..b7f0bd0f8 --- /dev/null +++ b/pages/reports/retention.js @@ -0,0 +1,13 @@ +import AppLayout from 'components/layout/AppLayout'; +import RetentionReport from 'components/pages/reports/retention/RetentionReport'; +import useMessages from 'hooks/useMessages'; + +export default function () { + const { formatMessage, labels } = useMessages(); + + return ( + + + + ); +} diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.ts new file mode 100644 index 000000000..b2c478827 --- /dev/null +++ b/queries/analytics/reports/getRetention.ts @@ -0,0 +1,209 @@ +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; + +export async function getRetention( + ...args: [ + websiteId: string, + criteria: { + windowMinutes: number; + startDate: Date; + endDate: Date; + urls: string[]; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + criteria: { + windowMinutes: number; + startDate: Date; + endDate: Date; + urls: string[]; + }, +): Promise< + { + x: string; + y: number; + z: number; + }[] +> { + const { windowMinutes, startDate, endDate, urls } = criteria; + const { rawQuery, getAddMinutesQuery } = prisma; + const { levelQuery, sumQuery } = getRetentionQuery(urls, windowMinutes); + + function getRetentionQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += ` + , level${levelNumber} AS ( + select distinct we.session_id, we.created_at + from level${i} l + join website_event we + on l.session_id = we.session_id + where we.created_at between l.created_at + and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} + and we.referrer_path = {{${i - 1}}} + and we.url_path = {{${i}}} + and we.created_at <= {{endDate}} + and we.website_id = {{websiteId::uuid}} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + }, + ); + } + + return rawQuery( + ` + WITH level1 AS ( + select distinct session_id, created_at + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and url_path = {{0}} + ) + ${levelQuery} + ${sumQuery} + ORDER BY level; + `, + { + websiteId, + startDate, + endDate, + ...urls, + }, + ).then(results => { + return urls.map((a, i) => ({ + x: a, + y: results[i]?.count || 0, + z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off + })); + }); +} + +async function clickhouseQuery( + websiteId: string, + criteria: { + windowMinutes: number; + startDate: Date; + endDate: Date; + urls: string[]; + }, +): Promise< + { + x: string; + y: number; + z: number; + }[] +> { + const { windowMinutes, startDate, endDate, urls } = criteria; + const { rawQuery } = clickhouse; + const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery( + urls, + windowMinutes, + ); + + function getRetentionQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + urlFilterQuery: string; + urlParams: { [key: string]: string }; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union all ' : ''; + const startFilter = i > 0 ? ', ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += `\n + , level${levelNumber} AS ( + select distinct y.session_id as session_id, + y.url_path as url_path, + y.referrer_path as referrer_path, + y.created_at as created_at + from level${i} x + join level0 y + on x.session_id = y.session_id + where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute + and y.referrer_path = {url${i - 1}:String} + and y.url_path = {url${i}:String} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + pv.urlFilterQuery += `${startFilter}{url${i}:String} `; + pv.urlParams[`url${i}`] = cv; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + urlFilterQuery: '', + urlParams: {}, + }, + ); + } + + return rawQuery<{ level: number; count: number }[]>( + ` + WITH level0 AS ( + select distinct session_id, url_path, referrer_path, created_at + from umami.website_event + where url_path in (${urlFilterQuery}) + and website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + ), + level1 AS ( + select * + from level0 + where url_path = {url0:String} + ) + ${levelQuery} + select * + from ( + ${sumQuery} + ) ORDER BY level; + `, + { + websiteId, + startDate, + endDate, + ...urlParams, + }, + ).then(results => { + return urls.map((a, i) => ({ + x: a, + y: results[i]?.count || 0, + z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off + })); + }); +} diff --git a/queries/index.js b/queries/index.js index f509e0392..0fb2bf2cf 100644 --- a/queries/index.js +++ b/queries/index.js @@ -12,6 +12,7 @@ export * from './analytics/eventData/getEventDataFields'; export * from './analytics/eventData/getEventDataUsage'; export * from './analytics/events/saveEvent'; export * from './analytics/reports/getFunnel'; +export * from './analytics/reports/getRetention'; export * from './analytics/reports/getInsights'; export * from './analytics/pageviews/getPageviewMetrics'; export * from './analytics/pageviews/getPageviewStats'; From 13530c9cdcf38289c80cc42563b05cb92dcf53ea Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 7 Aug 2023 14:01:53 -0700 Subject: [PATCH 2/5] add psql query for retention --- .../reports/retention/RetentionParameters.js | 14 +- .../reports/retention/RetentionReport.js | 6 +- .../pages/reports/retention/RetentionTable.js | 25 +- pages/api/reports/retention.ts | 15 +- queries/analytics/reports/getRetention.ts | 328 +++++++++--------- 5 files changed, 193 insertions(+), 195 deletions(-) diff --git a/components/pages/reports/retention/RetentionParameters.js b/components/pages/reports/retention/RetentionParameters.js index 29c0eff2f..bf40236df 100644 --- a/components/pages/reports/retention/RetentionParameters.js +++ b/components/pages/reports/retention/RetentionParameters.js @@ -4,6 +4,11 @@ import { Form, FormButtons, FormInput, FormRow, SubmitButton, TextField } from ' import { ReportContext } from 'components/pages/reports/Report'; import BaseParameters from '../BaseParameters'; +const fieldOptions = [ + { name: 'daily', type: 'string' }, + { name: 'weekly', type: 'string' }, +]; + export function RetentionParameters() { const { report, runReport, isRunning } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); @@ -24,14 +29,7 @@ export function RetentionParameters() { return (
- - - - - + {formatMessage(labels.runQuery)} diff --git a/components/pages/reports/retention/RetentionReport.js b/components/pages/reports/retention/RetentionReport.js index 31d085f7b..cab3c16c2 100644 --- a/components/pages/reports/retention/RetentionReport.js +++ b/components/pages/reports/retention/RetentionReport.js @@ -8,8 +8,8 @@ import ReportBody from '../ReportBody'; import Funnel from 'assets/funnel.svg'; const defaultParameters = { - type: 'Retention', - parameters: { window: 60, urls: [] }, + type: 'retention', + parameters: {}, }; export default function RetentionReport({ reportId }) { @@ -20,7 +20,7 @@ export default function RetentionReport({ reportId }) { - + {/* */} diff --git a/components/pages/reports/retention/RetentionTable.js b/components/pages/reports/retention/RetentionTable.js index 4ef879862..53db7841d 100644 --- a/components/pages/reports/retention/RetentionTable.js +++ b/components/pages/reports/retention/RetentionTable.js @@ -1,18 +1,29 @@ import { useContext } from 'react'; -import DataTable from 'components/metrics/DataTable'; +import { GridTable, GridColumn } from 'react-basics'; import { useMessages } from 'hooks'; import { ReportContext } from '../Report'; export function RetentionTable() { const { report } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); + const { fields = [] } = report?.parameters || {}; + + // return ( + // + // {fields.map(({ name }) => { + // return ; + // })} + // + // + // ); return ( - + + {row => row.cohortDate} + {row => row.date_number} + + {row => row.date_number} + + ); } diff --git a/pages/api/reports/retention.ts b/pages/api/reports/retention.ts index 6b8aebcc3..0e2c71b86 100644 --- a/pages/api/reports/retention.ts +++ b/pages/api/reports/retention.ts @@ -7,17 +7,12 @@ import { getRetention } from 'queries'; export interface RetentionRequestBody { websiteId: string; - urls: string[]; - window: number; - dateRange: { - startDate: string; - endDate: string; - }; + window: string; + dateRange: { window; startDate: string; endDate: string }; } export interface RetentionResponse { - urls: string[]; - window: number; + window: string; startAt: number; endAt: number; } @@ -32,7 +27,6 @@ export default async ( if (req.method === 'POST') { const { websiteId, - urls, window, dateRange: { startDate, endDate }, } = req.body; @@ -44,8 +38,7 @@ export default async ( const data = await getRetention(websiteId, { startDate: new Date(startDate), endDate: new Date(endDate), - urls, - windowMinutes: +window, + window: window, }); return ok(res, data); diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.ts index b2c478827..68d3b4b2a 100644 --- a/queries/analytics/reports/getRetention.ts +++ b/queries/analytics/reports/getRetention.ts @@ -6,204 +6,200 @@ export async function getRetention( ...args: [ websiteId: string, criteria: { - windowMinutes: number; + window: string; startDate: Date; endDate: Date; - urls: string[]; }, ] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), - [CLICKHOUSE]: () => clickhouseQuery(...args), + // [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery( websiteId: string, criteria: { - windowMinutes: number; + window: string; startDate: Date; endDate: Date; - urls: string[]; }, ): Promise< { - x: string; - y: number; - z: number; + date: Date; + visitors: number; + day: number; + percentage: number; }[] > { - const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getAddMinutesQuery } = prisma; - const { levelQuery, sumQuery } = getRetentionQuery(urls, windowMinutes); - - function getRetentionQuery( - urls: string[], - windowMinutes: number, - ): { - levelQuery: string; - sumQuery: string; - } { - return urls.reduce( - (pv, cv, i) => { - const levelNumber = i + 1; - const startSum = i > 0 ? 'union ' : ''; - - if (levelNumber >= 2) { - pv.levelQuery += ` - , level${levelNumber} AS ( - select distinct we.session_id, we.created_at - from level${i} l - join website_event we - on l.session_id = we.session_id - where we.created_at between l.created_at - and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} - and we.referrer_path = {{${i - 1}}} - and we.url_path = {{${i}}} - and we.created_at <= {{endDate}} - and we.website_id = {{websiteId::uuid}} - )`; - } - - pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - - return pv; - }, - { - levelQuery: '', - sumQuery: '', - }, - ); - } + const { window, startDate, endDate } = criteria; + const { rawQuery } = prisma; return rawQuery( ` - WITH level1 AS ( - select distinct session_id, created_at - from website_event + WITH cohort_items AS ( + select + date_trunc('week', created_at)::date as cohort_date, + session_id + from session where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} - and url_path = {{0}} - ) - ${levelQuery} - ${sumQuery} - ORDER BY level; - `, - { - websiteId, - startDate, - endDate, - ...urls, - }, - ).then(results => { - return urls.map((a, i) => ({ - x: a, - y: results[i]?.count || 0, - z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off - })); - }); -} - -async function clickhouseQuery( - websiteId: string, - criteria: { - windowMinutes: number; - startDate: Date; - endDate: Date; - urls: string[]; - }, -): Promise< - { - x: string; - y: number; - z: number; - }[] -> { - const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery } = clickhouse; - const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery( - urls, - windowMinutes, - ); - - function getRetentionQuery( - urls: string[], - windowMinutes: number, - ): { - levelQuery: string; - sumQuery: string; - urlFilterQuery: string; - urlParams: { [key: string]: string }; - } { - return urls.reduce( - (pv, cv, i) => { - const levelNumber = i + 1; - const startSum = i > 0 ? 'union all ' : ''; - const startFilter = i > 0 ? ', ' : ''; - - if (levelNumber >= 2) { - pv.levelQuery += `\n - , level${levelNumber} AS ( - select distinct y.session_id as session_id, - y.url_path as url_path, - y.referrer_path as referrer_path, - y.created_at as created_at - from level${i} x - join level0 y - on x.session_id = y.session_id - where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute - and y.referrer_path = {url${i - 1}:String} - and y.url_path = {url${i}:String} - )`; - } - - pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - pv.urlFilterQuery += `${startFilter}{url${i}:String} `; - pv.urlParams[`url${i}`] = cv; - - return pv; - }, - { - levelQuery: '', - sumQuery: '', - urlFilterQuery: '', - urlParams: {}, - }, - ); - } - - return rawQuery<{ level: number; count: number }[]>( - ` - WITH level0 AS ( - select distinct session_id, url_path, referrer_path, created_at - from umami.website_event - where url_path in (${urlFilterQuery}) - and website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} + order by 1, 2 ), - level1 AS ( - select * - from level0 - where url_path = {url0:String} + user_activities AS ( + select distinct + w.session_id, + (date_trunc('week', w.created_at)::date - c.cohort_date::date) / 7 as date_number + from website_event w + left join cohort_items c + on w.session_id = c.session_id + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + ), + cohort_size as ( + select cohort_date, + count(*) as visitors + from cohort_items + group by 1 + order by 1 + ), + cohort_date as ( + select + c.cohort_date, + a.date_number, + count(*) as visitors + from user_activities a + left join cohort_items c + on a.session_id = c.session_id + group by 1, 2 ) - ${levelQuery} - select * - from ( - ${sumQuery} - ) ORDER BY level; - `, + select + c.cohort_date, + c.date_number, + s.visitors, + c.visitors, + c.visitors::float * 100 / s.visitors as percentage + from cohort_date c + left join cohort_size s + on c.cohort_date = s.cohort_date + where c.cohort_date IS NOT NULL + order by 1, 2`, { websiteId, startDate, endDate, - ...urlParams, + window, }, ).then(results => { - return urls.map((a, i) => ({ - x: a, - y: results[i]?.count || 0, - z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off - })); + return results; + // return results.map((a, i) => ({ + // x: a, + // y: results[i]?.count || 0, + // z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off + // })); }); } + +// async function clickhouseQuery( +// websiteId: string, +// criteria: { +// windowMinutes: number; +// startDate: Date; +// endDate: Date; +// urls: string[]; +// }, +// ): Promise< +// { +// x: string; +// y: number; +// z: number; +// }[] +// > { +// const { windowMinutes, startDate, endDate, urls } = criteria; +// const { rawQuery } = clickhouse; +// const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery( +// urls, +// windowMinutes, +// ); + +// function getRetentionQuery( +// urls: string[], +// windowMinutes: number, +// ): { +// levelQuery: string; +// sumQuery: string; +// urlFilterQuery: string; +// urlParams: { [key: string]: string }; +// } { +// return urls.reduce( +// (pv, cv, i) => { +// const levelNumber = i + 1; +// const startSum = i > 0 ? 'union all ' : ''; +// const startFilter = i > 0 ? ', ' : ''; + +// if (levelNumber >= 2) { +// pv.levelQuery += `\n +// , level${levelNumber} AS ( +// select distinct y.session_id as session_id, +// y.url_path as url_path, +// y.referrer_path as referrer_path, +// y.created_at as created_at +// from level${i} x +// join level0 y +// on x.session_id = y.session_id +// where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute +// and y.referrer_path = {url${i - 1}:String} +// and y.url_path = {url${i}:String} +// )`; +// } + +// pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; +// pv.urlFilterQuery += `${startFilter}{url${i}:String} `; +// pv.urlParams[`url${i}`] = cv; + +// return pv; +// }, +// { +// levelQuery: '', +// sumQuery: '', +// urlFilterQuery: '', +// urlParams: {}, +// }, +// ); +// } + +// return rawQuery<{ level: number; count: number }[]>( +// ` +// WITH level0 AS ( +// select distinct session_id, url_path, referrer_path, created_at +// from umami.website_event +// where url_path in (${urlFilterQuery}) +// and website_id = {websiteId:UUID} +// and created_at between {startDate:DateTime64} and {endDate:DateTime64} +// ), +// level1 AS ( +// select * +// from level0 +// where url_path = {url0:String} +// ) +// ${levelQuery} +// select * +// from ( +// ${sumQuery} +// ) ORDER BY level; +// `, +// { +// websiteId, +// startDate, +// endDate, +// ...urlParams, +// }, +// ).then(results => { +// return urls.map((a, i) => ({ +// x: a, +// y: results[i]?.count || 0, +// z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off +// })); +// }); +// } From 15575d7783be1b1a8b1a3d13ddc2199fd037dde6 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 7 Aug 2023 14:41:41 -0700 Subject: [PATCH 3/5] fix column data in relational event data query --- .../analytics/eventData/getEventDataEvents.ts | 16 ++++++++-------- .../analytics/eventData/getEventDataFields.ts | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/queries/analytics/eventData/getEventDataEvents.ts b/queries/analytics/eventData/getEventDataEvents.ts index ec0939b64..a3a19bb1c 100644 --- a/queries/analytics/eventData/getEventDataEvents.ts +++ b/queries/analytics/eventData/getEventDataEvents.ts @@ -21,10 +21,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` select - website_event.event_name as eventName, - event_data.event_key as fieldName, - event_data.data_type as dataType, - event_data.string_value as value, + website_event.event_name as "eventName", + event_data.event_key as "fieldName", + event_data.data_type as "dataType", + event_data.string_value as "value", count(*) as total from event_data inner join website_event @@ -42,10 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` select - website_event.event_name as eventName, - event_data.event_key as fieldName, - event_data.data_type as dataType, - count(*) as total + website_event.event_name as "eventName", + event_data.event_key as "fieldName", + event_data.data_type as "dataType", + count(*) as "total" from event_data inner join website_event on website_event.event_id = event_data.website_event_id diff --git a/queries/analytics/eventData/getEventDataFields.ts b/queries/analytics/eventData/getEventDataFields.ts index c61de517c..f5f426e0c 100644 --- a/queries/analytics/eventData/getEventDataFields.ts +++ b/queries/analytics/eventData/getEventDataFields.ts @@ -21,10 +21,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel return rawQuery( ` select - event_key as fieldName, - data_type as dataType, - string_value as fieldValue, - count(*) as total + event_key as "fieldName", + data_type as "dataType", + string_value as "fieldValue", + count(*) as "total" from event_data where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} From bf507037c73e1f23dc56c7a2aa04be017ae4277d Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 8 Aug 2023 11:57:58 -0700 Subject: [PATCH 4/5] finish CH query and clean up objects --- .../reports/retention/RetentionParameters.js | 1 - .../pages/reports/retention/RetentionTable.js | 25 +- pages/api/reports/retention.ts | 4 - queries/analytics/reports/getRetention.ts | 231 ++++++++---------- 4 files changed, 109 insertions(+), 152 deletions(-) diff --git a/components/pages/reports/retention/RetentionParameters.js b/components/pages/reports/retention/RetentionParameters.js index bf40236df..f6bde0b16 100644 --- a/components/pages/reports/retention/RetentionParameters.js +++ b/components/pages/reports/retention/RetentionParameters.js @@ -29,7 +29,6 @@ export function RetentionParameters() { return ( - {formatMessage(labels.runQuery)} diff --git a/components/pages/reports/retention/RetentionTable.js b/components/pages/reports/retention/RetentionTable.js index 53db7841d..35d55a640 100644 --- a/components/pages/reports/retention/RetentionTable.js +++ b/components/pages/reports/retention/RetentionTable.js @@ -6,22 +6,23 @@ import { ReportContext } from '../Report'; export function RetentionTable() { const { report } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); - const { fields = [] } = report?.parameters || {}; - // return ( - // - // {fields.map(({ name }) => { - // return ; - // })} - // - // - // ); return ( - {row => row.cohortDate} - {row => row.date_number} + + {row => row.date} + + + {row => row.day} + - {row => row.date_number} + {row => row.visitors} + + + {row => row.returnVisitors} + + + {row => row.percentage} ); diff --git a/pages/api/reports/retention.ts b/pages/api/reports/retention.ts index 0e2c71b86..83ed0b574 100644 --- a/pages/api/reports/retention.ts +++ b/pages/api/reports/retention.ts @@ -7,12 +7,10 @@ import { getRetention } from 'queries'; export interface RetentionRequestBody { websiteId: string; - window: string; dateRange: { window; startDate: string; endDate: string }; } export interface RetentionResponse { - window: string; startAt: number; endAt: number; } @@ -27,7 +25,6 @@ export default async ( if (req.method === 'POST') { const { websiteId, - window, dateRange: { startDate, endDate }, } = req.body; @@ -38,7 +35,6 @@ export default async ( const data = await getRetention(websiteId, { startDate: new Date(startDate), endDate: new Date(endDate), - window: window, }); return ok(res, data); diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.ts index 68d3b4b2a..c34ba068e 100644 --- a/queries/analytics/reports/getRetention.ts +++ b/queries/analytics/reports/getRetention.ts @@ -5,8 +5,7 @@ import prisma from 'lib/prisma'; export async function getRetention( ...args: [ websiteId: string, - criteria: { - window: string; + dateRange: { startDate: Date; endDate: Date; }, @@ -14,48 +13,121 @@ export async function getRetention( ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), - // [CLICKHOUSE]: () => clickhouseQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), }); } async function relationalQuery( websiteId: string, - criteria: { - window: string; + dateRange: { startDate: Date; endDate: Date; }, ): Promise< { date: Date; - visitors: number; day: number; + visitors: number; + returnVisitors: number; percentage: number; }[] > { - const { window, startDate, endDate } = criteria; + const { startDate, endDate } = dateRange; const { rawQuery } = prisma; return rawQuery( ` WITH cohort_items AS ( - select - date_trunc('week', created_at)::date as cohort_date, - session_id + select session_id, + date_trunc('day', created_at)::date as cohort_date from session where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} - order by 1, 2 ), user_activities AS ( select distinct w.session_id, - (date_trunc('week', w.created_at)::date - c.cohort_date::date) / 7 as date_number + (date_trunc('day', w.created_at)::date - c.cohort_date::date) as day_number from website_event w - left join cohort_items c + join cohort_items c on w.session_id = c.session_id where website_id = {{websiteId::uuid}} - and created_at between {{startDate}} and {{endDate}} + and created_at between {{startDate}} and {{endDate}} + ), + cohort_size as ( + select cohort_date, + count(*) as visitors + from cohort_items + group by 1 + order by 1 + ), + cohort_date as ( + select + c.cohort_date, + a.day_number, + count(*) as visitors + from user_activities a + join cohort_items c + on a.session_id = c.session_id + where a.day_number IN (0,1,2,3,4,5,6,7,14,21,30) + group by 1, 2 + ) + select + c.cohort_date as date, + c.day_number as day, + s.visitors, + c.visitors as "returnVisitors", + c.visitors::float * 100 / s.visitors as percentage + from cohort_date c + join cohort_size s + on c.cohort_date = s.cohort_date + order by 1, 2`, + { + websiteId, + startDate, + endDate, + }, + ); +} + +async function clickhouseQuery( + websiteId: string, + dateRange: { + startDate: Date; + endDate: Date; + }, +): Promise< + { + date: Date; + day: number; + visitors: number; + returnVisitors: number; + percentage: number; + }[] +> { + const { startDate, endDate } = dateRange; + const { rawQuery } = clickhouse; + + return rawQuery( + ` + WITH cohort_items AS ( + select + min(date_trunc('day', created_at)) as cohort_date, + session_id + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + group by session_id + ), + user_activities AS ( + select distinct + w.session_id, + (date_trunc('day', w.created_at) - c.cohort_date) / 86400 as day_number + from website_event w + join cohort_items c + on w.session_id = c.session_id + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} ), cohort_size as ( select cohort_date, @@ -67,139 +139,28 @@ async function relationalQuery( cohort_date as ( select c.cohort_date, - a.date_number, + a.day_number, count(*) as visitors from user_activities a - left join cohort_items c + join cohort_items c on a.session_id = c.session_id + where a.day_number IN (0,1,2,3,4,5,6,7,14,21,30) group by 1, 2 ) select - c.cohort_date, - c.date_number, - s.visitors, - c.visitors, - c.visitors::float * 100 / s.visitors as percentage + c.cohort_date as date, + c.day_number as day, + s.visitors as visitors, + c.visitors returnVisitors, + c.visitors * 100 / s.visitors as percentage from cohort_date c - left join cohort_size s + join cohort_size s on c.cohort_date = s.cohort_date - where c.cohort_date IS NOT NULL order by 1, 2`, { websiteId, startDate, endDate, - window, }, - ).then(results => { - return results; - // return results.map((a, i) => ({ - // x: a, - // y: results[i]?.count || 0, - // z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off - // })); - }); + ); } - -// async function clickhouseQuery( -// websiteId: string, -// criteria: { -// windowMinutes: number; -// startDate: Date; -// endDate: Date; -// urls: string[]; -// }, -// ): Promise< -// { -// x: string; -// y: number; -// z: number; -// }[] -// > { -// const { windowMinutes, startDate, endDate, urls } = criteria; -// const { rawQuery } = clickhouse; -// const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery( -// urls, -// windowMinutes, -// ); - -// function getRetentionQuery( -// urls: string[], -// windowMinutes: number, -// ): { -// levelQuery: string; -// sumQuery: string; -// urlFilterQuery: string; -// urlParams: { [key: string]: string }; -// } { -// return urls.reduce( -// (pv, cv, i) => { -// const levelNumber = i + 1; -// const startSum = i > 0 ? 'union all ' : ''; -// const startFilter = i > 0 ? ', ' : ''; - -// if (levelNumber >= 2) { -// pv.levelQuery += `\n -// , level${levelNumber} AS ( -// select distinct y.session_id as session_id, -// y.url_path as url_path, -// y.referrer_path as referrer_path, -// y.created_at as created_at -// from level${i} x -// join level0 y -// on x.session_id = y.session_id -// where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute -// and y.referrer_path = {url${i - 1}:String} -// and y.url_path = {url${i}:String} -// )`; -// } - -// pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; -// pv.urlFilterQuery += `${startFilter}{url${i}:String} `; -// pv.urlParams[`url${i}`] = cv; - -// return pv; -// }, -// { -// levelQuery: '', -// sumQuery: '', -// urlFilterQuery: '', -// urlParams: {}, -// }, -// ); -// } - -// return rawQuery<{ level: number; count: number }[]>( -// ` -// WITH level0 AS ( -// select distinct session_id, url_path, referrer_path, created_at -// from umami.website_event -// where url_path in (${urlFilterQuery}) -// and website_id = {websiteId:UUID} -// and created_at between {startDate:DateTime64} and {endDate:DateTime64} -// ), -// level1 AS ( -// select * -// from level0 -// where url_path = {url0:String} -// ) -// ${levelQuery} -// select * -// from ( -// ${sumQuery} -// ) ORDER BY level; -// `, -// { -// websiteId, -// startDate, -// endDate, -// ...urlParams, -// }, -// ).then(results => { -// return urls.map((a, i) => ({ -// x: a, -// y: results[i]?.count || 0, -// z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off -// })); -// }); -// } From 577294191d6991e2f5a7de06e00f438940a56d38 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 8 Aug 2023 12:03:03 -0700 Subject: [PATCH 5/5] remove retention chart --- .../pages/reports/retention/RetentionChart.js | 74 ------------------- .../retention/RetentionChart.module.css | 3 - .../reports/retention/RetentionReport.js | 2 - 3 files changed, 79 deletions(-) delete mode 100644 components/pages/reports/retention/RetentionChart.js delete mode 100644 components/pages/reports/retention/RetentionChart.module.css diff --git a/components/pages/reports/retention/RetentionChart.js b/components/pages/reports/retention/RetentionChart.js deleted file mode 100644 index 5f7361fdb..000000000 --- a/components/pages/reports/retention/RetentionChart.js +++ /dev/null @@ -1,74 +0,0 @@ -import { useCallback, useContext, useMemo } from 'react'; -import { Loading, StatusLight } from 'react-basics'; -import useMessages from 'hooks/useMessages'; -import useTheme from 'hooks/useTheme'; -import BarChart from 'components/metrics/BarChart'; -import { formatLongNumber } from 'lib/format'; -import styles from './RetentionChart.module.css'; -import { ReportContext } from '../Report'; - -export function RetentionChart({ className, loading }) { - const { report } = useContext(ReportContext); - const { formatMessage, labels } = useMessages(); - const { colors } = useTheme(); - - const { parameters, data } = report || {}; - - const renderXLabel = useCallback( - (label, index) => { - return parameters.urls[index]; - }, - [parameters], - ); - - const renderTooltipPopup = useCallback((setTooltipPopup, model) => { - const { opacity, labelColors, dataPoints } = model.tooltip; - - if (!dataPoints?.length || !opacity) { - setTooltipPopup(null); - return; - } - - setTooltipPopup( - <> -
- {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)} -
-
- - {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)} - -
- , - ); - }, []); - - const datasets = useMemo(() => { - return [ - { - label: formatMessage(labels.uniqueVisitors), - data: data, - borderWidth: 1, - ...colors.chart.visitors, - }, - ]; - }, [data]); - - if (loading) { - return ; - } - - return ( - - ); -} - -export default RetentionChart; diff --git a/components/pages/reports/retention/RetentionChart.module.css b/components/pages/reports/retention/RetentionChart.module.css deleted file mode 100644 index 9e1690b31..000000000 --- a/components/pages/reports/retention/RetentionChart.module.css +++ /dev/null @@ -1,3 +0,0 @@ -.loading { - height: 300px; -} diff --git a/components/pages/reports/retention/RetentionReport.js b/components/pages/reports/retention/RetentionReport.js index cab3c16c2..333496d82 100644 --- a/components/pages/reports/retention/RetentionReport.js +++ b/components/pages/reports/retention/RetentionReport.js @@ -1,4 +1,3 @@ -import RetentionChart from './RetentionChart'; import RetentionTable from './RetentionTable'; import RetentionParameters from './RetentionParameters'; import Report from '../Report'; @@ -20,7 +19,6 @@ export default function RetentionReport({ reportId }) { - {/* */}