build out retention reports

components/messages.js

@@ -161,6 +161,7 @@ export const labels = defineMessages({
   overview: { id: 'labels.overview', defaultMessage: 'Overview' },
   totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' },
   insights: { id: 'label.insights', defaultMessage: 'Insights' },
+  retention: { id: 'label.retention', defaultMessage: 'Retention' },
   dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' },
 });
 

components/pages/reports/ReportDetails.js

@@ -1,9 +1,11 @@
 import FunnelReport from './funnel/FunnelReport';
 import EventDataReport from './event-data/EventDataReport';
+import RetentionReport from './retention/RetentionReport';
 
 const reports = {
   funnel: FunnelReport,
   'event-data': EventDataReport,
+  retention: RetentionReport,
 };
 
 export default function ReportDetails({ reportId, reportType }) {

components/pages/reports/ReportTemplates.js

@@ -47,6 +47,12 @@ export function ReportTemplates() {
       url: '/reports/funnel',
       icon: <Funnel />,
     },
+    {
+      title: formatMessage(labels.retention),
+      description: 'Track your websites user retention',
+      url: '/reports/retention',
+      icon: <Funnel />,
+    },
   ];
 
   return (

components/pages/reports/funnel/FunnelChart.js

@@ -1,5 +1,5 @@
 import { useCallback, useContext, useMemo } from 'react';
-import { Loading } from 'react-basics';
+import { Loading, StatusLight } from 'react-basics';
 import useMessages from 'hooks/useMessages';
 import useTheme from 'hooks/useTheme';
 import BarChart from 'components/metrics/BarChart';
@@ -22,14 +22,25 @@ export function FunnelChart({ className, loading }) {
   );
 
   const renderTooltipPopup = useCallback((setTooltipPopup, model) => {
-    const { opacity, dataPoints } = model.tooltip;
+    const { opacity, labelColors, dataPoints } = model.tooltip;
 
     if (!dataPoints?.length || !opacity) {
       setTooltipPopup(null);
       return;
     }
 
-    setTooltipPopup(`${formatLongNumber(dataPoints[0].raw.y)} ${formatMessage(labels.visitors)}`);
+    setTooltipPopup(
+      <>
+        <div>
+          {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)}
+        </div>
+        <div>
+          <StatusLight color={labelColors?.[0]?.backgroundColor}>
+            {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)}
+          </StatusLight>
+        </div>
+      </>,
+    );
   }, []);
 
   const datasets = useMemo(() => {

components/pages/reports/retention/RetentionChart.js

@@ -0,0 +1,74 @@
+import { useCallback, useContext, useMemo } from 'react';
+import { Loading, StatusLight } from 'react-basics';
+import useMessages from 'hooks/useMessages';
+import useTheme from 'hooks/useTheme';
+import BarChart from 'components/metrics/BarChart';
+import { formatLongNumber } from 'lib/format';
+import styles from './RetentionChart.module.css';
+import { ReportContext } from '../Report';
+
+export function RetentionChart({ className, loading }) {
+  const { report } = useContext(ReportContext);
+  const { formatMessage, labels } = useMessages();
+  const { colors } = useTheme();
+
+  const { parameters, data } = report || {};
+
+  const renderXLabel = useCallback(
+    (label, index) => {
+      return parameters.urls[index];
+    },
+    [parameters],
+  );
+
+  const renderTooltipPopup = useCallback((setTooltipPopup, model) => {
+    const { opacity, labelColors, dataPoints } = model.tooltip;
+
+    if (!dataPoints?.length || !opacity) {
+      setTooltipPopup(null);
+      return;
+    }
+
+    setTooltipPopup(
+      <>
+        <div>
+          {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)}
+        </div>
+        <div>
+          <StatusLight color={labelColors?.[0]?.backgroundColor}>
+            {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)}
+          </StatusLight>
+        </div>
+      </>,
+    );
+  }, []);
+
+  const datasets = useMemo(() => {
+    return [
+      {
+        label: formatMessage(labels.uniqueVisitors),
+        data: data,
+        borderWidth: 1,
+        ...colors.chart.visitors,
+      },
+    ];
+  }, [data]);
+
+  if (loading) {
+    return <Loading icon="dots" className={styles.loading} />;
+  }
+
+  return (
+    <BarChart
+      className={className}
+      datasets={datasets}
+      unit="day"
+      loading={loading}
+      renderXLabel={renderXLabel}
+      renderTooltipPopup={renderTooltipPopup}
+      XAxisType="category"
+    />
+  );
+}
+
+export default RetentionChart;

components/pages/reports/retention/RetentionChart.module.css

@@ -0,0 +1,3 @@
+.loading {
+  height: 300px;
+}

components/pages/reports/retention/RetentionParameters.js

@@ -0,0 +1,44 @@
+import { useContext, useRef } from 'react';
+import { useMessages } from 'hooks';
+import { Form, FormButtons, FormInput, FormRow, SubmitButton, TextField } from 'react-basics';
+import { ReportContext } from 'components/pages/reports/Report';
+import BaseParameters from '../BaseParameters';
+
+export function RetentionParameters() {
+  const { report, runReport, isRunning } = useContext(ReportContext);
+  const { formatMessage, labels } = useMessages();
+  const ref = useRef(null);
+
+  const { parameters } = report || {};
+  const { websiteId, dateRange } = parameters || {};
+  const queryDisabled = !websiteId || !dateRange;
+
+  const handleSubmit = (data, e) => {
+    e.stopPropagation();
+    e.preventDefault();
+    if (!queryDisabled) {
+      runReport(data);
+    }
+  };
+
+  return (
+    <Form ref={ref} values={parameters} onSubmit={handleSubmit} preventSubmit={true}>
+      <BaseParameters />
+      <FormRow label={formatMessage(labels.window)}>
+        <FormInput
+          name="window"
+          rules={{ required: formatMessage(labels.required), pattern: /[0-9]+/ }}
+        >
+          <TextField autoComplete="off" />
+        </FormInput>
+      </FormRow>
+      <FormButtons>
+        <SubmitButton variant="primary" disabled={queryDisabled} loading={isRunning}>
+          {formatMessage(labels.runQuery)}
+        </SubmitButton>
+      </FormButtons>
+    </Form>
+  );
+}
+
+export default RetentionParameters;

components/pages/reports/retention/RetentionReport.js

@@ -0,0 +1,28 @@
+import RetentionChart from './RetentionChart';
+import RetentionTable from './RetentionTable';
+import RetentionParameters from './RetentionParameters';
+import Report from '../Report';
+import ReportHeader from '../ReportHeader';
+import ReportMenu from '../ReportMenu';
+import ReportBody from '../ReportBody';
+import Funnel from 'assets/funnel.svg';
+
+const defaultParameters = {
+  type: 'Retention',
+  parameters: { window: 60, urls: [] },
+};
+
+export default function RetentionReport({ reportId }) {
+  return (
+    <Report reportId={reportId} defaultParameters={defaultParameters}>
+      <ReportHeader icon={<Funnel />} />
+      <ReportMenu>
+        <RetentionParameters />
+      </ReportMenu>
+      <ReportBody>
+        <RetentionChart />
+        <RetentionTable />
+      </ReportBody>
+    </Report>
+  );
+}

components/pages/reports/retention/RetentionReport.module.css

@@ -0,0 +1,10 @@
+.filters {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  border: 1px solid var(--base400);
+  border-radius: var(--border-radius);
+  line-height: 32px;
+  padding: 10px;
+  overflow: hidden;
+}

components/pages/reports/retention/RetentionTable.js

@@ -0,0 +1,19 @@
+import { useContext } from 'react';
+import DataTable from 'components/metrics/DataTable';
+import { useMessages } from 'hooks';
+import { ReportContext } from '../Report';
+
+export function RetentionTable() {
+  const { report } = useContext(ReportContext);
+  const { formatMessage, labels } = useMessages();
+  return (
+    <DataTable
+      data={report?.data}
+      title={formatMessage(labels.url)}
+      metric={formatMessage(labels.visitors)}
+      showPercentage={true}
+    />
+  );
+}
+
+export default RetentionTable;

pages/api/reports/retention.ts

@@ -0,0 +1,55 @@
+import { canViewWebsite } from 'lib/auth';
+import { useCors, useAuth } from 'lib/middleware';
+import { NextApiRequestQueryBody } from 'lib/types';
+import { NextApiResponse } from 'next';
+import { ok, methodNotAllowed, unauthorized } from 'next-basics';
+import { getRetention } from 'queries';
+
+export interface RetentionRequestBody {
+  websiteId: string;
+  urls: string[];
+  window: number;
+  dateRange: {
+    startDate: string;
+    endDate: string;
+  };
+}
+
+export interface RetentionResponse {
+  urls: string[];
+  window: number;
+  startAt: number;
+  endAt: number;
+}
+
+export default async (
+  req: NextApiRequestQueryBody<any, RetentionRequestBody>,
+  res: NextApiResponse<RetentionResponse>,
+) => {
+  await useCors(req, res);
+  await useAuth(req, res);
+
+  if (req.method === 'POST') {
+    const {
+      websiteId,
+      urls,
+      window,
+      dateRange: { startDate, endDate },
+    } = req.body;
+
+    if (!(await canViewWebsite(req.auth, websiteId))) {
+      return unauthorized(res);
+    }
+
+    const data = await getRetention(websiteId, {
+      startDate: new Date(startDate),
+      endDate: new Date(endDate),
+      urls,
+      windowMinutes: +window,
+    });
+
+    return ok(res, data);
+  }
+
+  return methodNotAllowed(res);
+};

pages/reports/retention.js

@@ -0,0 +1,13 @@
+import AppLayout from 'components/layout/AppLayout';
+import RetentionReport from 'components/pages/reports/retention/RetentionReport';
+import useMessages from 'hooks/useMessages';
+
+export default function () {
+  const { formatMessage, labels } = useMessages();
+
+  return (
+    <AppLayout title={`${formatMessage(labels.retention)} - ${formatMessage(labels.reports)}`}>
+      <RetentionReport />
+    </AppLayout>
+  );
+}

queries/analytics/reports/getRetention.ts

@@ -0,0 +1,209 @@
+import clickhouse from 'lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+
+export async function getRetention(
+  ...args: [
+    websiteId: string,
+    criteria: {
+      windowMinutes: number;
+      startDate: Date;
+      endDate: Date;
+      urls: string[];
+    },
+  ]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  criteria: {
+    windowMinutes: number;
+    startDate: Date;
+    endDate: Date;
+    urls: string[];
+  },
+): Promise<
+  {
+    x: string;
+    y: number;
+    z: number;
+  }[]
+> {
+  const { windowMinutes, startDate, endDate, urls } = criteria;
+  const { rawQuery, getAddMinutesQuery } = prisma;
+  const { levelQuery, sumQuery } = getRetentionQuery(urls, windowMinutes);
+
+  function getRetentionQuery(
+    urls: string[],
+    windowMinutes: number,
+  ): {
+    levelQuery: string;
+    sumQuery: string;
+  } {
+    return urls.reduce(
+      (pv, cv, i) => {
+        const levelNumber = i + 1;
+        const startSum = i > 0 ? 'union ' : '';
+
+        if (levelNumber >= 2) {
+          pv.levelQuery += `
+          , level${levelNumber} AS (
+            select distinct we.session_id, we.created_at
+            from level${i} l
+            join website_event we
+                on l.session_id = we.session_id
+            where we.created_at between l.created_at 
+                and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}
+                and we.referrer_path = {{${i - 1}}}
+                and we.url_path = {{${i}}}
+                and we.created_at <= {{endDate}}
+                and we.website_id = {{websiteId::uuid}}
+          )`;
+        }
+
+        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
+
+        return pv;
+      },
+      {
+        levelQuery: '',
+        sumQuery: '',
+      },
+    );
+  }
+
+  return rawQuery(
+    `
+    WITH level1 AS (
+      select distinct session_id, created_at
+      from website_event
+      where website_id = {{websiteId::uuid}}
+        and created_at between {{startDate}} and {{endDate}}
+        and url_path = {{0}}
+    )
+    ${levelQuery}
+    ${sumQuery}
+    ORDER BY level;
+    `,
+    {
+      websiteId,
+      startDate,
+      endDate,
+      ...urls,
+    },
+  ).then(results => {
+    return urls.map((a, i) => ({
+      x: a,
+      y: results[i]?.count || 0,
+      z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
+    }));
+  });
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  criteria: {
+    windowMinutes: number;
+    startDate: Date;
+    endDate: Date;
+    urls: string[];
+  },
+): Promise<
+  {
+    x: string;
+    y: number;
+    z: number;
+  }[]
+> {
+  const { windowMinutes, startDate, endDate, urls } = criteria;
+  const { rawQuery } = clickhouse;
+  const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery(
+    urls,
+    windowMinutes,
+  );
+
+  function getRetentionQuery(
+    urls: string[],
+    windowMinutes: number,
+  ): {
+    levelQuery: string;
+    sumQuery: string;
+    urlFilterQuery: string;
+    urlParams: { [key: string]: string };
+  } {
+    return urls.reduce(
+      (pv, cv, i) => {
+        const levelNumber = i + 1;
+        const startSum = i > 0 ? 'union all ' : '';
+        const startFilter = i > 0 ? ', ' : '';
+
+        if (levelNumber >= 2) {
+          pv.levelQuery += `\n
+          , level${levelNumber} AS (
+            select distinct y.session_id as session_id,
+                y.url_path as url_path,
+                y.referrer_path as referrer_path,
+                y.created_at as created_at
+            from level${i} x
+            join level0 y
+            on x.session_id = y.session_id
+            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute
+                and y.referrer_path = {url${i - 1}:String}
+                and y.url_path = {url${i}:String}
+          )`;
+        }
+
+        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
+        pv.urlFilterQuery += `${startFilter}{url${i}:String} `;
+        pv.urlParams[`url${i}`] = cv;
+
+        return pv;
+      },
+      {
+        levelQuery: '',
+        sumQuery: '',
+        urlFilterQuery: '',
+        urlParams: {},
+      },
+    );
+  }
+
+  return rawQuery<{ level: number; count: number }[]>(
+    `
+    WITH level0 AS (
+      select distinct session_id, url_path, referrer_path, created_at
+      from umami.website_event
+      where url_path in (${urlFilterQuery})
+        and website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    ),
+    level1 AS (
+      select *
+      from level0
+      where url_path = {url0:String}
+    )
+    ${levelQuery}
+    select *
+    from (
+      ${sumQuery} 
+    ) ORDER BY level;
+    `,
+    {
+      websiteId,
+      startDate,
+      endDate,
+      ...urlParams,
+    },
+  ).then(results => {
+    return urls.map((a, i) => ({
+      x: a,
+      y: results[i]?.count || 0,
+      z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
+    }));
+  });
+}

queries/index.js

@@ -12,6 +12,7 @@ export * from './analytics/eventData/getEventDataFields';
 export * from './analytics/eventData/getEventDataUsage';
 export * from './analytics/events/saveEvent';
 export * from './analytics/reports/getFunnel';
+export * from './analytics/reports/getRetention';
 export * from './analytics/reports/getInsights';
 export * from './analytics/pageviews/getPageviewMetrics';
 export * from './analytics/pageviews/getPageviewStats';