Merge pull request #2149 from umami-software/bug/um-362-relational-funnels-query

Bug/um 362 relational funnels query

components/messages.js

@@ -161,6 +161,7 @@ export const labels = defineMessages({
   overview: { id: 'labels.overview', defaultMessage: 'Overview' },
   totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' },
   insights: { id: 'label.insights', defaultMessage: 'Insights' },
+  dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' },
 });
 
 export const messages = defineMessages({

components/pages/reports/funnel/FunnelTable.js

@@ -6,13 +6,12 @@ import { ReportContext } from '../Report';
 export function FunnelTable() {
   const { report } = useContext(ReportContext);
   const { formatMessage, labels } = useMessages();
-
   return (
     <DataTable
       data={report?.data}
       title={formatMessage(labels.url)}
       metric={formatMessage(labels.visitors)}
-      showPercentage={false}
+      showPercentage={true}
     />
   );
 }

lib/clickhouse.ts

@@ -77,29 +77,6 @@ function getFilterQuery(filters = {}, params = {}) {
   return query.join('\n');
 }
 
-function getFunnelQuery(urls: string[]): {
-  columnsQuery: string;
-  conditionQuery: string;
-  urlParams: { [key: string]: string };
-} {
-  return urls.reduce(
-    (pv, cv, i) => {
-      pv.columnsQuery += `\n,url_path = {url${i}:String}${
-        i > 0 && urls[i - 1] ? ` AND referrer_path = {url${i - 1}:String}` : ''
-      }`;
-      pv.conditionQuery += `${i > 0 ? ',' : ''} {url${i}:String}`;
-      pv.urlParams[`url${i}`] = cv;
-
-      return pv;
-    },
-    {
-      columnsQuery: '',
-      conditionQuery: '',
-      urlParams: {},
-    },
-  );
-}
-
 function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
   return {
     filterQuery: getFilterQuery(filters, params),
@@ -146,7 +123,6 @@ export default {
   getDateQuery,
   getDateFormat,
   getFilterQuery,
-  getFunnelQuery,
   parseFilters,
   findUnique,
   findFirst,

lib/prisma.ts

@@ -19,8 +19,8 @@ const POSTGRESQL_DATE_FORMATS = {
   year: 'YYYY-01-01',
 };
 
-function getAddMinutesQuery(field: string, minutes: number) {
-  const db = getDatabaseType();
+function getAddMinutesQuery(field: string, minutes: number): string {
+  const db = getDatabaseType(process.env.DATABASE_URL);
 
   if (db === POSTGRESQL) {
     return `${field} + interval '${minutes} minute'`;
@@ -80,53 +80,6 @@ function getFilterQuery(filters = {}, params = []): string {
   return query.join('\n');
 }
 
-function getFunnelQuery(
-  urls: string[],
-  windowMinutes: number,
-): {
-  levelQuery: string;
-  sumQuery: string;
-  urlFilterQuery: string;
-} {
-  const initParamLength = 3;
-
-  return urls.reduce(
-    (pv, cv, i) => {
-      const levelNumber = i + 1;
-      const start = i > 0 ? ',' : '';
-
-      if (levelNumber >= 2) {
-        pv.levelQuery += `\n
-        , level${levelNumber} AS (
-          select cl.*,
-            l0.created_at level_${levelNumber}_created_at,
-            l0.url_path as level_${levelNumber}_url
-          from level${i} cl
-              left join website_event l0
-                  on cl.session_id = l0.session_id
-                  and l0.created_at between cl.level_${i}_created_at 
-                    and ${getAddMinutesQuery(`cl.level_${i}_created_at`, windowMinutes)}
-                  and l0.referrer_path = $${i + initParamLength}
-                  and l0.url_path = $${levelNumber + initParamLength}
-                  and created_at between $2 and $3
-                  and website_id = $1
-        )`;
-      }
-
-      pv.sumQuery += `\n${start}SUM(CASE WHEN level_${levelNumber}_url is not null THEN 1 ELSE 0 END) AS level${levelNumber}`;
-
-      pv.urlFilterQuery += `\n${start}$${levelNumber + initParamLength} `;
-
-      return pv;
-    },
-    {
-      levelQuery: '',
-      sumQuery: '',
-      urlFilterQuery: '',
-    },
-  );
-}
-
 function parseFilters(
   filters: { [key: string]: any } = {},
   params = [],
@@ -150,10 +103,8 @@ async function rawQuery(sql: string, data: object): Promise<any> {
   if (db !== POSTGRESQL && db !== MYSQL) {
     return Promise.reject(new Error('Unknown database.'));
   }
-
   const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
     const [, name, type] = args;
-
     params.push(data[name]);
 
     return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`;
@@ -168,7 +119,6 @@ export default {
   getDateQuery,
   getTimestampIntervalQuery,
   getFilterQuery,
-  getFunnelQuery,
   parseFilters,
   rawQuery,
 };

queries/analytics/reports/getFunnel.ts

@@ -31,31 +31,74 @@ async function relationalQuery(
   {
     x: string;
     y: number;
+    z: number;
   }[]
 > {
   const { windowMinutes, startDate, endDate, urls } = criteria;
-  const { rawQuery, getFunnelQuery } = prisma;
-  const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes);
+  const { rawQuery, getAddMinutesQuery } = prisma;
+  const { levelQuery, sumQuery } = getFunnelQuery(urls, windowMinutes);
+
+  function getFunnelQuery(
+    urls: string[],
+    windowMinutes: number,
+  ): {
+    levelQuery: string;
+    sumQuery: string;
+  } {
+    return urls.reduce(
+      (pv, cv, i) => {
+        const levelNumber = i + 1;
+        const startSum = i > 0 ? 'union ' : '';
+
+        if (levelNumber >= 2) {
+          pv.levelQuery += `
+          , level${levelNumber} AS (
+            select distinct we.session_id, we.created_at
+            from level${i} l
+            join website_event we
+                on l.session_id = we.session_id
+            where we.created_at between l.created_at 
+                and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}
+                and we.referrer_path = {{${i - 1}}}
+                and we.url_path = {{${i}}}
+                and we.created_at <= {{endDate}}
+                and we.website_id = {{websiteId::uuid}}
+          )`;
+        }
+
+        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
+
+        return pv;
+      },
+      {
+        levelQuery: '',
+        sumQuery: '',
+      },
+    );
+  }
 
   return rawQuery(
-    `WITH level0 AS (
-      select distinct session_id, url_path, referrer_path, created_at
+    `WITH level1 AS (
+      select distinct session_id, created_at
       from website_event
-      where url_path in (${urlFilterQuery})
-          and website_id = {{websiteId::uuid}}
+      where website_id = {{websiteId::uuid}}
       and created_at between {{startDate}} and {{endDate}}
-  ),level1 AS (
-      select distinct session_id, url_path as level_1_url, created_at as level_1_created_at
-      from level0
-      where url_path = $4
-  )${levelQuery}
-  
-  SELECT ${sumQuery}
-  from level${urls.length};
-  `,
-    { websiteId, startDate, endDate, ...urls },
-  ).then((a: { [key: string]: number }) => {
-    return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 }));
+          and url_path = {{0}})
+    ${levelQuery}
+    ${sumQuery}
+    ORDER BY level;`,
+    {
+      websiteId,
+      startDate,
+      endDate,
+      ...urls,
+    },
+  ).then(results => {
+    return urls.map((a, i) => ({
+      x: a,
+      y: results[i]?.count || 0,
+      z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
+    }));
   });
 }
 
@@ -71,42 +114,87 @@ async function clickhouseQuery(
   {
     x: string;
     y: number;
+    z: number;
   }[]
 > {
   const { windowMinutes, startDate, endDate, urls } = criteria;
-  const { rawQuery, getFunnelQuery } = clickhouse;
-  const { columnsQuery, urlParams } = getFunnelQuery(urls);
+  const { rawQuery } = clickhouse;
+  const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getFunnelQuery(urls, windowMinutes);
+
+  function getFunnelQuery(
+    urls: string[],
+    windowMinutes: number,
+  ): {
+    levelQuery: string;
+    sumQuery: string;
+    urlFilterQuery: string;
+    urlParams: { [key: string]: string };
+  } {
+    return urls.reduce(
+      (pv, cv, i) => {
+        const levelNumber = i + 1;
+        const startSum = i > 0 ? 'union all ' : '';
+        const startFilter = i > 0 ? ', ' : '';
+
+        if (levelNumber >= 2) {
+          pv.levelQuery += `\n
+          , level${levelNumber} AS (
+            select distinct y.session_id as session_id,
+                y.url_path as url_path,
+                y.referrer_path as referrer_path,
+                y.created_at as created_at
+            from level${i} x
+            join level0 y
+            on x.session_id = y.session_id
+            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute
+                and y.referrer_path = {url${i - 1}:String}
+                and y.url_path = {url${i}:String}
+          )`;
+        }
+
+        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
+        pv.urlFilterQuery += `${startFilter}{url${i}:String} `;
+        pv.urlParams[`url${i}`] = cv;
+
+        return pv;
+      },
+      {
+        levelQuery: '',
+        sumQuery: '',
+        urlFilterQuery: '',
+        urlParams: {},
+      },
+    );
+  }
 
   return rawQuery<{ level: number; count: number }[]>(
     `
-    SELECT level,
-        count(*) AS count
-    FROM (
-    SELECT session_id,
-            windowFunnel({window:UInt32}, 'strict_increase')
-            (
-                created_at
-                ${columnsQuery}
-            ) AS level
-        FROM website_event
-        WHERE website_id = {websiteId:UUID}
-        AND created_at BETWEEN {startDate:DateTime} AND {endDate:DateTime}       
-        GROUP BY 1
-        )
-    GROUP BY level
-    ORDER BY level ASC;
-    `,
+    WITH level0 AS (
+      select distinct session_id, url_path, referrer_path, created_at
+      from umami.website_event
+      where url_path in (${urlFilterQuery})
+          and website_id = {websiteId:UUID}
+          and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+    ), level1 AS (
+      select *
+      from level0
+      where url_path = {url0:String})
+    ${levelQuery}
+    select *
+    from (
+    ${sumQuery} 
+    ) ORDER BY level;`,
     {
       websiteId,
       startDate,
       endDate,
-      window: windowMinutes * 60,
       ...urlParams,
     },
   ).then(results => {
     return urls.map((a, i) => ({
       x: a,
-      y: results[i + 1]?.count || 0,
+      y: results[i]?.count || 0,
+      z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
     }));
   });
 }