ch attribution report, schema changes, and migration

db/clickhouse/migrations/05_add_utm_clid.sql

@@ -28,8 +28,13 @@ CREATE TABLE umami.website_event_new
     referrer_query String,
     referrer_domain String,
     page_title String,
+    --clickIDs
     gclid String,
     fbclid String,
+    msclkid String,
+    ttclid String,
+    li_fat_id String,
+    twclid String,
     --events
     event_type UInt32,
     event_name String,
@@ -71,6 +76,10 @@ CREATE TABLE umami.website_event_stats_hourly_new
     page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
     gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
     fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
+    twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
     event_type UInt32,
     event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
     views SimpleAggregateFunction(sum, UInt64),
@@ -119,6 +128,10 @@ SELECT
     page_title,
     gclid,
     fbclid,
+    msclkid,
+    ttclid,
+    li_fat_id,
+    twclid,
     event_type,
     event_name,
     views,
@@ -152,6 +165,10 @@ FROM (SELECT
     arrayFilter(x -> x != '', groupArray(page_title)) page_title,
     arrayFilter(x -> x != '', groupArray(gclid)) gclid,
     arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+    arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+    arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+    arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+    arrayFilter(x -> x != '', groupArray(twclid)) twclid,
     event_type,
     if(event_type = 2, groupArray(event_name), []) event_name,
     sumIf(1, event_type = 1) views,
@@ -201,6 +218,10 @@ SELECT website_id, session_id, visit_id, event_id, hostname, browser, os, device
     page_title,
     extract(url_query, 'gclid=([^&]*)') gclid,
     extract(url_query, 'fbclid=([^&]*)') fbclid,
+    extract(url_query, 'msclkid=([^&]*)') msclkid,
+    extract(url_query, 'ttclid=([^&]*)') ttclid,
+    extract(url_query, 'li_fat_id=([^&]*)') li_fat_id,
+    extract(url_query, 'twclid=([^&]*)') twclid,
     event_type, event_name, tag, created_at, job_id
 FROM umami.website_event
 
@@ -246,6 +267,10 @@ SELECT
     page_title,
     gclid,
     fbclid,
+    msclkid,
+    ttclid,
+    li_fat_id,
+    twclid,
     event_type,
     event_name,
     views,
@@ -279,6 +304,10 @@ FROM (SELECT
     arrayFilter(x -> x != '', groupArray(page_title)) page_title,
     arrayFilter(x -> x != '', groupArray(gclid)) gclid,
     arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+    arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+    arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+    arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+    arrayFilter(x -> x != '', groupArray(twclid)) twclid,
     event_type,
     if(event_type = 2, groupArray(event_name), []) event_name,
     sumIf(1, event_type = 1) views,

db/clickhouse/schema.sql

@@ -28,8 +28,13 @@ CREATE TABLE umami.website_event
     referrer_query String,
     referrer_domain String,
     page_title String,
+    --clickIDs
     gclid String,
     fbclid String,
+    msclkid String,
+    ttclid String,
+    li_fat_id String,
+    twclid String,
     --events
     event_type UInt32,
     event_name String,
@@ -106,6 +111,10 @@ CREATE TABLE umami.website_event_stats_hourly
     page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
     gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
     fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+    li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
+    twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
     event_type UInt32,
     event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
     views SimpleAggregateFunction(sum, UInt64),
@@ -154,6 +163,10 @@ SELECT
     page_title,
     gclid,
     fbclid,
+    msclkid,
+    ttclid,
+    li_fat_id,
+    twclid,
     event_type,
     event_name,
     views,
@@ -187,6 +200,10 @@ FROM (SELECT
     arrayFilter(x -> x != '', groupArray(page_title)) page_title,
     arrayFilter(x -> x != '', groupArray(gclid)) gclid,
     arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+    arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+    arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+    arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+    arrayFilter(x -> x != '', groupArray(twclid)) twclid,
     event_type,
     if(event_type = 2, groupArray(event_name), []) event_name,
     sumIf(1, event_type = 1) views,

src/app/(main)/reports/attribution/AttributionParameters.tsx

@@ -13,6 +13,7 @@ import {
   Popup,
   PopupTrigger,
   SubmitButton,
+  Toggle,
 } from 'react-basics';
 import BaseParameters from '../[reportId]/BaseParameters';
 import ParameterList from '../[reportId]/ParameterList';
@@ -21,6 +22,7 @@ import { ReportContext } from '../[reportId]/Report';
 import FunnelStepAddForm from '../funnel/FunnelStepAddForm';
 import styles from './AttributionParameters.module.css';
 import AttributionStepAddForm from './AttributionStepAddForm';
+import useRevenueValues from '@/components/hooks/queries/useRevenueValues';
 
 export function AttributionParameters() {
   const { report, runReport, updateReport, isRunning } = useContext(ReportContext);
@@ -29,14 +31,32 @@ export function AttributionParameters() {
   const { websiteId, dateRange, steps } = parameters || {};
   const queryEnabled = websiteId && dateRange && steps.length > 0;
   const [model, setModel] = useState('');
+  const [revenueMode, setRevenueMode] = useState(false);
+
+  const { data: currencyValues = [] } = useRevenueValues(
+    websiteId,
+    dateRange?.startDate,
+    dateRange?.endDate,
+  );
 
   const handleSubmit = (data: any, e: any) => {
+    if (revenueMode === false) {
+      delete data.currency;
+    }
+
     e.stopPropagation();
     e.preventDefault();
     runReport(data);
   };
 
+  const handleCheck = () => {
+    setRevenueMode(!revenueMode);
+  };
+
   const handleAddStep = (step: { type: string; value: string }) => {
+    if (step.type === 'url') {
+      setRevenueMode(false);
+    }
     updateReport({ parameters: { steps: parameters.steps.concat(step) } });
   };
 
@@ -45,6 +65,9 @@ export function AttributionParameters() {
     index: number,
     step: { type: string; value: string },
   ) => {
+    if (step.type === 'url') {
+      setRevenueMode(false);
+    }
     const steps = [...parameters.steps];
     steps[index] = step;
     updateReport({ parameters: { steps } });
@@ -135,6 +158,24 @@ export function AttributionParameters() {
           })}
         </ParameterList>
       </FormRow>
+      <FormRow>
+        <Toggle
+          checked={revenueMode}
+          onChecked={handleCheck}
+          disabled={currencyValues.length === 0 || steps[0]?.type === 'url'}
+        >
+          <b>Revenue Mode</b>
+        </Toggle>
+      </FormRow>
+      {revenueMode && (
+        <FormRow label={formatMessage(labels.currency)}>
+          <FormInput name="currency" rules={{ required: formatMessage(labels.required) }}>
+            <Dropdown items={currencyValues.map(item => item.currency)}>
+              {item => <Item key={item}>{item}</Item>}
+            </Dropdown>
+          </FormInput>
+        </FormRow>
+      )}
       <FormButtons>
         <SubmitButton variant="primary" disabled={!queryEnabled} isLoading={isRunning}>
           {formatMessage(labels.runQuery)}

src/app/(main)/reports/attribution/AttributionView.module.css

@@ -4,8 +4,17 @@
   margin-bottom: 40px;
 }
 
-.row {
+.title {
-  display: flex;
+  font-size: 24px;
-  align-items: center;
+  line-height: 36px;
-  gap: 10px;
+  font-weight: 700;
+}
+
+.row {
+  display: grid;
+  grid-template-columns: 50% 50%;
+  gap: 20px;
+  border-top: 1px solid var(--base300);
+  padding-top: 30px;
+  margin-bottom: 30px;
 }

src/app/(main)/reports/attribution/AttributionView.tsx

@@ -1,19 +1,13 @@
-import classNames from 'classnames';
-import { colord } from 'colord';
-import BarChart from '@/components/charts/BarChart';
 import PieChart from '@/components/charts/PieChart';
-import TypeIcon from '@/components/common/TypeIcon';
+import { useMessages } from '@/components/hooks';
-import { useCountryNames, useLocale, useMessages } from '@/components/hooks';
+import { Grid, GridRow } from '@/components/layout/Grid';
-import { GridRow } from '@/components/layout/Grid';
 import ListTable from '@/components/metrics/ListTable';
 import MetricCard from '@/components/metrics/MetricCard';
 import MetricsBar from '@/components/metrics/MetricsBar';
-import { renderDateLabels } from '@/lib/charts';
 import { CHART_COLORS } from '@/lib/constants';
-import { formatLongCurrency, formatLongNumber } from '@/lib/format';
+import { formatLongNumber } from '@/lib/format';
-import { useCallback, useContext, useMemo } from 'react';
+import { useContext } from 'react';
 import { ReportContext } from '../[reportId]/Report';
-import AttributionTable from './AttributionTable';
 import styles from './AttributionView.module.css';
 
 export interface AttributionViewProps {
@@ -22,134 +16,118 @@ export interface AttributionViewProps {
 
 export function AttributionView({ isLoading }: AttributionViewProps) {
   const { formatMessage, labels } = useMessages();
-  const { locale } = useLocale();
-  const { countryNames } = useCountryNames(locale);
   const { report } = useContext(ReportContext);
   const {
     data,
-    parameters: { dateRange, currency },
+    parameters: { currency },
   } = report || {};
-  const showTable = data?.table.length > 1;
+  const ATTRIBUTION_PARAMS = [
+    { value: 'referrer', label: formatMessage(labels.referrers) },
+    { value: 'paidAds', label: formatMessage(labels.paidAds) },
+  ];
 
-  const renderCountryName = useCallback(
+  if (!data) {
-    ({ x: code }) => (
+    return null;
-      <span className={classNames(locale, styles.row)}>
+  }
-        <TypeIcon type="country" value={code?.toLowerCase()} />
-        {countryNames[code]}
-      </span>
-    ),
-    [countryNames, locale],
-  );
 
-  const chartData = useMemo(() => {
+  const { pageviews, visitors, visits } = data.total;
-    if (!data) return [];
 
-    const map = (data.chart as any[]).reduce((obj, { x, t, y }) => {
+  const metrics = data
-      if (!obj[x]) {
+    ? [
-        obj[x] = [];
+        {
-      }
+          value: pageviews,
+          label: formatMessage(labels.views),
+          formatValue: formatLongNumber,
+        },
+        {
+          value: visits,
+          label: formatMessage(labels.visits),
+          formatValue: formatLongNumber,
+        },
+        {
+          value: visitors,
+          label: formatMessage(labels.visitors),
+          formatValue: formatLongNumber,
+        },
+      ]
+    : [];
 
-      obj[x].push({ x: t, y });
+  function UTMTable(UTMTableProps: { data: any; title: string; utm: string }) {
+    const { data, title, utm } = UTMTableProps;
+    const total = data[utm].reduce((sum, { value }) => {
+      return +sum + +value;
+    }, 0);
 
-      return obj;
+    return (
-    }, {});
+      <ListTable
-
+        title={title}
-    return {
+        metric={formatMessage(currency ? labels.revenue : labels.visitors)}
-      datasets: Object.keys(map).map((key, index) => {
+        currency={currency}
-        const color = colord(CHART_COLORS[index % CHART_COLORS.length]);
+        data={data[utm].map(({ name, value }) => ({
-        return {
+          x: name,
-          label: key,
+          y: Number(value),
-          data: map[key],
+          z: (value / total) * 100,
-          lineTension: 0,
+        }))}
-          backgroundColor: color.alpha(0.6).toRgbString(),
+      />
-          borderColor: color.alpha(0.7).toRgbString(),
+    );
-          borderWidth: 1,
+  }
-        };
-      }),
-    };
-  }, [data]);
-
-  const countryData = useMemo(() => {
-    if (!data) return [];
-
-    const labels = data.country.map(({ name }) => name);
-    const datasets = [
-      {
-        data: data.country.map(({ value }) => value),
-        backgroundColor: CHART_COLORS,
-        borderWidth: 0,
-      },
-    ];
-
-    return { labels, datasets };
-  }, [data]);
-
-  const metricData = useMemo(() => {
-    if (!data) return [];
-
-    const { sum, count, unique_count } = data.total;
-
-    return [
-      {
-        value: sum,
-        label: formatMessage(labels.total),
-        formatValue: n => formatLongCurrency(n, currency),
-      },
-      {
-        value: count ? sum / count : 0,
-        label: formatMessage(labels.average),
-        formatValue: n => formatLongCurrency(n, currency),
-      },
-      {
-        value: count,
-        label: formatMessage(labels.transactions),
-        formatValue: formatLongNumber,
-      },
-      {
-        value: unique_count,
-        label: formatMessage(labels.uniqueCustomers),
-        formatValue: formatLongNumber,
-      },
-    ] as any;
-  }, [data, locale]);
 
   return (
-    <>
+    <div className={styles.container}>
-      <div className={styles.container}>
+      <MetricsBar isFetched={data}>
-        <MetricsBar isFetched={data}>
+        {metrics?.map(({ label, value, formatValue }) => {
-          {metricData?.map(({ label, value, formatValue }) => {
+          return <MetricCard key={label} value={value} label={label} formatValue={formatValue} />;
-            return <MetricCard key={label} value={value} label={label} formatValue={formatValue} />;
+        })}
-          })}
+      </MetricsBar>
-        </MetricsBar>
+      {ATTRIBUTION_PARAMS.map(({ value, label }) => {
-        {data && (
+        const items = data[value];
-          <>
+        const total = items.reduce((sum, { value }) => {
-            <BarChart
+          return +sum + +value;
-              minDate={dateRange?.startDate}
+        }, 0);
-              maxDate={dateRange?.endDate}
+
-              data={chartData}
+        const chartData = {
-              unit={dateRange?.unit}
+          labels: items.map(({ name }) => name),
-              stacked={true}
+          datasets: [
-              currency={currency}
+            {
-              renderXLabel={renderDateLabels(dateRange?.unit, locale)}
+              data: items.map(({ value }) => value),
-              isLoading={isLoading}
+              backgroundColor: CHART_COLORS,
-            />
+              borderWidth: 0,
-            <GridRow columns="two">
+            },
+          ],
+        };
+
+        return (
+          <div key={value} className={styles.row}>
+            <div>
+              <div className={styles.title}>{label}</div>
               <ListTable
-                metric={formatMessage(labels.country)}
+                metric={formatMessage(currency ? labels.revenue : labels.visitors)}
-                data={data?.country.map(({ name, value }) => ({
+                currency={currency}
+                data={items.map(({ name, value }) => ({
                   x: name,
                   y: Number(value),
-                  z: (value / data?.total.sum) * 100,
+                  z: (value / total) * 100,
                 }))}
-                renderLabel={renderCountryName}
               />
-              <PieChart type="doughnut" data={countryData} />
+            </div>
-            </GridRow>
+            <div>
-          </>
+              <PieChart type="doughnut" data={chartData} isLoading={isLoading} />
-        )}
+            </div>
-        {showTable && <AttributionTable />}
+          </div>
-      </div>
+        );
-    </>
+      })}
+      <Grid>
+        <GridRow columns="two">
+          <UTMTable data={data} title={formatMessage(labels.sources)} utm={'utm_source'} />
+          <UTMTable data={data} title={formatMessage(labels.medium)} utm={'utm_medium'} />
+        </GridRow>
+        <GridRow columns="three">
+          <UTMTable data={data} title={formatMessage(labels.campaigns)} utm={'utm_campaign'} />
+          <UTMTable data={data} title={formatMessage(labels.content)} utm={'utm_content'} />
+          <UTMTable data={data} title={formatMessage(labels.terms)} utm={'utm_term'} />
+        </GridRow>
+      </Grid>
+    </div>
   );
 }
 

src/app/api/reports/attribution/route.ts

@@ -1,14 +1,14 @@
-import { z } from 'zod';
 import { canViewWebsite } from '@/lib/auth';
-import { unauthorized, json } from '@/lib/response';
 import { parseRequest } from '@/lib/request';
-import { getFunnel } from '@/queries';
+import { json, unauthorized } from '@/lib/response';
 import { reportParms } from '@/lib/schema';
+import { getAttribution } from '@/queries/sql/reports/getAttribution';
+import { z } from 'zod';
 
 export async function POST(request: Request) {
   const schema = z.object({
     ...reportParms,
-    window: z.coerce.number().positive(),
+    model: z.string().regex(/firstClick|lastClick/i),
     steps: z
       .array(
         z.object({
@@ -16,7 +16,8 @@ export async function POST(request: Request) {
           value: z.string(),
         }),
       )
-      .min(2),
+      .min(1),
+    currency: z.string().optional(),
   });
 
   const { auth, body, error } = await parseRequest(request, schema);
@@ -27,8 +28,9 @@ export async function POST(request: Request) {
 
   const {
     websiteId,
+    model,
     steps,
-    window,
+    currency,
     dateRange: { startDate, endDate },
   } = body;
 
@@ -36,11 +38,12 @@ export async function POST(request: Request) {
     return unauthorized();
   }
 
-  const data = await getFunnel(websiteId, {
+  const data = await getAttribution(websiteId, {
     startDate: new Date(startDate),
     endDate: new Date(endDate),
+    model: model,
     steps,
-    windowMinutes: +window,
+    currency,
   });
 
   return json(data);

src/components/messages.ts

@@ -288,6 +288,11 @@ export const labels = defineMessages({
   firstSeen: { id: 'label.first-seen', defaultMessage: 'First seen' },
   properties: { id: 'label.properties', defaultMessage: 'Properties' },
   channels: { id: 'label.channels', defaultMessage: 'Channels' },
+  sources: { id: 'label.sources', defaultMessage: 'Sources' },
+  medium: { id: 'label.medium', defaultMessage: 'Medium' },
+  campaigns: { id: 'label.campaigns', defaultMessage: 'Campaigns' },
+  content: { id: 'label.content', defaultMessage: 'Content' },
+  terms: { id: 'label.terms', defaultMessage: 'Terms' },
   direct: { id: 'label.direct', defaultMessage: 'Direct' },
   referral: { id: 'label.referral', defaultMessage: 'Referral' },
   affiliate: { id: 'label.affiliate', defaultMessage: 'Affiliate' },

src/components/metrics/ListTable.tsx

@@ -1,11 +1,11 @@
-import { FixedSizeList } from 'react-window';
-import { useSpring, animated, config } from '@react-spring/web';
-import classNames from 'classnames';
 import Empty from '@/components/common/Empty';
-import { formatLongNumber } from '@/lib/format';
 import { useMessages } from '@/components/hooks';
-import styles from './ListTable.module.css';
+import { formatLongCurrency, formatLongNumber } from '@/lib/format';
+import { animated, config, useSpring } from '@react-spring/web';
+import classNames from 'classnames';
 import { ReactNode } from 'react';
+import { FixedSizeList } from 'react-window';
+import styles from './ListTable.module.css';
 
 const ITEM_SIZE = 30;
 
@@ -20,6 +20,7 @@ export interface ListTableProps {
   virtualize?: boolean;
   showPercentage?: boolean;
   itemCount?: number;
+  currency?: string;
 }
 
 export function ListTable({
@@ -33,6 +34,7 @@ export function ListTable({
   virtualize = false,
   showPercentage = true,
   itemCount = 10,
+  currency,
 }: ListTableProps) {
   const { formatMessage, labels } = useMessages();
 
@@ -48,6 +50,7 @@ export function ListTable({
         animate={animate && !virtualize}
         showPercentage={showPercentage}
         change={renderChange ? renderChange(row, index) : null}
+        currency={currency}
       />
     );
   };
@@ -81,7 +84,15 @@ export function ListTable({
   );
 }
 
-const AnimatedRow = ({ label, value = 0, percent, change, animate, showPercentage = true }) => {
+const AnimatedRow = ({
+  label,
+  value = 0,
+  percent,
+  change,
+  animate,
+  showPercentage = true,
+  currency,
+}) => {
   const props = useSpring({
     width: percent,
     y: value,
@@ -95,7 +106,9 @@ const AnimatedRow = ({ label, value = 0, percent, change, animate, showPercentag
       <div className={styles.value}>
         {change}
         <animated.div className={styles.value} title={props?.y as any}>
-          {props.y?.to(formatLongNumber)}
+          {currency
+            ? props.y?.to(n => formatLongCurrency(n, currency))
+            : props.y?.to(formatLongNumber)}
         </animated.div>
       </div>
       {showPercentage && (

src/queries/sql/reports/getAttribution.ts

@@ -1,34 +1,17 @@
 import clickhouse from '@/lib/clickhouse';
+import { EVENT_TYPE } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 
-const formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {
+export async function getAttribution(
-  return steps.map((step: { type: string; value: string }, i: number) => {
-    const visitors = Number(results[i]?.count) || 0;
-    const previous = Number(results[i - 1]?.count) || 0;
-    const dropped = previous > 0 ? previous - visitors : 0;
-    const dropoff = 1 - visitors / previous;
-    const remaining = visitors / Number(results[0].count);
-
-    return {
-      ...step,
-      visitors,
-      previous,
-      dropped,
-      dropoff,
-      remaining,
-    };
-  });
-};
-
-export async function getFunnel(
   ...args: [
     websiteId: string,
     criteria: {
-      windowMinutes: number;
       startDate: Date;
       endDate: Date;
+      model: string;
       steps: { type: string; value: string }[];
+      currency: string;
     },
   ]
 ) {
@@ -41,210 +24,452 @@ export async function getFunnel(
 async function relationalQuery(
   websiteId: string,
   criteria: {
-    windowMinutes: number;
     startDate: Date;
     endDate: Date;
+    model: string;
     steps: { type: string; value: string }[];
+    currency: string;
   },
-): Promise<
+): Promise<{
-  {
+  referrer: { name: string; value: number }[];
-    value: string;
+  paidAds: { name: string; value: number }[];
-    visitors: number;
+  utm_source: { name: string; value: number }[];
-    dropoff: number;
+  utm_medium: { name: string; value: number }[];
-  }[]
+  utm_campaign: { name: string; value: number }[];
-> {
+  utm_content: { name: string; value: number }[];
-  const { windowMinutes, startDate, endDate, steps } = criteria;
+  utm_term: { name: string; value: number }[];
-  const { rawQuery, getAddIntervalQuery } = prisma;
+  total: { pageviews: number; visitors: number; visits: number };
-  const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, windowMinutes);
+}> {
+  const { startDate, endDate, model, steps, currency } = criteria;
+  const { rawQuery } = prisma;
+  const conversionStep = steps[0].value;
+  const eventType = steps[0].type === 'url' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
+  const column = steps[0].type === 'url' ? 'url_path' : 'event_name';
+  //const db = getDatabaseType();
+  //const like = db === POSTGRESQL ? 'ilike' : 'like';
 
-  function getFunnelQuery(
+  function getUTMQuery(utmColumn: string) {
-    steps: { type: string; value: string }[],
+    return `
-    windowMinutes: number,
+    select 
-  ): {
+        we.${utmColumn} name,
-    levelOneQuery: string;
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
-    levelQuery: string;
+    from events e
-    sumQuery: string;
+    join model m
-    params: string[];
+    on m.session_id = e.session_id
-  } {
+    join website_event we
-    return steps.reduce(
+    on we.created_at = m.created_at
-      (pv, cv, i) => {
+        and we.session_id = m.session_id
-        const levelNumber = i + 1;
+    ${currency ? '' : `where we.${utmColumn} != ''`}  
-        const startSum = i > 0 ? 'union ' : '';
+    group by 1
-        const isURL = cv.type === 'url';
+    order by name desc
-        const column = isURL ? 'url_path' : 'event_name';
+    limit 20`;
-
-        let operator = '=';
-        let paramValue = cv.value;
-
-        if (cv.value.startsWith('*') || cv.value.endsWith('*')) {
-          operator = 'like';
-          paramValue = cv.value.replace(/^\*|\*$/g, '%');
-        }
-
-        if (levelNumber === 1) {
-          pv.levelOneQuery = `
-          WITH level1 AS (
-            select distinct session_id, created_at
-            from website_event
-            where website_id = {{websiteId::uuid}}
-              and created_at between {{startDate}} and {{endDate}}
-              and ${column} ${operator} {{${i}}}
-          )`;
-        } else {
-          pv.levelQuery += `
-          , level${levelNumber} AS (
-            select distinct we.session_id, we.created_at
-            from level${i} l
-            join website_event we
-                on l.session_id = we.session_id
-            where we.website_id = {{websiteId::uuid}}
-                and we.created_at between l.created_at and ${getAddIntervalQuery(
-                  `l.created_at `,
-                  `${windowMinutes} minute`,
-                )}
-                and we.${column} ${operator} {{${i}}}
-                and we.created_at <= {{endDate}}
-          )`;
-        }
-
-        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
-        pv.params.push(paramValue);
-
-        return pv;
-      },
-      {
-        levelOneQuery: '',
-        levelQuery: '',
-        sumQuery: '',
-        params: [],
-      },
-    );
   }
 
-  return rawQuery(
+  const eventQuery = `WITH events AS (
+        select distinct
+            session_id,
+            max(created_at) max_dt
+        from website_event
+        where website_id = {websiteId:UUID}
+          and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+          and ${column} = {conversionStep:String}
+          and event_type = {eventType:UInt32}
+        group by 1),`;
+
+  const revenueEventQuery = `WITH events AS (
+          select
+              ed.session_id,
+              max(ed.created_at) max_dt,
+              sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as value
+          from event_data ed
+          join (select event_id
+                from event_data
+                where website_id = {websiteId:UUID}
+                  and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+                  and positionCaseInsensitive(data_key, 'currency') > 0
+                  and string_value = {currency:String}) c
+            on c.event_id = ed.event_id
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and ${column} = {conversionStep:String} 
+            and positionCaseInsensitive(ed.data_key, 'revenue') > 0
+          group by 1),`;
+
+  function getModelQuery(model: string) {
+    return model === 'firstClick'
+      ? `\n 
+    model AS (select e.session_id,
+        min(we.created_at) created_at
+    from events e
+    join website_event we
+    on we.session_id = e.session_id
+    group by e.session_id)`
+      : `\n 
+    model AS (select e.session_id,
+        max(we.created_at) created_at
+    from events e
+    join website_event we
+    on we.session_id = e.session_id
+    where we.created_at < e.max_dt
+    group by e.session_id)`;
+  }
+
+  const referrerRes = await rawQuery(
     `
-    ${levelOneQuery}
+    ${currency ? revenueEventQuery : eventQuery}
-    ${levelQuery}
+    ${getModelQuery(model)}
-    ${sumQuery}
+    select we.referrer_domain name,
-    ORDER BY level;
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
+    from events e
+    join model m
+    on m.session_id = e.session_id
+    join website_event we
+    on we.created_at = m.created_at
+        and we.session_id = m.session_id
+    ${
+      currency
+        ? ''
+        : `where referrer_domain != hostname
+      and we.referrer_domain != ''`
+    }  
+    group by 1
+    order by 2 desc
+    limit 20
     `,
-    {
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
-      websiteId,
+  );
-      startDate,
+
-      endDate,
+  const paidAdsres = await rawQuery(
-      ...params,
+    `
-    },
+    ${currency ? revenueEventQuery : eventQuery}
-  ).then(formatResults(steps));
+    ${getModelQuery(model)}
+    select multiIf(gclid != '', 'Google', fbclid != '', 'Facebook', '') name,
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
+    from events e
+    join model m
+    on m.session_id = e.session_id
+    join website_event we
+    on we.created_at = m.created_at
+        and we.session_id = m.session_id
+    ${currency ? '' : `WHERE name != ''`}
+    group by 1
+    order by 2 desc
+    limit 20
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const sourceRes = await rawQuery(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_source')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const mediumRes = await rawQuery(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_medium')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const campaignRes = await rawQuery(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_campaign')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const contentRes = await rawQuery(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_content')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const termRes = await rawQuery(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_term')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const totalRes = await rawQuery(
+    `
+    select 
+        count(*) as "pageviews",
+        uniqExact(session_id) as "visitors",
+        uniqExact(visit_id) as "visits"
+    from website_event
+    where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and ${column} = {conversionStep:String}
+        and event_type = {eventType:UInt32}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  ).then(result => result?.[0]);
+
+  return {
+    referrer: referrerRes,
+    paidAds: paidAdsres,
+    utm_source: sourceRes,
+    utm_medium: mediumRes,
+    utm_campaign: campaignRes,
+    utm_content: contentRes,
+    utm_term: termRes,
+    total: totalRes,
+  };
 }
 
 async function clickhouseQuery(
   websiteId: string,
   criteria: {
-    windowMinutes: number;
     startDate: Date;
     endDate: Date;
+    model: string;
     steps: { type: string; value: string }[];
+    currency: string;
   },
-): Promise<
+): Promise<{
-  {
+  referrer: { name: string; value: number }[];
-    value: string;
+  paidAds: { name: string; value: number }[];
-    visitors: number;
+  utm_source: { name: string; value: number }[];
-    dropoff: number;
+  utm_medium: { name: string; value: number }[];
-  }[]
+  utm_campaign: { name: string; value: number }[];
-> {
+  utm_content: { name: string; value: number }[];
-  const { windowMinutes, startDate, endDate, steps } = criteria;
+  utm_term: { name: string; value: number }[];
+  total: { pageviews: number; visitors: number; visits: number };
+}> {
+  const { startDate, endDate, model, steps, currency } = criteria;
   const { rawQuery } = clickhouse;
-  const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(
+  const conversionStep = steps[0].value;
-    steps,
+  const eventType = steps[0].type === 'url' ? EVENT_TYPE.pageView : EVENT_TYPE.customEvent;
-    windowMinutes,
+  const column = steps[0].type === 'url' ? 'url_path' : 'event_name';
-  );
 
-  function getFunnelQuery(
+  function getUTMQuery(utmColumn: string) {
-    steps: { type: string; value: string }[],
+    return `
-    windowMinutes: number,
+    select 
-  ): {
+        we.${utmColumn} name,
-    levelOneQuery: string;
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
-    levelQuery: string;
+    from events e
-    sumQuery: string;
+    join model m
-    stepFilterQuery: string;
+    on m.session_id = e.session_id
-    params: { [key: string]: string };
+    join website_event we
-  } {
+    on we.created_at = m.created_at
-    return steps.reduce(
+        and we.session_id = m.session_id
-      (pv, cv, i) => {
+    ${currency ? '' : `where we.${utmColumn} != ''`}  
-        const levelNumber = i + 1;
+    group by 1
-        const startSum = i > 0 ? 'union all ' : '';
+    order by name desc
-        const startFilter = i > 0 ? 'or' : '';
+    limit 20`;
-        const isURL = cv.type === 'url';
-        const column = isURL ? 'url_path' : 'event_name';
-
-        let operator = '=';
-        let paramValue = cv.value;
-
-        if (cv.value.startsWith('*') || cv.value.endsWith('*')) {
-          operator = 'like';
-          paramValue = cv.value.replace(/^\*|\*$/g, '%');
-        }
-
-        if (levelNumber === 1) {
-          pv.levelOneQuery = `\n
-          level1 AS (
-            select *
-            from level0
-            where ${column} ${operator} {param${i}:String}
-          )`;
-        } else {
-          pv.levelQuery += `\n
-          , level${levelNumber} AS (
-            select distinct y.session_id as session_id,
-                y.url_path as url_path,
-                y.referrer_path as referrer_path,
-                y.event_name,
-                y.created_at as created_at
-            from level${i} x
-            join level0 y
-            on x.session_id = y.session_id
-            where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute
-                and y.${column} ${operator} {param${i}:String}
-          )`;
-        }
-
-        pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
-        pv.stepFilterQuery += `${startFilter} ${column} ${operator} {param${i}:String} `;
-        pv.params[`param${i}`] = paramValue;
-
-        return pv;
-      },
-      {
-        levelOneQuery: '',
-        levelQuery: '',
-        sumQuery: '',
-        stepFilterQuery: '',
-        params: {},
-      },
-    );
   }
 
-  return rawQuery(
+  const eventQuery = `WITH events AS (
-    `
+        select distinct
-    WITH level0 AS (
+            session_id,
-      select distinct session_id, url_path, referrer_path, event_name, created_at
+            max(created_at) max_dt
-      from umami.website_event
+        from website_event
-      where (${stepFilterQuery})
+        where website_id = {websiteId:UUID}
-        and website_id = {websiteId:UUID}
+          and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+          and ${column} = {conversionStep:String}
-    ),
+          and event_type = {eventType:UInt32}
-    ${levelOneQuery}
+        group by 1),`;
-    ${levelQuery}
+
-    select *
+  const revenueEventQuery = `WITH events AS (
-    from (
+          select
-      ${sumQuery} 
+              ed.session_id,
-    ) ORDER BY level;
+              max(ed.created_at) max_dt,
-    `,
+              sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as value
+          from event_data ed
+          join (select event_id
+                from event_data
+                where website_id = {websiteId:UUID}
+                  and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+                  and positionCaseInsensitive(data_key, 'currency') > 0
+                  and string_value = {currency:String}) c
+            on c.event_id = ed.event_id
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and ${column} = {conversionStep:String} 
+            and positionCaseInsensitive(ed.data_key, 'revenue') > 0
+          group by 1),`;
+
+  function getModelQuery(model: string) {
+    return model === 'firstClick'
+      ? `\n 
+    model AS (select e.session_id,
+        min(we.created_at) created_at
+    from events e
+    join website_event we
+    on we.session_id = e.session_id
+    group by e.session_id)`
+      : `\n 
+    model AS (select e.session_id,
+        max(we.created_at) created_at
+    from events e
+    join website_event we
+    on we.session_id = e.session_id
+    where we.created_at < e.max_dt
+    group by e.session_id)`;
+  }
+
+  const referrerRes = await rawQuery<
     {
-      websiteId,
+      name: string;
-      startDate,
+      value: number;
-      endDate,
+    }[]
-      ...params,
+  >(
-    },
+    `
-  ).then(formatResults(steps));
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    select we.referrer_domain name,
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
+    from events e
+    join model m
+    on m.session_id = e.session_id
+    join website_event we
+    on we.created_at = m.created_at
+        and we.session_id = m.session_id
+    ${
+      currency
+        ? ''
+        : `where referrer_domain != hostname
+      and we.referrer_domain != ''`
+    }  
+    group by 1
+    order by 2 desc
+    limit 20
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const paidAdsres = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    select multiIf(gclid != '', 'Google', fbclid != '', 'Facebook', '') name,
+        ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
+    from events e
+    join model m
+    on m.session_id = e.session_id
+    join website_event we
+    on we.created_at = m.created_at
+        and we.session_id = m.session_id
+    ${currency ? '' : `WHERE name != ''`}
+    group by 1
+    order by 2 desc
+    limit 20
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const sourceRes = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_source')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const mediumRes = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_medium')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const campaignRes = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_campaign')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const contentRes = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_content')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const termRes = await rawQuery<
+    {
+      name: string;
+      value: number;
+    }[]
+  >(
+    `
+    ${currency ? revenueEventQuery : eventQuery}
+    ${getModelQuery(model)}
+    ${getUTMQuery('utm_term')}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  );
+
+  const totalRes = await rawQuery<{ pageviews: number; visitors: number; visits: number }>(
+    `
+    select 
+        count(*) as "pageviews",
+        uniqExact(session_id) as "visitors",
+        uniqExact(visit_id) as "visits"
+    from website_event
+    where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and ${column} = {conversionStep:String}
+        and event_type = {eventType:UInt32}
+    `,
+    { websiteId, startDate, endDate, conversionStep, eventType, currency },
+  ).then(result => result?.[0]);
+
+  return {
+    referrer: referrerRes,
+    paidAds: paidAdsres,
+    utm_source: sourceRes,
+    utm_medium: mediumRes,
+    utm_campaign: campaignRes,
+    utm_content: contentRes,
+    utm_term: termRes,
+    total: totalRes,
+  };
 }

src/queries/sql/reports/getRevenue.ts

@@ -56,7 +56,9 @@ async function relationalQuery(
     on we.event_id = ed.website_event_id
     join (select website_event_id
           from event_data
-          where data_key ${like} '%currency%'
+          where website_id = {{websiteId::uuid}}
+            and created_at between {{startDate}} and {{endDate}}
+            and data_key ${like} '%currency%'
             and string_value = {{currency}}) currency
     on currency.website_event_id = ed.website_event_id
     where ed.website_id = {{websiteId::uuid}}
@@ -80,7 +82,9 @@ async function relationalQuery(
     on s.session_id = we.session_id
     join (select website_event_id
           from event_data
-          where data_key ${like} '%currency%'
+          where website_id = {{websiteId::uuid}}
+            and created_at between {{startDate}} and {{endDate}}
+            and data_key ${like} '%currency%'
             and string_value = {{currency}}) currency
     on currency.website_event_id = ed.website_event_id
     where ed.website_id = {{websiteId::uuid}}
@@ -102,7 +106,9 @@ async function relationalQuery(
     on we.event_id = ed.website_event_id
     join (select website_event_id
           from event_data
-          where data_key ${like} '%currency%'
+          where website_id = {{websiteId::uuid}}
+            and created_at between {{startDate}} and {{endDate}}
+            and data_key ${like} '%currency%'
             and string_value = {{currency}}) currency
       on currency.website_event_id = ed.website_event_id
     where ed.website_id = {{websiteId::uuid}}
@@ -124,7 +130,9 @@ async function relationalQuery(
       on we.event_id = ed.website_event_id
     join (select website_event_id, string_value as currency
           from event_data
-          where data_key ${like} '%currency%') c
+          where website_id = {{websiteId::uuid}}
+            and created_at between {{startDate}} and {{endDate}}
+            and data_key ${like} '%currency%') c
       on c.website_event_id = ed.website_event_id
     where ed.website_id = {{websiteId::uuid}}
       and ed.created_at between {{startDate}} and {{endDate}}
@@ -176,7 +184,9 @@ async function clickhouseQuery(
     from event_data
     join (select event_id 
           from event_data
-          where positionCaseInsensitive(data_key, 'currency') > 0
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and positionCaseInsensitive(data_key, 'currency') > 0
             and string_value = {currency:String}) currency
       on currency.event_id = event_data.event_id
     where website_id = {websiteId:UUID}
@@ -201,7 +211,9 @@ async function clickhouseQuery(
     from event_data ed
     join (select event_id
           from event_data
-          where positionCaseInsensitive(data_key, 'currency') > 0
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and positionCaseInsensitive(data_key, 'currency') > 0
             and string_value = {currency:String}) c
       on c.event_id = ed.event_id
     join (select distinct website_id, session_id, country
@@ -231,7 +243,9 @@ async function clickhouseQuery(
     from event_data
     join (select event_id 
           from event_data
-          where positionCaseInsensitive(data_key, 'currency') > 0
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and positionCaseInsensitive(data_key, 'currency') > 0
             and string_value = {currency:String}) currency
       on currency.event_id = event_data.event_id
     where website_id = {websiteId:UUID}
@@ -259,7 +273,9 @@ async function clickhouseQuery(
     from event_data ed
     join (select event_id, string_value as currency
           from event_data
-          where positionCaseInsensitive(data_key, 'currency') > 0) c
+          where website_id = {websiteId:UUID}
+            and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+            and positionCaseInsensitive(data_key, 'currency') > 0) c
       on c.event_id = ed.event_id
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}