finish expanded queries and ui.

db/clickhouse/schema.sql

@@ -57,7 +57,7 @@ CREATE TABLE umami.event_data
     event_name String,
     data_key String,
     string_value Nullable(String),
-    number_value Nullable(Decimal64(4)),
+    number_value Nullable(Decimal64(22, 4)),
     date_value Nullable(DateTime('UTC')),
     data_type UInt32,
     created_at DateTime('UTC'),
@@ -73,7 +73,7 @@ CREATE TABLE umami.session_data
     session_id UUID,
     data_key String,
     string_value Nullable(String),
-    number_value Nullable(Decimal64(4)),
+    number_value Nullable(Decimal64(22, 4)),
     date_value Nullable(DateTime('UTC')),
     data_type UInt32,
     distinct_id String,

src/app/api/reports/utm/route.ts

@@ -17,8 +17,8 @@ export async function POST(request: Request) {
     return unauthorized();
   }
 
-  const parameters = await setWebsiteDate(websiteId, body.parameters);
   const filters = await getQueryFilters(body.filters, websiteId);
+  const parameters = await setWebsiteDate(websiteId, body.parameters);
 
   const data = await getUTM(websiteId, parameters as UTMParameters, filters);
 

src/app/api/websites/[websiteId]/sessions/route.ts

@@ -2,7 +2,7 @@ import { z } from 'zod';
 import { getQueryFilters, parseRequest } from '@/lib/request';
 import { unauthorized, json } from '@/lib/response';
 import { canViewWebsite } from '@/lib/auth';
-import { dateRangeParams, filterParams, pagingParams } from '@/lib/schema';
+import { dateRangeParams, filterParams, pagingParams, searchParams } from '@/lib/schema';
 import { getWebsiteSessions } from '@/queries';
 
 export async function GET(
@@ -13,6 +13,7 @@ export async function GET(
     ...dateRangeParams,
     ...filterParams,
     ...pagingParams,
+    ...searchParams,
   });
 
   const { auth, query, error } = await parseRequest(request, schema);

src/components/metrics/ListExpandedTable.tsx

@@ -7,11 +7,10 @@ import styles from './ListExpandedTable.module.css';
 export interface ListExpandedTableProps {
   data?: any[];
   title?: string;
-  type?: string;
   renderLabel?: (row: any, index: number) => ReactNode;
 }
 
-export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) {
+export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) {
   const { formatMessage, labels } = useMessages();
 
   return (
@@ -32,26 +31,18 @@ export function ListExpandedTable({ data = [], title, type, renderLabel }: ListE
       <DataColumn id="pageviews" label={formatMessage(labels.views)} align="end">
         {row => row?.['pageviews']?.toLocaleString()}
       </DataColumn>
-      {type !== 'exit' && type !== 'entry' ? (
+      <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">
-        <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">
+        {row => {
-          {row => {
+          const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;
-            const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;
+          return Math.round(+n) + '%';
-            return Math.round(+n) + '%';
+        }}
-          }}
+      </DataColumn>
-        </DataColumn>
+      <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">
-      ) : (
+        {row => {
-        <></>
+          const n = (row?.['totaltime'] / row?.['visits']) * 100;
-      )}
+          return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;
-      {type !== 'exit' && type !== 'entry' ? (
+        }}
-        <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">
+      </DataColumn>
-          {row => {
-            const n = (row?.['totaltime'] / row?.['visits']) * 100;
-            return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;
-          }}
-        </DataColumn>
-      ) : (
-        <></>
-      )}
     </DataTable>
   );
 }

src/components/metrics/MetricsTable.tsx

@@ -114,7 +114,7 @@ export function MetricsTable({
 
   return (
     <Column gap="3" justifyContent="space-between">
-      <LoadingPanel data={data} isFetching={isFetching} isLoading={isLoading} error={error} gap>
+      <LoadingPanel isFetching={isFetching} isLoading={isLoading} error={error} gap>
         <Row alignItems="center" justifyContent="space-between">
           {allowSearch && <SearchField value={search} onSearch={setSearch} delay={300} />}
           <Row>
@@ -124,7 +124,7 @@ export function MetricsTable({
         </Row>
         {data &&
           (expanded ? (
-            <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} type={type} />
+            <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} />
           ) : (
             <ListTable {...(props as ListTableProps)} data={filteredData} />
           ))}

src/components/metrics/ReferrersTable.tsx

@@ -1,11 +1,11 @@
-import { Row } from '@umami/react-zen';
-import { FilterLink } from '@/components/common/FilterLink';
 import { Favicon } from '@/components/common/Favicon';
 import { FilterButtons } from '@/components/common/FilterButtons';
+import { FilterLink } from '@/components/common/FilterLink';
 import { useMessages, useNavigation } from '@/components/hooks';
-import { MetricsTable, MetricsTableProps } from './MetricsTable';
-import thenby from 'thenby';
 import { GROUPED_DOMAINS } from '@/lib/constants';
+import { emptyFilter } from '@/lib/filters';
+import { Row } from '@umami/react-zen';
+import { MetricsTable, MetricsTableProps } from './MetricsTable';
 
 export interface ReferrersTableProps extends MetricsTableProps {
   allowFilter?: boolean;
@@ -36,7 +36,7 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) {
 
   const renderLink = ({ x: referrer }) => {
     if (view === 'grouped') {
-      if (referrer === '_other') {
+      if (referrer === 'Other') {
         return `(${formatMessage(labels.other)})`;
       } else {
         return (
@@ -60,38 +60,13 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) {
     );
   };
 
-  const getDomain = (x: string) => {
-    for (const { domain, match } of GROUPED_DOMAINS) {
-      if (Array.isArray(match) ? match.some(str => x.includes(str)) : x.includes(match)) {
-        return domain;
-      }
-    }
-    return '_other';
-  };
-
-  const groupedFilter = (data: any[]) => {
-    const groups = { _other: 0 };
-
-    for (const { x, y } of data) {
-      const domain = getDomain(x);
-      if (!groups[domain]) {
-        groups[domain] = 0;
-      }
-      groups[domain] += +y;
-    }
-
-    return Object.keys(groups)
-      .map((key: any) => ({ x: key, y: groups[key] }))
-      .sort(thenby.firstBy('y', -1));
-  };
-
   return (
     <MetricsTable
       {...props}
       title={formatMessage(labels.referrers)}
-      type="referrer"
+      type={view}
       metric={formatMessage(labels.visitors)}
-      dataFilter={view === 'grouped' ? groupedFilter : undefined}
+      dataFilter={emptyFilter}
       renderLabel={renderLink}
     >
       {allowFilter && <FilterButtons items={buttons} value={view} onChange={handleSelect} />}

src/lib/constants.ts

@@ -26,11 +26,13 @@ export const REALTIME_RANGE = 30;
 export const REALTIME_INTERVAL = 10000;
 
 export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
+
 export const EVENT_COLUMNS = [
   'path',
   'entry',
   'exit',
   'referrer',
+  'grouped',
   'title',
   'query',
   'event',
@@ -59,6 +61,7 @@ export const FILTER_COLUMNS = {
   entry: 'url_path',
   exit: 'url_path',
   referrer: 'referrer_domain',
+  grouped: 'referrer_domain',
   hostname: 'hostname',
   title: 'page_title',
   query: 'url_query',
@@ -386,6 +389,9 @@ export const GROUPED_DOMAINS = [
   { name: 'Snapchat', domain: 'snapchat.com', match: 'snapchat.' },
   { name: 'Pinterest', domain: 'pinterest.com', match: 'pinterest.' },
   { name: 'ChatGPT', domain: 'chatgpt.com', match: 'chatgpt.' },
+  { name: 'Yahoo', domain: 'yahoo.com', match: 'yahoo.' },
+  { name: 'Yandex', domain: 'yandex.ru', match: 'yandex.' },
+  { name: 'Baidu', domain: 'baidu.com', match: 'baidu.' },
 ];
 
 export const MAP_FILE = '/datamaps.world.json';

src/queries/sql/events/getWebsiteEvents.ts

@@ -19,7 +19,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     search: `%${search}%`,
   });
 
-  const searchQuery = filters.search
+  const searchQuery = search
     ? `and ((event_name ilike {{search}} and event_type = 2)
            or (url_path ilike {{search}} and event_type = 1))`
     : '';
@@ -59,12 +59,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
 
 async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
   const { pagedRawQuery, parseFilters } = clickhouse;
+  const { search } = filters;
   const { queryParams, dateQuery, cohortQuery, filterQuery } = parseFilters({
     ...filters,
     websiteId,
   });
 
-  const searchQuery = filters.search
+  const searchQuery = search
     ? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = 2)
            or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = 1))`
     : '';

src/queries/sql/pageviews/getPageviewExpandedMetrics.ts

@@ -1,5 +1,5 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
@@ -99,7 +99,7 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
@@ -112,21 +112,24 @@ async function clickhouseQuery(
 
   if (column === 'referrer_domain') {
     excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
+    if (type === 'grouped') {
+      column = toClickHouseGroupedReferrer(GROUPED_DOMAINS);
+    }
   }
 
   if (type === 'entry' || type === 'exit') {
-    const aggregrate = type === 'entry' ? 'min' : 'max';
+    const aggregrate = type === 'entry' ? 'argMin' : 'argMax';
+    column = `x.${column}`;
 
     entryExitQuery = `
       JOIN (select visit_id,
-          ${aggregrate}(created_at) target_created_at
+          ${aggregrate}(url_path, created_at) url_path
       from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
       group by visit_id) x
-      ON x.visit_id = website_event.visit_id
+      ON x.visit_id = website_event.visit_id`;
-          and x.target_created_at = website_event.created_at`;
   }
 
   return rawQuery(
@@ -164,3 +167,19 @@ async function clickhouseQuery(
     { ...queryParams, ...parameters },
   );
 }
+
+export function toClickHouseGroupedReferrer(
+  domains: any[],
+  column: string = 'referrer_domain',
+): string {
+  return [
+    'CASE',
+    ...domains.map(group => {
+      const matches = Array.isArray(group.match) ? group.match : [group.match];
+      const formattedArray = matches.map(m => `'${m}'`).join(', ');
+      return `  WHEN multiSearchAny(${column}, [${formattedArray}]) != 0 THEN '${group.domain}'`;
+    }),
+    "  ELSE 'Other'",
+    'END',
+  ].join('\n');
+}

src/queries/sql/pageviews/getPageviewMetrics.ts

@@ -30,7 +30,7 @@ async function relationalQuery(
   filters: QueryFilters,
 ): Promise<PageviewMetricsData[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = prisma;
   const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
     {
@@ -50,20 +50,21 @@ async function relationalQuery(
   }
 
   if (type === 'entry' || type === 'exit') {
-    const aggregrate = type === 'entry' ? 'min' : 'max';
+    const order = type === 'entry' ? 'asc' : 'desc';
+    column = `x.${column}`;
 
     entryExitQuery = `
       join (
-        select visit_id,
+        select distinct on (visit_id)
-            ${aggregrate}(created_at) target_created_at
+          visit_id,
+          url_path
         from website_event
         where website_event.website_id = {{websiteId::uuid}}
           and website_event.created_at between {{startDate}} and {{endDate}}
           and event_type = {{eventType}}
-        group by visit_id
+        order by visit_id, created_at ${order}
       ) x
       on x.visit_id = website_event.visit_id
-          and x.target_created_at = website_event.created_at
     `;
   }
 
@@ -95,7 +96,7 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
@@ -114,18 +115,18 @@ async function clickhouseQuery(
     }
 
     if (type === 'entry' || type === 'exit') {
-      const aggregrate = type === 'entry' ? 'min' : 'max';
+      const aggregrate = type === 'entry' ? 'argMin' : 'argMax';
+      column = `x.${column}`;
 
       entryExitQuery = `
       JOIN (select visit_id,
-          ${aggregrate}(created_at) target_created_at
+          ${aggregrate}(url_path, created_at) url_path
       from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
       group by visit_id) x
-      ON x.visit_id = website_event.visit_id
+      ON x.visit_id = website_event.visit_id`;
-          and x.target_created_at = website_event.created_at`;
     }
 
     sql = `

src/queries/sql/reports/getUTM.ts

@@ -1,16 +1,15 @@
 import clickhouse from '@/lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface UTMParameters {
+  startDate: Date;
+  endDate: Date;
+}
 
 export async function getUTM(
-  ...args: [
+  ...args: [websiteId: string, parameters: UTMParameters, filters: QueryFilters]
-    websiteId: string,
-    filters: {
-      startDate: Date;
-      endDate: Date;
-      timezone?: string;
-    },
-  ]
 ) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -20,14 +19,18 @@ export async function getUTM(
 
 async function relationalQuery(
   websiteId: string,
-  filters: {
+  parameters: UTMParameters,
-    startDate: Date;
+  filters: QueryFilters,
-    endDate: Date;
-    timezone?: string;
-  },
 ) {
-  const { startDate, endDate } = filters;
+  const { startDate, endDate } = parameters;
-  const { rawQuery } = prisma;
+  const { parseFilters, rawQuery } = prisma;
+
+  const { filterQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    startDate,
+    endDate,
+  });
 
   return rawQuery(
     `
@@ -37,26 +40,26 @@ async function relationalQuery(
       and created_at between {{startDate}} and {{endDate}}
       and coalesce(url_query, '') != ''
       and event_type = 1
+      ${filterQuery}
     group by 1
     `,
-    {
+    queryParams,
-      websiteId,
-      startDate,
-      endDate,
-    },
   );
 }
 
 async function clickhouseQuery(
   websiteId: string,
-  filters: {
+  parameters: UTMParameters,
-    startDate: Date;
+  filters: QueryFilters,
-    endDate: Date;
-    timezone?: string;
-  },
 ) {
-  const { startDate, endDate } = filters;
+  const { startDate, endDate } = parameters;
-  const { rawQuery } = clickhouse;
+  const { parseFilters, rawQuery } = clickhouse;
+  const { filterQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    startDate,
+    endDate,
+  });
 
   return rawQuery(
     `
@@ -66,12 +69,9 @@ async function clickhouseQuery(
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and url_query != ''
       and event_type = 1
+      ${filterQuery}
     group by 1
     `,
-    {
+    queryParams,
-      websiteId,
-      startDate,
-      endDate,
-    },
   );
 }

src/queries/sql/sessions/getWebsiteSessions.ts

@@ -20,7 +20,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     search: search ? `%${search}%` : undefined,
   });
 
-  const searchQuery = search ? `and session.distinct_id ilike {{search}}` : '';
+  const searchQuery = search
+    ? `and (distinct_id ilike {{search}}
+           or city ilike {{search}}
+           or browser ilike {{search}}
+           or os ilike {{search}}
+           or device ilike {{search}})`
+    : '';
 
   return pagedRawQuery(
     `
@@ -74,7 +80,13 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     websiteId,
   });
 
-  const searchQuery = search ? `and positionCaseInsensitive(distinct_id, {search:String}) > 0` : '';
+  const searchQuery = search
+    ? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0)
+           or (positionCaseInsensitive(city, {search:String}) > 0)
+           or (positionCaseInsensitive(browser, {search:String}) > 0)
+           or (positionCaseInsensitive(os, {search:String}) > 0)
+           or (positionCaseInsensitive(device, {search:String}) > 0))`
+    : '';
 
   let sql = '';