finish expanded queries and ui.

src/queries/sql/events/getWebsiteEvents.ts

@@ -19,7 +19,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     search: `%${search}%`,
   });
 
-  const searchQuery = filters.search
+  const searchQuery = search
     ? `and ((event_name ilike {{search}} and event_type = 2)
            or (url_path ilike {{search}} and event_type = 1))`
     : '';
@@ -59,12 +59,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
 
 async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
   const { pagedRawQuery, parseFilters } = clickhouse;
+  const { search } = filters;
   const { queryParams, dateQuery, cohortQuery, filterQuery } = parseFilters({
     ...filters,
     websiteId,
   });
 
-  const searchQuery = filters.search
+  const searchQuery = search
     ? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = 2)
            or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = 1))`
     : '';

src/queries/sql/pageviews/getPageviewExpandedMetrics.ts

@@ -1,5 +1,5 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
@@ -99,7 +99,7 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
@@ -112,21 +112,24 @@ async function clickhouseQuery(
 
   if (column === 'referrer_domain') {
     excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
+    if (type === 'grouped') {
+      column = toClickHouseGroupedReferrer(GROUPED_DOMAINS);
+    }
   }
 
   if (type === 'entry' || type === 'exit') {
-    const aggregrate = type === 'entry' ? 'min' : 'max';
+    const aggregrate = type === 'entry' ? 'argMin' : 'argMax';
+    column = `x.${column}`;
 
     entryExitQuery = `
       JOIN (select visit_id,
-          ${aggregrate}(created_at) target_created_at
+          ${aggregrate}(url_path, created_at) url_path
       from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
       group by visit_id) x
-      ON x.visit_id = website_event.visit_id
-          and x.target_created_at = website_event.created_at`;
+      ON x.visit_id = website_event.visit_id`;
   }
 
   return rawQuery(
@@ -164,3 +167,19 @@ async function clickhouseQuery(
     { ...queryParams, ...parameters },
   );
 }
+
+export function toClickHouseGroupedReferrer(
+  domains: any[],
+  column: string = 'referrer_domain',
+): string {
+  return [
+    'CASE',
+    ...domains.map(group => {
+      const matches = Array.isArray(group.match) ? group.match : [group.match];
+      const formattedArray = matches.map(m => `'${m}'`).join(', ');
+      return `  WHEN multiSearchAny(${column}, [${formattedArray}]) != 0 THEN '${group.domain}'`;
+    }),
+    "  ELSE 'Other'",
+    'END',
+  ].join('\n');
+}

src/queries/sql/pageviews/getPageviewMetrics.ts

@@ -30,7 +30,7 @@ async function relationalQuery(
   filters: QueryFilters,
 ): Promise<PageviewMetricsData[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = prisma;
   const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
     {
@@ -50,20 +50,21 @@ async function relationalQuery(
   }
 
   if (type === 'entry' || type === 'exit') {
-    const aggregrate = type === 'entry' ? 'min' : 'max';
+    const order = type === 'entry' ? 'asc' : 'desc';
+    column = `x.${column}`;
 
     entryExitQuery = `
       join (
-        select visit_id,
-            ${aggregrate}(created_at) target_created_at
+        select distinct on (visit_id)
+          visit_id,
+          url_path
         from website_event
         where website_event.website_id = {{websiteId::uuid}}
           and website_event.created_at between {{startDate}} and {{endDate}}
           and event_type = {{eventType}}
-        group by visit_id
+        order by visit_id, created_at ${order}
       ) x
       on x.visit_id = website_event.visit_id
-          and x.target_created_at = website_event.created_at
     `;
   }
 
@@ -95,7 +96,7 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { rawQuery, parseFilters } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
@@ -114,18 +115,18 @@ async function clickhouseQuery(
     }
 
     if (type === 'entry' || type === 'exit') {
-      const aggregrate = type === 'entry' ? 'min' : 'max';
+      const aggregrate = type === 'entry' ? 'argMin' : 'argMax';
+      column = `x.${column}`;
 
       entryExitQuery = `
       JOIN (select visit_id,
-          ${aggregrate}(created_at) target_created_at
+          ${aggregrate}(url_path, created_at) url_path
       from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and event_type = {eventType:UInt32}
       group by visit_id) x
-      ON x.visit_id = website_event.visit_id
-          and x.target_created_at = website_event.created_at`;
+      ON x.visit_id = website_event.visit_id`;
     }
 
     sql = `

src/queries/sql/reports/getUTM.ts

@@ -1,16 +1,15 @@
 import clickhouse from '@/lib/clickhouse';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface UTMParameters {
+  startDate: Date;
+  endDate: Date;
+}
 
 export async function getUTM(
-  ...args: [
-    websiteId: string,
-    filters: {
-      startDate: Date;
-      endDate: Date;
-      timezone?: string;
-    },
-  ]
+  ...args: [websiteId: string, parameters: UTMParameters, filters: QueryFilters]
 ) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -20,14 +19,18 @@ export async function getUTM(
 
 async function relationalQuery(
   websiteId: string,
-  filters: {
-    startDate: Date;
-    endDate: Date;
-    timezone?: string;
-  },
+  parameters: UTMParameters,
+  filters: QueryFilters,
 ) {
-  const { startDate, endDate } = filters;
-  const { rawQuery } = prisma;
+  const { startDate, endDate } = parameters;
+  const { parseFilters, rawQuery } = prisma;
+
+  const { filterQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    startDate,
+    endDate,
+  });
 
   return rawQuery(
     `
@@ -37,26 +40,26 @@ async function relationalQuery(
       and created_at between {{startDate}} and {{endDate}}
       and coalesce(url_query, '') != ''
       and event_type = 1
+      ${filterQuery}
     group by 1
     `,
-    {
-      websiteId,
-      startDate,
-      endDate,
-    },
+    queryParams,
   );
 }
 
 async function clickhouseQuery(
   websiteId: string,
-  filters: {
-    startDate: Date;
-    endDate: Date;
-    timezone?: string;
-  },
+  parameters: UTMParameters,
+  filters: QueryFilters,
 ) {
-  const { startDate, endDate } = filters;
-  const { rawQuery } = clickhouse;
+  const { startDate, endDate } = parameters;
+  const { parseFilters, rawQuery } = clickhouse;
+  const { filterQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    startDate,
+    endDate,
+  });
 
   return rawQuery(
     `
@@ -66,12 +69,9 @@ async function clickhouseQuery(
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
       and url_query != ''
       and event_type = 1
+      ${filterQuery}
     group by 1
     `,
-    {
-      websiteId,
-      startDate,
-      endDate,
-    },
+    queryParams,
   );
 }

src/queries/sql/sessions/getWebsiteSessions.ts

@@ -20,7 +20,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     search: search ? `%${search}%` : undefined,
   });
 
-  const searchQuery = search ? `and session.distinct_id ilike {{search}}` : '';
+  const searchQuery = search
+    ? `and (distinct_id ilike {{search}}
+           or city ilike {{search}}
+           or browser ilike {{search}}
+           or os ilike {{search}}
+           or device ilike {{search}})`
+    : '';
 
   return pagedRawQuery(
     `
@@ -74,7 +80,13 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
     websiteId,
   });
 
-  const searchQuery = search ? `and positionCaseInsensitive(distinct_id, {search:String}) > 0` : '';
+  const searchQuery = search
+    ? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0)
+           or (positionCaseInsensitive(city, {search:String}) > 0)
+           or (positionCaseInsensitive(browser, {search:String}) > 0)
+           or (positionCaseInsensitive(os, {search:String}) > 0)
+           or (positionCaseInsensitive(device, {search:String}) > 0))`
+    : '';
 
   let sql = '';