Merge remote-tracking branch 'origin/dev' into dev

# Conflicts:
#	src/app/(main)/App.tsx

src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx

@@ -156,6 +156,7 @@ export function WebsiteExpandedView({
             itemCount={25}
             allowFilter={true}
             allowSearch={true}
+            expanded={true}
           />
         </Column>
       </Grid>

src/app/api/websites/[websiteId]/metrics/expanded/route.ts

@@ -0,0 +1,71 @@
+import { canViewWebsite } from '@/lib/auth';
+import { EVENT_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { getQueryFilters, parseRequest } from '@/lib/request';
+import { badRequest, json, unauthorized } from '@/lib/response';
+import { dateRangeParams, filterParams, searchParams } from '@/lib/schema';
+import {
+  getChannelExpandedMetrics,
+  getEventExpandedMetrics,
+  getPageviewExpandedMetrics,
+  getSessionExpandedMetrics,
+} from '@/queries';
+import { z } from 'zod';
+
+export async function GET(
+  request: Request,
+  { params }: { params: Promise<{ websiteId: string }> },
+) {
+  const schema = z.object({
+    type: z.string(),
+    limit: z.coerce.number().optional(),
+    offset: z.coerce.number().optional(),
+    ...dateRangeParams,
+    ...searchParams,
+    ...filterParams,
+  });
+
+  const { auth, query, error } = await parseRequest(request, schema);
+
+  if (error) {
+    return error();
+  }
+
+  const { websiteId } = await params;
+
+  if (!(await canViewWebsite(auth, websiteId))) {
+    return unauthorized();
+  }
+
+  const { type, limit, offset, search } = query;
+  const filters = await getQueryFilters(query, websiteId);
+
+  if (search) {
+    filters[type] = `c.${search}`;
+  }
+
+  if (SESSION_COLUMNS.includes(type)) {
+    const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters);
+
+    return json(data);
+  }
+
+  if (EVENT_COLUMNS.includes(type)) {
+    let data;
+
+    if (type === 'event') {
+      data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters);
+    } else {
+      data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters);
+    }
+
+    return json(data);
+  }
+
+  if (type === 'channel') {
+    const data = await getChannelExpandedMetrics(websiteId, { limit, offset }, filters);
+
+    return json(data);
+  }
+
+  return badRequest();
+}

src/app/api/websites/[websiteId]/metrics/route.ts

@@ -46,22 +46,6 @@ export async function GET(
   if (SESSION_COLUMNS.includes(type)) {
     const data = await getSessionMetrics(websiteId, { type, limit, offset }, filters);
 
-    if (type === 'language') {
-      const combined = {};
-
-      for (const { x, y } of data) {
-        const key = String(x).toLowerCase().split('-')[0];
-
-        if (combined[key] === undefined) {
-          combined[key] = { x: key, y };
-        } else {
-          combined[key].y += y;
-        }
-      }
-
-      return json(Object.values(combined));
-    }
-
     return json(data);
   }
 

src/components/hooks/index.ts

@@ -32,6 +32,7 @@ export * from './queries/useWebsitesQuery';
 export * from './queries/useWebsiteEventsQuery';
 export * from './queries/useWebsiteEventsSeriesQuery';
 export * from './queries/useWebsiteMetricsQuery';
+export * from './queries/useWebsiteExpandedMetricsQuery';
 export * from './queries/useWebsiteValuesQuery';
 export * from './useApi';
 export * from './useConfig';

src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts

@@ -0,0 +1,45 @@
+import { keepPreviousData } from '@tanstack/react-query';
+import { useApi } from '../useApi';
+import { useFilterParameters } from '../useFilterParameters';
+import { useDateParameters } from '../useDateParameters';
+import { ReactQueryOptions } from '@/lib/types';
+
+export type WebsiteExpandedMetricsData = {
+  name: string;
+  pageviews: number;
+  visitors: number;
+  visits: number;
+  bounces: number;
+  totaltime: number;
+}[];
+
+export function useWebsiteExpandedMetricsQuery(
+  websiteId: string,
+  params: { type: string; limit?: number; search?: string },
+  options?: ReactQueryOptions<WebsiteExpandedMetricsData>,
+) {
+  const { get, useQuery } = useApi();
+  const date = useDateParameters(websiteId);
+  const filters = useFilterParameters();
+
+  return useQuery<WebsiteExpandedMetricsData>({
+    queryKey: [
+      'websites:metrics:expanded',
+      {
+        websiteId,
+        ...date,
+        ...filters,
+        ...params,
+      },
+    ],
+    queryFn: async () =>
+      get(`/websites/${websiteId}/metrics/expanded`, {
+        ...date,
+        ...filters,
+        ...params,
+      }),
+    enabled: !!websiteId,
+    placeholderData: keepPreviousData,
+    ...options,
+  });
+}

src/components/metrics/ListExpandedTable.module.css

@@ -0,0 +1,7 @@
+.truncate {
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  width: 300px;
+  display: block;
+}

src/components/metrics/ListExpandedTable.tsx

@@ -0,0 +1,57 @@
+import { useMessages } from '@/components/hooks';
+import { formatShortTime } from '@/lib/format';
+import { DataColumn, DataTable } from '@umami/react-zen';
+import { ReactNode } from 'react';
+import styles from './ListExpandedTable.module.css';
+
+export interface ListExpandedTableProps {
+  data?: any[];
+  title?: string;
+  type?: string;
+  renderLabel?: (row: any, index: number) => ReactNode;
+}
+
+export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) {
+  const { formatMessage, labels } = useMessages();
+
+  return (
+    <DataTable data={data}>
+      <DataColumn id="label" label={title} align="start" className={styles.truncate}>
+        {row =>
+          renderLabel
+            ? renderLabel({ x: row?.['name'], country: row?.['country'] }, Number(row.id))
+            : (row?.['name'] ?? formatMessage(labels.unknown))
+        }
+      </DataColumn>
+      <DataColumn id="visitors" label={formatMessage(labels.visitors)} align="end">
+        {row => row?.['visitors']?.toLocaleString()}
+      </DataColumn>
+      <DataColumn id="visits" label={formatMessage(labels.visits)} align="end">
+        {row => row?.['visits']?.toLocaleString()}
+      </DataColumn>
+      <DataColumn id="pageviews" label={formatMessage(labels.views)} align="end">
+        {row => row?.['pageviews']?.toLocaleString()}
+      </DataColumn>
+      {type !== 'exit' && type !== 'entry' ? (
+        <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">
+          {row => {
+            const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;
+            return Math.round(+n) + '%';
+          }}
+        </DataColumn>
+      ) : (
+        <></>
+      )}
+      {type !== 'exit' && type !== 'entry' ? (
+        <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">
+          {row => {
+            const n = (row?.['totaltime'] / row?.['visits']) * 100;
+            return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;
+          }}
+        </DataColumn>
+      ) : (
+        <></>
+      )}
+    </DataTable>
+  );
+}

src/components/metrics/MetricsTable.tsx

@@ -1,13 +1,20 @@
-import { ReactNode, useMemo, useState } from 'react';
-import { Icon, Text, SearchField, Row, Column } from '@umami/react-zen';
 import { LinkButton } from '@/components/common/LinkButton';
+import { LoadingPanel } from '@/components/common/LoadingPanel';
+import {
+  useFormat,
+  useMessages,
+  useNavigation,
+  useWebsiteExpandedMetricsQuery,
+  useWebsiteMetricsQuery,
+} from '@/components/hooks';
+import { Arrow } from '@/components/icons';
+import { DownloadButton } from '@/components/input/DownloadButton';
 import { DEFAULT_ANIMATION_DURATION } from '@/lib/constants';
 import { percentFilter } from '@/lib/filters';
-import { useNavigation, useWebsiteMetricsQuery, useMessages, useFormat } from '@/components/hooks';
+import { Column, Icon, Row, SearchField, Text } from '@umami/react-zen';
-import { Arrow } from '@/components/icons';
+import { ReactNode, useMemo, useState } from 'react';
+import { ListExpandedTable, ListExpandedTableProps } from './ListExpandedTable';
 import { ListTable, ListTableProps } from './ListTable';
-import { LoadingPanel } from '@/components/common/LoadingPanel';
-import { DownloadButton } from '@/components/input/DownloadButton';
 
 export interface MetricsTableProps extends ListTableProps {
   websiteId: string;
@@ -21,6 +28,7 @@ export interface MetricsTableProps extends ListTableProps {
   showMore?: boolean;
   params?: { [key: string]: any };
   allowDownload?: boolean;
+  expanded?: boolean;
   children?: ReactNode;
 }
 
@@ -35,6 +43,7 @@ export function MetricsTable({
   showMore = true,
   params,
   allowDownload = true,
+  expanded = false,
   children,
   ...props
 }: MetricsTableProps) {
@@ -43,7 +52,20 @@ export function MetricsTable({
   const { updateParams } = useNavigation();
   const { formatMessage, labels } = useMessages();
 
-  const { data, isLoading, isFetching, error } = useWebsiteMetricsQuery(
+  const expandedQuery = useWebsiteExpandedMetricsQuery(
+    websiteId,
+    {
+      type,
+      search: searchFormattedValues ? undefined : search,
+      ...params,
+    },
+    {
+      retryDelay: delay || DEFAULT_ANIMATION_DURATION,
+      enabled: expanded,
+    },
+  );
+
+  const query = useWebsiteMetricsQuery(
     websiteId,
     {
       type,
@@ -53,9 +75,12 @@ export function MetricsTable({
     },
     {
       retryDelay: delay || DEFAULT_ANIMATION_DURATION,
+      enabled: !expanded,
     },
   );
 
+  const { data, isLoading, isFetching, error } = expanded ? expandedQuery : query;
+
   const filteredData = useMemo(() => {
     if (data) {
       let items = data as any[];
@@ -85,6 +110,8 @@ export function MetricsTable({
     return [];
   }, [data, dataFilter, search, limit, formatValue, type]);
 
+  const downloadData = expanded ? data : filteredData;
+
   return (
     <Column gap="3" justifyContent="space-between">
       <LoadingPanel data={data} isFetching={isFetching} isLoading={isLoading} error={error} gap>
@@ -92,10 +119,15 @@ export function MetricsTable({
           {allowSearch && <SearchField value={search} onSearch={setSearch} delay={300} />}
           <Row>
             {children}
-            {allowDownload && <DownloadButton filename={type} data={filteredData} />}
+            {allowDownload && <DownloadButton filename={type} data={downloadData} />}
           </Row>
         </Row>
-        {data && <ListTable {...(props as ListTableProps)} data={filteredData} />}
+        {data &&
+          (expanded ? (
+            <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} type={type} />
+          ) : (
+            <ListTable {...(props as ListTableProps)} data={filteredData} />
+          ))}
         <Row justifyContent="center">
           {showMore && data && !error && limit && (
             <LinkButton href={updateParams({ view: type })} variant="quiet">

src/components/metrics/QueryParametersTable.tsx

@@ -58,6 +58,7 @@ export function QueryParametersTable({
       dataFilter={filters[filter]}
       renderLabel={renderLabel}
       delay={0}
+      expanded={false}
     >
       {allowFilter && <FilterButtons items={buttons} value={filter} onChange={setFilter} />}
     </MetricsTable>

src/lib/clickhouse.ts

@@ -90,7 +90,11 @@ function mapFilter(column: string, operator: string, name: string, type: string
 function getFilterQuery(filters: Record<string, any>, options: QueryOptions = {}) {
   const query = filtersObjectToArray(filters, options).reduce((arr, { name, column, operator }) => {
     if (column) {
-      arr.push(`and ${mapFilter(column, operator, name)}`);
+      if (name === 'eventType') {
+        arr.push(`and ${mapFilter(column, operator, name, 'UInt32')}`);
+      } else {
+        arr.push(`and ${mapFilter(column, operator, name)}`);
+      }
 
       if (name === 'referrer') {
         arr.push(`and referrer_domain != hostname`);

src/lib/constants.ts

@@ -35,7 +35,7 @@ export const EVENT_COLUMNS = [
   'query',
   'event',
   'tag',
-  'host',
+  'hostname',
 ];
 
 export const SESSION_COLUMNS = [

src/queries/index.ts

@@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues';
 export * from '@/queries/sql/events/getEventDataStats';
 export * from '@/queries/sql/events/getEventDataUsage';
 export * from '@/queries/sql/events/getEventMetrics';
+export * from '@/queries/sql/events/getEventExpandedMetrics';
 export * from '@/queries/sql/events/getEventStats';
 export * from '@/queries/sql/events/getWebsiteEvents';
 export * from '@/queries/sql/events/getEventUsage';
@@ -21,6 +22,7 @@ export * from '@/queries/sql/reports/getRetention';
 export * from '@/queries/sql/reports/getBreakdown';
 export * from '@/queries/sql/reports/getUTM';
 export * from '@/queries/sql/pageviews/getPageviewMetrics';
+export * from '@/queries/sql/pageviews/getPageviewExpandedMetrics';
 export * from '@/queries/sql/pageviews/getPageviewStats';
 export * from '@/queries/sql/sessions/createSession';
 export * from '@/queries/sql/sessions/getWebsiteSession';
@@ -28,6 +30,7 @@ export * from '@/queries/sql/sessions/getSessionData';
 export * from '@/queries/sql/sessions/getSessionDataProperties';
 export * from '@/queries/sql/sessions/getSessionDataValues';
 export * from '@/queries/sql/sessions/getSessionMetrics';
+export * from '@/queries/sql/sessions/getSessionExpandedMetrics';
 export * from '@/queries/sql/sessions/getWebsiteSessions';
 export * from '@/queries/sql/sessions/getWebsiteSessionStats';
 export * from '@/queries/sql/sessions/getWebsiteSessionsWeekly';
@@ -36,6 +39,7 @@ export * from '@/queries/sql/sessions/getSessionStats';
 export * from '@/queries/sql/sessions/saveSessionData';
 export * from '@/queries/sql/getActiveVisitors';
 export * from '@/queries/sql/getChannelMetrics';
+export * from '@/queries/sql/getChannelExpandedMetrics';
 export * from '@/queries/sql/getRealtimeActivity';
 export * from '@/queries/sql/getRealtimeData';
 export * from '@/queries/sql/getValues';

src/queries/sql/events/getEventExpandedMetrics.ts

@@ -0,0 +1,113 @@
+import clickhouse from '@/lib/clickhouse';
+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface EventExpandedMetricParameters {
+  type: string;
+  limit?: string;
+  offset?: string;
+}
+
+export interface EventExpandedMetricData {
+  name: string;
+  pageviews: number;
+  visitors: number;
+  visits: number;
+  bounces: number;
+  totaltime: number;
+}
+
+export async function getEventExpandedMetrics(
+  ...args: [websiteId: string, parameters: EventExpandedMetricParameters, filters: QueryFilters]
+): Promise<EventExpandedMetricData[]> {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  parameters: EventExpandedMetricParameters,
+  filters: QueryFilters,
+) {
+  const { type, limit = 500, offset = 0 } = parameters;
+  const column = FILTER_COLUMNS[type] || type;
+  const { rawQuery, parseFilters } = prisma;
+  const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters(
+    {
+      ...filters,
+      websiteId,
+      eventType: EVENT_TYPE.customEvent,
+    },
+    { joinSession: SESSION_COLUMNS.includes(type) },
+  );
+
+  return rawQuery(
+    `
+    select ${column} x,
+      count(*) as y
+    from website_event
+    ${cohortQuery}
+    ${joinSessionQuery}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and event_type = {{eventType}}
+      ${filterQuery}
+    group by 1
+    order by 2 desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    queryParams,
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  parameters: EventExpandedMetricParameters,
+  filters: QueryFilters,
+): Promise<EventExpandedMetricData[]> {
+  const { type, limit = 500, offset = 0 } = parameters;
+  const column = FILTER_COLUMNS[type] || type;
+  const { rawQuery, parseFilters } = clickhouse;
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+  });
+
+  return rawQuery(
+    `
+    select
+      name,
+      sum(t.c) as "pageviews",
+      uniq(t.session_id) as "visitors",
+      uniq(t.visit_id) as "visits",
+      sum(if(t.c = 1, 1, 0)) as "bounces",
+      sum(max_time-min_time) as "totaltime"
+    from (
+      select
+        ${column} name,
+        session_id,
+        visit_id,
+        count(*) c,
+        min(created_at) min_time,
+        max(created_at) max_time
+      from website_event
+      ${cohortQuery}
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and name != ''
+        ${filterQuery}
+      group by name, session_id, visit_id
+    ) as t
+    group by name 
+    order by visitors desc, visits desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    { ...queryParams, ...parameters },
+  );
+}

src/queries/sql/events/getEventMetrics.ts

@@ -83,7 +83,6 @@ async function clickhouseQuery(
               ${cohortQuery}
      where website_id = {websiteId:UUID}
        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-       and event_type = {eventType:UInt32}
          ${filterQuery}
      group by x
      order by y desc

src/queries/sql/events/getEventStats.ts

@@ -72,7 +72,6 @@ async function clickhouseQuery(
     ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-      and event_type = {eventType:UInt32}
       ${filterQuery}
     group by x, t
     order by t

src/queries/sql/getChannelExpandedMetrics.ts

@@ -0,0 +1,162 @@
+import clickhouse from '@/lib/clickhouse';
+import {
+  EMAIL_DOMAINS,
+  EVENT_TYPE,
+  PAID_AD_PARAMS,
+  SEARCH_DOMAINS,
+  SHOPPING_DOMAINS,
+  SOCIAL_DOMAINS,
+  VIDEO_DOMAINS,
+} from '@/lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface ChannelExpandedMetricsParameters {
+  limit?: number | string;
+  offset?: number | string;
+}
+
+export interface ChannelExpandedMetricsData {
+  name: string;
+  pageviews: number;
+  visitors: number;
+  visits: number;
+  bounces: number;
+  totaltime: number;
+}
+
+export async function getChannelExpandedMetrics(
+  ...args: [websiteId: string, parameters: ChannelExpandedMetricsParameters, filters?: QueryFilters]
+): Promise<ChannelExpandedMetricsData[]> {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  parameters: ChannelExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<ChannelExpandedMetricsData[]> {
+  const { rawQuery, parseFilters } = prisma;
+  const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({
+    ...filters,
+    websiteId,
+    eventType: EVENT_TYPE.pageView,
+  });
+
+  return rawQuery(
+    `
+    WITH channels as (
+      select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix,
+          case
+          when referrer_domain = '' and url_query = '' then 'direct'
+          when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds'
+          when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral'
+          when position(utm_medium, 'affiliate') > 0 then 'affiliate'
+          when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'
+          when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')
+          when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social')
+          when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email'
+          when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')
+          when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')
+          else '' end AS x,
+        count(distinct session_id) y
+      from website_event
+      ${cohortQuery}
+      where website_id = {{websiteId::uuid}}
+        and event_type = {{eventType}}
+        ${dateQuery}
+        ${filterQuery}
+      group by 1, 2
+      order by y desc)
+
+    select x, sum(y) y
+    from channels
+    where x != ''
+    group by x
+    order by y desc;
+    `,
+    { ...queryParams, ...parameters },
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  parameters: ChannelExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<ChannelExpandedMetricsData[]> {
+  const { limit = 500, offset = 0 } = parameters;
+  const { rawQuery, parseFilters } = clickhouse;
+  const { queryParams, filterQuery, cohortQuery } = parseFilters({
+    ...filters,
+    websiteId,
+    eventType: EVENT_TYPE.pageView,
+  });
+
+  return rawQuery(
+    `
+    select
+      name,
+      sum(t.c) as "pageviews",
+      uniq(t.session_id) as "visitors",
+      uniq(t.visit_id) as "visits",
+      sum(if(t.c = 1, 1, 0)) as "bounces",
+      sum(max_time-min_time) as "totaltime"
+    from (
+      select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,
+          case
+          when referrer_domain = '' and url_query = '' then 'direct'
+          when multiSearchAny(url_query, [${toClickHouseStringArray(
+            PAID_AD_PARAMS,
+          )}]) != 0 then 'paidAds'
+          when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral'
+          when position(utm_medium, 'affiliate') > 0 then 'affiliate'
+          when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'
+          when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
+            SEARCH_DOMAINS,
+          )}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')
+          when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
+            SOCIAL_DOMAINS,
+          )}]) != 0 then concat(prefix, 'Social')
+          when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
+            EMAIL_DOMAINS,
+          )}]) != 0 or position(utm_medium, 'mail') > 0 then 'email'
+          when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
+            SHOPPING_DOMAINS,
+          )}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')
+          when multiSearchAny(referrer_domain, [${toClickHouseStringArray(
+            VIDEO_DOMAINS,
+          )}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')
+          else '' end AS name,
+        session_id,
+        visit_id,
+        count(*) c,
+        min(created_at) min_time,
+        max(created_at) max_time
+      from website_event
+      ${cohortQuery}
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and name != ''
+        ${filterQuery}
+      group by prefix, name, session_id, visit_id
+    ) as t
+    group by name 
+    order by visitors desc, visits desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    { ...queryParams, ...parameters },
+  );
+}
+
+function toClickHouseStringArray(arr: string[]): string {
+  return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', ');
+}
+
+function toPostgresPositionClause(column: string, arr: string[]) {
+  return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n  ');
+}

src/queries/sql/getChannelMetrics.ts

@@ -29,17 +29,35 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
 
   return rawQuery(
     `
-    select
+    WITH channels as (
-      referrer_domain as domain,
+      select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix,
-      url_query as query,
+          case
-      count(distinct session_id) as visitors
+          when referrer_domain = '' and url_query = '' then 'direct'
-    from website_event
+          when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds'
-    ${cohortQuery}
+          when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral'
-    where website_id = {{websiteId::uuid}}
+          when position(utm_medium, 'affiliate') > 0 then 'affiliate'
-        ${filterQuery}
+          when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'
+          when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')
+          when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social')
+          when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email'
+          when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')
+          when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')
+          else '' end AS x,
+        count(distinct session_id) y
+      from website_event
+      ${cohortQuery}
+      where website_id = {{websiteId::uuid}}
+        and event_type = {{eventType}}
         ${dateQuery}
-    group by 1, 2
+        ${filterQuery}
-    order by visitors desc
+      group by 1, 2
+      order by y desc)
+
+    select x, sum(y) y
+    from channels
+    where x != ''
+    group by x
+    order by y desc;
     `,
     queryParams,
   );
@@ -87,7 +105,6 @@ async function clickhouseQuery(
       from website_event
       ${cohortQuery}
       where website_id = {websiteId:UUID}
-        and event_type = {eventType:UInt32}
         ${dateQuery}
         ${filterQuery}
       group by 1, 2
@@ -106,3 +123,7 @@ async function clickhouseQuery(
 function toClickHouseStringArray(arr: string[]): string {
   return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', ');
 }
+
+function toPostgresPositionClause(column: string, arr: string[]) {
+  return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n  ');
+}

src/queries/sql/getWebsiteStats.ts

@@ -94,7 +94,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by session_id, visit_id
     ) as t;
@@ -117,7 +116,6 @@ async function clickhouseQuery(
         ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-      and event_type = {eventType:UInt32}
       ${filterQuery}
       group by session_id, visit_id
     ) as t;

src/queries/sql/pageviews/getPageviewExpandedMetrics.ts

@@ -0,0 +1,166 @@
+import clickhouse from '@/lib/clickhouse';
+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface PageviewExpandedMetricsParameters {
+  type: string;
+  limit?: number | string;
+  offset?: number | string;
+}
+
+export interface PageviewExpandedMetricsData {
+  name: string;
+  pageviews: number;
+  visitors: number;
+  visits: number;
+  bounces: number;
+  totaltime: number;
+}
+
+export async function getPageviewExpandedMetrics(
+  ...args: [websiteId: string, parameters: PageviewExpandedMetricsParameters, filters: QueryFilters]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  parameters: PageviewExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<PageviewExpandedMetricsData[]> {
+  const { type, limit = 500, offset = 0 } = parameters;
+  const column = FILTER_COLUMNS[type] || type;
+  const { rawQuery, parseFilters } = prisma;
+  const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
+    {
+      ...filters,
+      websiteId,
+      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+    },
+    { joinSession: SESSION_COLUMNS.includes(type) },
+  );
+
+  let entryExitQuery = '';
+  let excludeDomain = '';
+
+  if (column === 'referrer_domain') {
+    excludeDomain = `and website_event.referrer_domain != website_event.hostname
+      and website_event.referrer_domain != ''`;
+  }
+
+  if (type === 'entry' || type === 'exit') {
+    const aggregrate = type === 'entry' ? 'min' : 'max';
+
+    entryExitQuery = `
+      join (
+        select visit_id,
+            ${aggregrate}(created_at) target_created_at
+        from website_event
+        where website_event.website_id = {{websiteId::uuid}}
+          and website_event.created_at between {{startDate}} and {{endDate}}
+          and event_type = {{eventType}}
+        group by visit_id
+      ) x
+      on x.visit_id = website_event.visit_id
+          and x.target_created_at = website_event.created_at
+    `;
+  }
+
+  return rawQuery(
+    `
+    select ${column} x,
+      count(distinct website_event.session_id) as y
+    from website_event
+    ${joinSessionQuery}
+    ${cohortQuery}
+    ${entryExitQuery}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and event_type = {{eventType}}
+      ${excludeDomain}
+      ${filterQuery}
+    group by 1
+    order by 2 desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    queryParams,
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  parameters: PageviewExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
+  const { type, limit = 500, offset = 0 } = parameters;
+  const column = FILTER_COLUMNS[type] || type;
+  const { rawQuery, parseFilters } = clickhouse;
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+  });
+
+  let excludeDomain = '';
+  let entryExitQuery = '';
+
+  if (column === 'referrer_domain') {
+    excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
+  }
+
+  if (type === 'entry' || type === 'exit') {
+    const aggregrate = type === 'entry' ? 'min' : 'max';
+
+    entryExitQuery = `
+      JOIN (select visit_id,
+          ${aggregrate}(created_at) target_created_at
+      from website_event
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and event_type = {eventType:UInt32}
+      group by visit_id) x
+      ON x.visit_id = website_event.visit_id
+          and x.target_created_at = website_event.created_at`;
+  }
+
+  return rawQuery(
+    `
+    select
+      name,
+      sum(t.c) as "pageviews",
+      uniq(t.session_id) as "visitors",
+      uniq(t.visit_id) as "visits",
+      sum(if(t.c = 1, 1, 0)) as "bounces",
+      sum(max_time-min_time) as "totaltime"
+    from (
+      select
+        ${column} name,
+        session_id,
+        visit_id,
+        count(*) c,
+        min(created_at) min_time,
+        max(created_at) max_time
+      from website_event
+      ${cohortQuery}
+      ${entryExitQuery}
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and name != ''
+        ${excludeDomain}
+        ${filterQuery}
+      group by name, session_id, visit_id
+    ) as t
+    group by name 
+    order by visitors desc, visits desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    { ...queryParams, ...parameters },
+  );
+}

src/queries/sql/pageviews/getPageviewMetrics.ts

@@ -136,7 +136,6 @@ async function clickhouseQuery(
     ${entryExitQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-      and event_type = {eventType:UInt32}
       ${excludeDomain}
       ${filterQuery}
     group by x
@@ -174,7 +173,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${excludeDomain}
         ${filterQuery}
       ${groupByQuery}) as g

src/queries/sql/pageviews/getPageviewStats.ts

@@ -66,7 +66,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
     ) as g
@@ -85,7 +84,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
     ) as g

src/queries/sql/reports/getAttribution.ts

@@ -477,7 +477,6 @@ async function clickhouseQuery(
     where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
         and ${column} = {step:String}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
     `,
     queryParams,

src/queries/sql/reports/getBreakdown.ts

@@ -115,7 +115,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by ${parseFieldsByName(fields)}, 
         session_id, visit_id

src/queries/sql/sessions/getSessionDataValues.ts

@@ -28,9 +28,9 @@ async function relationalQuery(
         else string_value
       end as "value",
       count(distinct session_data.session_id) as "total"
-    from website_event e
+    from website_event
     ${cohortQuery}
-    join session_data d 
+    join session_data
         on session_data.session_id = website_event.session_id
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
@@ -58,9 +58,9 @@ async function clickhouseQuery(
               data_type = 4, toString(date_trunc('hour', date_value)),
               string_value) as "value",
       uniq(session_data.session_id) as "total"
-    from website_event e
+    from website_event
     ${cohortQuery}
-    join session_data d final
+    join session_data final
       on session_data.session_id = website_event.session_id
     where website_event.website_id = {websiteId:UUID}
       and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}

src/queries/sql/sessions/getSessionExpandedMetrics.ts

@@ -0,0 +1,133 @@
+import clickhouse from '@/lib/clickhouse';
+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
+import prisma from '@/lib/prisma';
+import { QueryFilters } from '@/lib/types';
+
+export interface SessionExpandedMetricsParameters {
+  type: string;
+  limit?: number | string;
+  offset?: number | string;
+}
+
+export interface SessionExpandedMetricsData {
+  name: string;
+  pageviews: number;
+  visitors: number;
+  visits: number;
+  bounces: number;
+  totaltime: number;
+}
+
+export async function getSessionExpandedMetrics(
+  ...args: [websiteId: string, parameters: SessionExpandedMetricsParameters, filters: QueryFilters]
+): Promise<SessionExpandedMetricsData[]> {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  parameters: SessionExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<SessionExpandedMetricsData[]> {
+  const { type, limit = 500, offset = 0 } = parameters;
+  let column = FILTER_COLUMNS[type] || type;
+  const { parseFilters, rawQuery } = prisma;
+  const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
+    {
+      ...filters,
+      websiteId,
+      eventType: EVENT_TYPE.pageView,
+    },
+    {
+      joinSession: SESSION_COLUMNS.includes(type),
+    },
+  );
+  const includeCountry = column === 'city' || column === 'region';
+
+  if (type === 'language') {
+    column = `lower(left(${type}, 2))`;
+  }
+
+  return rawQuery(
+    `
+    select 
+      ${column} x,
+      count(distinct website_event.session_id) y
+      ${includeCountry ? ', country' : ''}
+    from website_event
+    ${cohortQuery}
+    ${joinSessionQuery}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and website_event.event_type = {{eventType}}
+    ${filterQuery}
+    group by 1 
+    ${includeCountry ? ', 3' : ''}
+    order by 2 desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    { ...queryParams, ...parameters },
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  parameters: SessionExpandedMetricsParameters,
+  filters: QueryFilters,
+): Promise<SessionExpandedMetricsData[]> {
+  const { type, limit = 500, offset = 0 } = parameters;
+  let column = FILTER_COLUMNS[type] || type;
+  const { parseFilters, rawQuery } = clickhouse;
+  const { filterQuery, cohortQuery, queryParams } = parseFilters({
+    ...filters,
+    websiteId,
+    eventType: EVENT_TYPE.pageView,
+  });
+  const includeCountry = column === 'city' || column === 'region';
+
+  if (type === 'language') {
+    column = `lower(left(${type}, 2))`;
+  }
+
+  return rawQuery(
+    `
+    select
+      name,
+      ${includeCountry ? 'country,' : ''}
+      sum(t.c) as "pageviews",
+      uniq(t.session_id) as "visitors",
+      uniq(t.visit_id) as "visits",
+      sum(if(t.c = 1, 1, 0)) as "bounces",
+      sum(max_time-min_time) as "totaltime"
+    from (
+      select
+        ${column} name,
+        ${includeCountry ? 'country,' : ''}
+        session_id,
+        visit_id,
+        count(*) c,
+        min(created_at) min_time,
+        max(created_at) max_time
+      from website_event
+      ${cohortQuery}
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+        and name != ''
+        ${filterQuery}
+      group by name, session_id, visit_id
+      ${includeCountry ? ', country' : ''}
+    ) as t
+    group by name 
+    ${includeCountry ? ', country' : ''}
+    order by visitors desc, visits desc
+    limit ${limit}
+    offset ${offset}
+    `,
+    { ...queryParams, ...parameters },
+  );
+}

src/queries/sql/sessions/getSessionMetrics.ts

@@ -25,7 +25,7 @@ async function relationalQuery(
   filters: QueryFilters,
 ) {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { parseFilters, rawQuery } = prisma;
   const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
     {
@@ -39,6 +39,10 @@ async function relationalQuery(
   );
   const includeCountry = column === 'city' || column === 'region';
 
+  if (type === 'language') {
+    column = `lower(left(${type}, 2))`;
+  }
+
   return rawQuery(
     `
     select 
@@ -68,7 +72,7 @@ async function clickhouseQuery(
   filters: QueryFilters,
 ): Promise<{ x: string; y: number }[]> {
   const { type, limit = 500, offset = 0 } = parameters;
-  const column = FILTER_COLUMNS[type] || type;
+  let column = FILTER_COLUMNS[type] || type;
   const { parseFilters, rawQuery } = clickhouse;
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
@@ -77,6 +81,10 @@ async function clickhouseQuery(
   });
   const includeCountry = column === 'city' || column === 'region';
 
+  if (type === 'language') {
+    column = `lower(left(${type}, 2))`;
+  }
+
   let sql = '';
 
   if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
@@ -89,7 +97,6 @@ async function clickhouseQuery(
     ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-      and event_type = {eventType:UInt32}
       ${filterQuery}
     group by x 
     ${includeCountry ? ', country' : ''}
@@ -107,7 +114,6 @@ async function clickhouseQuery(
     ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-      and event_type = {eventType:UInt32}
       ${filterQuery}
     group by x 
     ${includeCountry ? ', country' : ''}

src/queries/sql/sessions/getSessionStats.ts

@@ -66,7 +66,6 @@ async function clickhouseQuery(
       ${cohortQuery}
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
     ) as g
@@ -84,7 +83,6 @@ async function clickhouseQuery(
       from website_event_stats_hourly as website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime64} and {endDate:DateTime64}
-        and event_type = {eventType:UInt32}
         ${filterQuery}
       group by t
     ) as g