Merge branch 'dev' of https://github.com/umami-software/umami into analytics

src/lib/clickhouse.ts

@@ -89,6 +89,21 @@ function mapFilter(column: string, operator: string, name: string, type: string
   }
 }
 
+function mapCohortFilter(column: string, operator: string, value: string) {
+  switch (operator) {
+    case OPERATORS.equals:
+      return `${column} = '${value}'`;
+    case OPERATORS.notEquals:
+      return `${column} != '${value}'`;
+    case OPERATORS.contains:
+      return `positionCaseInsensitive(${column}, '${value}') > 0`;
+    case OPERATORS.doesNotContain:
+      return `positionCaseInsensitive(${column}, '${value}') = 0`;
+    default:
+      return '';
+  }
+}
+
 function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
   const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
     if (column) {
@@ -105,6 +120,42 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
   return query.join('\n');
 }
 
+function getCohortQuery(websiteId: string, filters: QueryFilters = {}, options: QueryOptions = {}) {
+  const query = filtersToArray(filters, options).reduce(
+    (arr, { name, column, operator, value }) => {
+      if (column) {
+        arr.push(
+          `${arr.length === 0 ? 'where' : 'and'} ${mapCohortFilter(column, operator, value)}`,
+        );
+
+        if (name === 'referrer') {
+          arr.push(`and referrer_domain != hostname`);
+        }
+      }
+
+      return arr;
+    },
+    [],
+  );
+
+  if (query.length > 0) {
+    // add website and date range filters
+    query.push(`and website_id = '${websiteId}'`);
+    query.push(
+      `and created_at between parseDateTimeBestEffort('${filters.startDate}') and parseDateTimeBestEffort('${filters.endDate}')`,
+    );
+
+    return `join
+    (select distinct session_id
+    from website_event
+    ${query.join('\n')}) cohort
+    on cohort.session_id = website_event.session_id
+    `;
+  }
+
+  return '';
+}
+
 function getDateQuery(filters: QueryFilters = {}) {
   const { startDate, endDate, timezone } = filters;
 
@@ -146,6 +197,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
       websiteId,
       startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
     },
+    cohortQuery: getCohortQuery(websiteId, filters?.cohort),
   };
 }
 

src/lib/constants.ts

@@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';
 export const FILTER_PAGES = 'filter-pages';
 
 export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
-export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];
+export const EVENT_COLUMNS = [
+  'url',
+  'entry',
+  'exit',
+  'referrer',
+  'title',
+  'query',
+  'event',
+  'tag',
+  'host',
+];
 
 export const SESSION_COLUMNS = [
   'browser',
@@ -44,9 +54,13 @@ export const SESSION_COLUMNS = [
   'country',
   'city',
   'region',
-  'host',
 ];
 
+export const FILTER_GROUPS = {
+  segment: 'segment',
+  cohort: 'cohort',
+};
+
 export const FILTER_COLUMNS = {
   url: 'url_path',
   entry: 'url_path',

src/lib/detect.ts

@@ -5,12 +5,13 @@ import ipaddr from 'ipaddr.js';
 import maxmind from 'maxmind';
 import {
   DESKTOP_OS,
-  MOBILE_OS,
   DESKTOP_SCREEN_WIDTH,
-  LAPTOP_SCREEN_WIDTH,
-  MOBILE_SCREEN_WIDTH,
   IP_ADDRESS_HEADERS,
+  LAPTOP_SCREEN_WIDTH,
+  MOBILE_OS,
+  MOBILE_SCREEN_WIDTH,
 } from './constants';
+import { safeDecodeURIComponent } from '@/lib/url';
 
 const MAXMIND = 'maxmind';
 
@@ -124,10 +125,14 @@ export async function getLocation(ip: string = '', headers: Headers, hasPayloadI
   if (!global[MAXMIND]) {
     const dir = path.join(process.cwd(), 'geo');
 
-    global[MAXMIND] = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));
+    global[MAXMIND] = await maxmind.open(
+      process.env.GEOLITE_DB_PATH || path.resolve(dir, 'GeoLite2-City.mmdb'),
+    );
   }
 
-  const result = global[MAXMIND].get(ip);
+  // When the client IP is extracted from headers, sometimes the value includes a port
+  const cleanIp = ip?.split(':')[0];
+  const result = global[MAXMIND].get(cleanIp);
 
   if (result) {
     const country = result.country?.iso_code ?? result?.registered_country?.iso_code;
@@ -146,9 +151,9 @@ export async function getClientInfo(request: Request, payload: Record<string, an
   const userAgent = payload?.userAgent || request.headers.get('user-agent');
   const ip = payload?.ip || getIpAddress(request.headers);
   const location = await getLocation(ip, request.headers, !!payload?.ip);
-  const country = location?.country;
-  const region = location?.region;
-  const city = location?.city;
+  const country = safeDecodeURIComponent(location?.country);
+  const region = safeDecodeURIComponent(location?.region);
+  const city = safeDecodeURIComponent(location?.city);
   const browser = browserName(userAgent);
   const os = detectOS(userAgent) as string;
   const device = getDevice(payload?.screen, os);

src/lib/prisma.ts

@@ -155,6 +155,24 @@ function mapFilter(column: string, operator: string, name: string, type: string
   }
 }
 
+function mapCohortFilter(column: string, operator: string, value: string) {
+  const db = getDatabaseType();
+  const like = db === POSTGRESQL ? 'ilike' : 'like';
+
+  switch (operator) {
+    case OPERATORS.equals:
+      return `${column} = '${value}'`;
+    case OPERATORS.notEquals:
+      return `${column} != '${value}'`;
+    case OPERATORS.contains:
+      return `${column} ${like} '${value}'`;
+    case OPERATORS.doesNotContain:
+      return `${column} not ${like} '${value}'`;
+    default:
+      return '';
+  }
+}
+
 function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {
   const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
     if (column) {
@@ -173,6 +191,43 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):
   return query.join('\n');
 }
 
+function getCohortQuery(websiteId: string, filters: QueryFilters = {}, options: QueryOptions = {}) {
+  const query = filtersToArray(filters, options).reduce(
+    (arr, { name, column, operator, value }) => {
+      if (column) {
+        arr.push(
+          `${arr.length === 0 ? 'where' : 'and'} ${mapCohortFilter(column, operator, value)}`,
+        );
+
+        if (name === 'referrer') {
+          arr.push(`and referrer_domain != hostname`);
+        }
+      }
+
+      return arr;
+    },
+    [],
+  );
+
+  if (query.length > 0) {
+    // add website and date range filters
+    query.push(`and website_event.website_id = '${websiteId}'`);
+    query.push(
+      `and website_event.created_at between '${filters.startDate}'::timestamptz and '${filters.endDate}'::timestamptz`,
+    );
+
+    return `join
+    (select distinct website_event.session_id
+    from website_event
+    join session on session.session_id = website_event.session_id
+    ${query.join('\n')}) cohort
+    on cohort.session_id = website_event.session_id
+    `;
+  }
+
+  return '';
+}
+
 function getDateQuery(filters: QueryFilters = {}) {
   const { startDate, endDate } = filters;
 
@@ -219,6 +274,7 @@ async function parseFilters(
       websiteId,
       startDate: maxDate(filters.startDate, website?.resetAt),
     },
+    cohortQuery: getCohortQuery(websiteId, filters?.cohort),
   };
 }
 

src/lib/request.ts

@@ -1,9 +1,9 @@
 import { z, ZodSchema } from 'zod';
-import { FILTER_COLUMNS } from '@/lib/constants';
+import { FILTER_COLUMNS, FILTER_GROUPS } from '@/lib/constants';
 import { badRequest, unauthorized } from '@/lib/response';
 import { getAllowedUnits, getMinimumUnit } from '@/lib/date';
 import { checkAuth } from '@/lib/auth';
-import { getWebsiteDateRange } from '@/queries';
+import { getWebsiteSegment, getWebsiteDateRange } from '@/queries';
 
 export async function getJsonBody(request: Request) {
   try {
@@ -85,14 +85,28 @@ export async function getRequestDateRange(query: Record<string, any>) {
   };
 }
 
-export function getRequestFilters(query: Record<string, any>) {
-  return Object.keys(FILTER_COLUMNS).reduce((obj, key) => {
+export async function getRequestFilters(query: Record<string, any>, websiteId?: string) {
+  const result: Record<string, any> = {};
+
+  for (const key of Object.keys(FILTER_COLUMNS)) {
     const value = query[key];
-
     if (value !== undefined) {
-      obj[key] = value;
+      result[key] = value;
     }
+  }
 
-    return obj;
-  }, {});
+  for (const key of Object.keys(FILTER_GROUPS)) {
+    const value = query[key];
+    if (value !== undefined) {
+      const segment = await getWebsiteSegment(websiteId, key, value);
+      if (key === 'segment') {
+        // merge filters into result
+        Object.assign(result, segment.parameters);
+      } else {
+        result[key] = segment.parameters;
+      }
+    }
+  }
+
+  return result;
 }

src/lib/schema.ts

@@ -17,6 +17,8 @@ export const filterParams = {
   host: z.string().optional(),
   language: z.string().optional(),
   event: z.string().optional(),
+  segment: z.string().optional(),
+  cohort: z.string().optional(),
 };
 
 export const pagingParams = {
@@ -74,3 +76,5 @@ export const reportParms = {
     value: z.string().optional(),
   }),
 };
+
+export const segmentTypeParam = z.enum(['segment', 'cohort']);

src/lib/types.ts

@@ -158,6 +158,7 @@ export interface QueryFilters {
   event?: string;
   search?: string;
   tag?: string;
+  cohort?: { [key: string]: string };
 }
 
 export interface QueryOptions {