Feat/um 305 unique session ch (#2065)

* Add session_data / session redis to CH.

* Add mysql migration.

lib/clickhouse.ts

@@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { WebsiteMetricFilter } from './types';
 import { FILTER_COLUMNS } from './constants';
 
@@ -74,7 +74,7 @@ function getEventDataFilterQuery(
   params: any,
 ) {
   const query = filters.reduce((ac, cv, i) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 

lib/constants.ts

@@ -42,6 +42,11 @@ export const SESSION_COLUMNS = [
   'city',
 ];
 
+export const COLLECTION_TYPE = {
+  event: 'event',
+  identify: 'identify',
+};
+
 export const FILTER_COLUMNS = {
   url: 'url_path',
   referrer: 'referrer_domain',
@@ -56,7 +61,7 @@ export const EVENT_TYPE = {
   customEvent: 2,
 } as const;
 
-export const EVENT_DATA_TYPE = {
+export const DYNAMIC_DATA_TYPE = {
   string: 1,
   number: 2,
   boolean: 3,

lib/dynamicData.ts

@@ -1,12 +1,12 @@
 import { isValid, parseISO } from 'date-fns';
-import { EVENT_DATA_TYPE } from './constants';
-import { EventDataTypes } from './types';
+import { DYNAMIC_DATA_TYPE } from './constants';
+import { DynamicDataType } from './types';
 
 export function flattenJSON(
   eventData: { [key: string]: any },
-  keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],
+  keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
   parentKey = '',
-): { key: string; value: any; eventDataType: EventDataTypes }[] {
+): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
   return Object.keys(eventData).reduce(
     (acc, key) => {
       const value = eventData[key];
@@ -25,7 +25,7 @@ export function flattenJSON(
   ).keyValues;
 }
 
-export function getEventDataType(value: any): string {
+export function getDynamicDataType(value: any): string {
   let type: string = typeof value;
 
   if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
@@ -36,34 +36,34 @@ export function getEventDataType(value: any): string {
 }
 
 function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
-  const type = getEventDataType(value);
+  const type = getDynamicDataType(value);
 
-  let eventDataType = null;
+  let dynamicDataType = null;
 
   switch (type) {
     case 'number':
-      eventDataType = EVENT_DATA_TYPE.number;
+      dynamicDataType = DYNAMIC_DATA_TYPE.number;
       break;
     case 'string':
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DYNAMIC_DATA_TYPE.string;
       break;
     case 'boolean':
-      eventDataType = EVENT_DATA_TYPE.boolean;
+      dynamicDataType = DYNAMIC_DATA_TYPE.boolean;
       value = value ? 'true' : 'false';
       break;
     case 'date':
-      eventDataType = EVENT_DATA_TYPE.date;
+      dynamicDataType = DYNAMIC_DATA_TYPE.date;
       break;
     case 'object':
-      eventDataType = EVENT_DATA_TYPE.array;
+      dynamicDataType = DYNAMIC_DATA_TYPE.array;
       value = JSON.stringify(value);
       break;
     default:
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DYNAMIC_DATA_TYPE.string;
       break;
   }
 
-  acc.keyValues.push({ key, value, eventDataType });
+  acc.keyValues.push({ key, value, dynamicDataType });
 }
 
 function getKeyName(key, parentKey) {

lib/prisma.ts

@@ -1,7 +1,7 @@
 import prisma from '@umami/prisma-client';
 import moment from 'moment-timezone';
 import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { FILTER_COLUMNS } from './constants';
 
 const MYSQL_DATE_FORMATS = {
@@ -85,7 +85,7 @@ function getEventDataFilterQuery(
   params: any[],
 ) {
   const query = filters.reduce((ac, cv) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 

lib/session.ts

@@ -1,12 +1,11 @@
-import clickhouse from 'lib/clickhouse';
 import { secret, uuid } from 'lib/crypto';
 import { getClientInfo, getJsonBody } from 'lib/detect';
 import { parseToken } from 'next-basics';
 import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
 import { createSession } from 'queries';
 import { validate } from 'uuid';
-import { loadSession, loadWebsite } from './query';
 import cache from './cache';
+import { loadSession, loadWebsite } from './query';
 
 export async function findSession(req: NextApiRequestCollect) {
   const { payload } = getJsonBody<CollectRequestBody>(req);
@@ -46,26 +45,8 @@ export async function findSession(req: NextApiRequestCollect) {
 
   const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
     await getClientInfo(req, payload);
-  const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
-  // Clickhouse does not require session lookup
-  if (clickhouse.enabled) {
-    return {
-      id: sessionId,
-      websiteId,
-      hostname,
-      browser,
-      os,
-      device,
-      screen,
-      language,
-      country,
-      subdivision1,
-      subdivision2,
-      city,
-      ownerId: website.userId,
-    };
-  }
+  const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
   // Find session
   let session = await loadSession(sessionId);

lib/types.ts

@@ -1,18 +1,20 @@
 import { NextApiRequest } from 'next';
-import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
+import { COLLECTION_TYPE, DYNAMIC_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
 
 type ObjectValues<T> = T[keyof T];
 
-export type Roles = ObjectValues<typeof ROLES>;
+export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;
 
-export type EventTypes = ObjectValues<typeof EVENT_TYPE>;
+export type Role = ObjectValues<typeof ROLES>;
 
-export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;
+export type EventType = ObjectValues<typeof EVENT_TYPE>;
 
-export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;
+export type DynamicDataType = ObjectValues<typeof DYNAMIC_DATA_TYPE>;
 
-export interface EventData {
-  [key: string]: number | string | EventData | number[] | string[] | EventData[];
+export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;
+
+export interface DynamicData {
+  [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];
 }
 
 export interface Auth {