Feat/um 202 event data new (#1841)

* Add event_data base.

* Add url_path.

* Add eventData back.

* Finish event_data relational.

* resolve comments.

queries/analytics/event/saveEvent.ts

@@ -4,6 +4,7 @@ import kafka from 'lib/kafka';
 import prisma from 'lib/prisma';
 import { uuid } from 'lib/crypto';
 import cache from 'lib/cache';
+import { saveEventData } from '../eventData/saveEventData';
 
 export async function saveEvent(args: {
   id: string;
@@ -12,6 +13,7 @@ export async function saveEvent(args: {
   urlQuery?: string;
   pageTitle?: string;
   eventName?: string;
+  eventData?: any;
   hostname?: string;
   browser?: string;
   os?: string;
@@ -36,12 +38,15 @@ async function relationalQuery(data: {
   urlQuery?: string;
   pageTitle?: string;
   eventName?: string;
+  eventData?: any;
 }) {
-  const { websiteId, id: sessionId, urlPath, urlQuery, eventName, pageTitle } = data;
+  const { websiteId, id: sessionId, urlPath, urlQuery, eventName, eventData, pageTitle } = data;
+  const website = await cache.fetchWebsite(websiteId);
+  const websiteEventId = uuid();
 
-  return prisma.client.websiteEvent.create({
+  const websiteEvent = prisma.client.websiteEvent.create({
     data: {
-      id: uuid(),
+      id: websiteEventId,
       websiteId,
       sessionId,
       urlPath: urlPath?.substring(0, URL_LENGTH),
@@ -51,6 +56,20 @@ async function relationalQuery(data: {
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
     },
   });
+
+  if (eventData) {
+    await saveEventData({
+      websiteId,
+      sessionId,
+      eventId: websiteEventId,
+      revId: website?.revId,
+      urlPath: urlPath?.substring(0, URL_LENGTH),
+      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      eventData,
+    });
+  }
+
+  return websiteEvent;
 }
 
 async function clickhouseQuery(data: {
@@ -60,6 +79,7 @@ async function clickhouseQuery(data: {
   urlQuery?: string;
   pageTitle?: string;
   eventName?: string;
+  eventData?: any;
   hostname?: string;
   browser?: string;
   os?: string;
@@ -78,6 +98,7 @@ async function clickhouseQuery(data: {
     urlQuery,
     pageTitle,
     eventName,
+    eventData,
     country,
     subdivision1,
     subdivision2,
@@ -86,11 +107,13 @@ async function clickhouseQuery(data: {
   } = data;
   const { getDateFormat, sendMessage } = kafka;
   const website = await cache.fetchWebsite(websiteId);
+  const eventId = uuid();
+  const createdAt = getDateFormat(new Date());
 
   const message = {
     website_id: websiteId,
     session_id: sessionId,
-    event_id: uuid(),
+    event_id: eventId,
     country: country ? country : null,
     subdivision1: subdivision1 ? subdivision1 : null,
     subdivision2: subdivision2 ? subdivision2 : null,
@@ -101,11 +124,24 @@ async function clickhouseQuery(data: {
     event_type: EVENT_TYPE.customEvent,
     event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
     rev_id: website?.revId || 0,
-    created_at: getDateFormat(new Date()),
+    created_at: createdAt,
     ...args,
   };
 
   await sendMessage(message, 'event');
 
+  if (eventData) {
+    await saveEventData({
+      websiteId,
+      sessionId,
+      eventId,
+      revId: website?.revId,
+      urlPath: urlPath?.substring(0, URL_LENGTH),
+      eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
+      eventData,
+      createdAt,
+    });
+  }
+
   return data;
 }

queries/analytics/eventData/getEventData.ts

@@ -0,0 +1,117 @@
+import cache from 'lib/cache';
+import clickhouse from 'lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+import { WebsiteEventDataMetric } from 'lib/types';
+
+export async function getEventData(
+  ...args: [
+    websiteId: string,
+    data: {
+      startDate: Date;
+      endDate: Date;
+      eventName: string;
+      urlPath?: string;
+      filters: [
+        {
+          eventKey?: string;
+          eventValue?: string | number | boolean | Date;
+        },
+      ];
+    },
+  ]
+): Promise<WebsiteEventDataMetric[]> {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(
+  websiteId: string,
+  data: {
+    startDate: Date;
+    endDate: Date;
+    timeSeries?: {
+      unit: string;
+      timezone: string;
+    };
+    eventName: string;
+    urlPath?: string;
+    filters: [
+      {
+        eventKey?: string;
+        eventValue?: string | number | boolean | Date;
+      },
+    ];
+  },
+) {
+  const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
+  const { toUuid, rawQuery, getEventDataFilterQuery, getDateQuery } = prisma;
+  const params: any = [websiteId, startDate, endDate, eventName || ''];
+
+  return rawQuery(
+    `select
+        count(*) x
+        ${eventName ? `,event_name eventName` : ''}
+        ${urlPath ? `,url_path urlPath` : ''}
+        ${
+          timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
+        }
+    from event_data
+      ${
+        eventName || urlPath
+          ? 'join website_event on event_data.id = website_event.website_event_id'
+          : ''
+      }
+    where website_id = $1${toUuid()}
+      and created_at between $2 and $3
+      ${eventName ? `and eventName = $4` : ''}
+      ${getEventDataFilterQuery(filters, params)}
+      ${timeSeries ? 'group by t' : ''}`,
+    params,
+  );
+}
+
+async function clickhouseQuery(
+  websiteId: string,
+  data: {
+    startDate: Date;
+    endDate: Date;
+    timeSeries?: {
+      unit: string;
+      timezone: string;
+    };
+    eventName?: string;
+    urlPath?: string;
+    filters: [
+      {
+        eventKey?: string;
+        eventValue?: string | number | boolean | Date;
+      },
+    ];
+  },
+) {
+  const { startDate, endDate, timeSeries, eventName, urlPath, filters } = data;
+  const { rawQuery, getBetweenDates, getDateQuery, getEventDataFilterQuery } = clickhouse;
+  const website = await cache.fetchWebsite(websiteId);
+  const params = { websiteId, revId: website?.revId || 0 };
+
+  return rawQuery(
+    `select
+        count(*) x
+        ${eventName ? `,event_name eventName` : ''}
+        ${urlPath ? `,url_path urlPath` : ''}
+        ${
+          timeSeries ? `,${getDateQuery('created_at', timeSeries.unit, timeSeries.timezone)} t` : ''
+        }
+    from event_data
+    where website_id = {websiteId:UUID}
+      and rev_id = {revId:UInt32}
+      ${eventName ? `and eventName = ${eventName}` : ''}
+      and ${getBetweenDates('created_at', startDate, endDate)}
+      ${getEventDataFilterQuery(filters, params)}
+      ${timeSeries ? 'group by t' : ''}`,
+    params,
+  );
+}

queries/analytics/eventData/saveEventData.ts

@@ -0,0 +1,96 @@
+import { Prisma } from '@prisma/client';
+import { EVENT_DATA_TYPE } from 'lib/constants';
+import { uuid } from 'lib/crypto';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import { flattenJSON } from 'lib/eventData';
+import kafka from 'lib/kafka';
+import prisma from 'lib/prisma';
+import { EventData } from 'lib/types';
+
+export async function saveEventData(args: {
+  websiteId: string;
+  eventId: string;
+  sessionId?: string;
+  revId?: number;
+  urlPath?: string;
+  eventName?: string;
+  eventData: EventData;
+  createdAt?: string;
+}) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(args),
+    [CLICKHOUSE]: () => clickhouseQuery(args),
+  });
+}
+
+async function relationalQuery(data: {
+  websiteId: string;
+  eventId: string;
+  eventData: EventData;
+}): Promise<Prisma.BatchPayload> {
+  const { websiteId, eventId, eventData } = data;
+
+  const jsonKeys = flattenJSON(eventData);
+
+  //id, websiteEventId, eventStringValue
+  const flattendData = jsonKeys.map(a => ({
+    id: uuid(),
+    websiteEventId: eventId,
+    websiteId,
+    eventKey: a.key,
+    eventStringValue:
+      a.eventDataType === EVENT_DATA_TYPE.string ||
+      a.eventDataType === EVENT_DATA_TYPE.boolean ||
+      a.eventDataType === EVENT_DATA_TYPE.array
+        ? a.value
+        : null,
+    eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
+    eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,
+    eventDataType: a.eventDataType,
+  }));
+
+  return prisma.client.eventData.createMany({
+    data: flattendData,
+  });
+}
+
+async function clickhouseQuery(data: {
+  websiteId: string;
+  eventId: string;
+  sessionId?: string;
+  revId?: number;
+  urlPath?: string;
+  eventName?: string;
+  eventData: EventData;
+  createdAt?: string;
+}) {
+  const { websiteId, sessionId, eventId, revId, urlPath, eventName, eventData, createdAt } = data;
+
+  const { getDateFormat, sendMessages } = kafka;
+
+  const jsonKeys = flattenJSON(eventData);
+
+  const messages = jsonKeys.map(a => ({
+    website_id: websiteId,
+    session_id: sessionId,
+    event_id: eventId,
+    rev_id: revId,
+    url_path: urlPath,
+    event_name: eventName,
+    event_key: a.key,
+    event_string_value:
+      a.eventDataType === EVENT_DATA_TYPE.string ||
+      a.eventDataType === EVENT_DATA_TYPE.boolean ||
+      a.eventDataType === EVENT_DATA_TYPE.array
+        ? a.value
+        : null,
+    event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,
+    event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null,
+    event_data_type: a.eventDataType,
+    created_at: createdAt,
+  }));
+
+  await sendMessages(messages, 'event_data');
+
+  return data;
+}

queries/analytics/pageview/savePageView.ts

@@ -109,6 +109,7 @@ async function clickhouseQuery(data: {
   const message = {
     website_id: websiteId,
     session_id: sessionId,
+    event_id: uuid(),
     rev_id: website?.revId || 0,
     country: country ? country : null,
     subdivision1: subdivision1 ? subdivision1 : null,

queries/index.js

@@ -4,6 +4,7 @@ export * from './admin/user';
 export * from './admin/website';
 export * from './analytics/event/getEventMetrics';
 export * from './analytics/event/getEvents';
+export * from './analytics/eventData/getEventData';
 export * from './analytics/event/saveEvent';
 export * from './analytics/pageview/getPageviewMetrics';
 export * from './analytics/pageview/getPageviews';