Merge pull request #3 from autodity/feature/tracking-batch-data

feat: add batch data for tracking payload

src/lib/types.ts

@@ -52,6 +52,12 @@ export interface DynamicData {
   [key: string]: number | string | number[] | string[];
 }
 
+export interface JsonKeyDynamicData {
+  key: string;
+  value: any;
+  dataType: DynamicDataType;
+}
+
 export interface Auth {
   user?: {
     id: string;

src/pages/api/send.ts

@@ -21,6 +21,7 @@ export interface CollectRequestBody {
   payload: {
     website: string;
     data?: { [key: string]: any };
+    batchData?: Array<{ [key: string]: any }>;
     hostname?: string;
     ip?: string;
     language?: string;
@@ -61,7 +62,8 @@ const schema = {
     payload: yup
       .object()
       .shape({
-        data: yup.object(),
+        data: yup.object().optional(),
+        batchData: yup.array().of(yup.object()).optional(),
         hostname: yup.string().matches(HOSTNAME_REGEX).max(100),
         ip: yup.string().matches(IP_REGEX),
         language: yup.string().max(35),
@@ -90,13 +92,16 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
     }
 
     await useValidate(schema, req, res);
+    if (req.body.payload.batchData && req.body.payload.data) {
+      return badRequest(res, 'cannot send both data and batchData.');
+    }
 
     if (hasBlockedIp(req)) {
       return forbidden(res);
     }
 
     const { type, payload } = req.body;
-    const { url, referrer, name: eventName, data, title } = payload;
+    const { url, referrer, name: eventName, data, title, batchData } = payload;
     const pageTitle = safeDecodeURI(title);
 
     await useSession(req, res);
@@ -141,6 +146,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {
         pageTitle,
         eventName,
         eventData: data,
+        eventBatchData: batchData,
         ...session,
         sessionId: session.id,
       });

src/queries/analytics/events/saveEvent.ts

@@ -18,6 +18,7 @@ export async function saveEvent(args: {
   pageTitle?: string;
   eventName?: string;
   eventData?: any;
+  eventBatchData?: any[];
   hostname?: string;
   browser?: string;
   os?: string;
@@ -47,6 +48,7 @@ async function relationalQuery(data: {
   pageTitle?: string;
   eventName?: string;
   eventData?: any;
+  eventBatchData?: Array<any>;
 }) {
   const {
     websiteId,
@@ -60,6 +62,7 @@ async function relationalQuery(data: {
     eventName,
     eventData,
     pageTitle,
+    eventBatchData,
   } = data;
   const websiteEventId = uuid();
 
@@ -80,7 +83,7 @@ async function relationalQuery(data: {
     },
   });
 
-  if (eventData) {
+  if (eventData || eventBatchData) {
     await saveEventData({
       websiteId,
       sessionId,
@@ -89,6 +92,7 @@ async function relationalQuery(data: {
       urlPath: urlPath?.substring(0, URL_LENGTH),
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
       eventData,
+      eventBatchData,
     });
   }
 
@@ -107,6 +111,7 @@ async function clickhouseQuery(data: {
   pageTitle?: string;
   eventName?: string;
   eventData?: any;
+  eventBatchData?: any[];
   hostname?: string;
   browser?: string;
   os?: string;
@@ -130,6 +135,7 @@ async function clickhouseQuery(data: {
     pageTitle,
     eventName,
     eventData,
+    eventBatchData,
     country,
     subdivision1,
     subdivision2,
@@ -173,7 +179,7 @@ async function clickhouseQuery(data: {
     await insert('website_event', [message]);
   }
 
-  if (eventData) {
+  if (eventData || eventBatchData) {
     await saveEventData({
       websiteId,
       sessionId,
@@ -182,6 +188,7 @@ async function clickhouseQuery(data: {
       urlPath: urlPath?.substring(0, URL_LENGTH),
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
       eventData,
+      eventBatchData,
       createdAt,
     });
   }

src/queries/analytics/events/saveEventData.ts

@@ -6,7 +6,7 @@ import { flattenDynamicData, flattenJSON, getStringValue } from 'lib/data';
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import kafka from 'lib/kafka';
 import prisma from 'lib/prisma';
-import { DynamicData } from 'lib/types';
+import { DynamicData, JsonKeyDynamicData } from 'lib/types';
 
 export async function saveEventData(data: {
   websiteId: string;
@@ -15,7 +15,8 @@ export async function saveEventData(data: {
   visitId?: string;
   urlPath?: string;
   eventName?: string;
-  eventData: DynamicData;
+  eventData?: DynamicData;
+  eventBatchData?: Array<DynamicData>;
   createdAt?: string;
 }) {
   return runQuery({
@@ -27,11 +28,17 @@ export async function saveEventData(data: {
 async function relationalQuery(data: {
   websiteId: string;
   eventId: string;
-  eventData: DynamicData;
+  eventData?: DynamicData;
+  eventBatchData?: Array<DynamicData>;
 }): Promise<Prisma.BatchPayload> {
-  const { websiteId, eventId, eventData } = data;
+  const { websiteId, eventId, eventData, eventBatchData } = data;
 
-  const jsonKeys = flattenJSON(eventData);
+  let jsonKeys: Array<JsonKeyDynamicData> = [];
+  if (eventData) {
+    jsonKeys = flattenJSON(eventData);
+  } else if (eventBatchData) {
+    jsonKeys = eventBatchData.flatMap(d => flattenJSON(d));
+  }
 
   // id, websiteEventId, eventStringValue
   const flattenedData = jsonKeys.map(a => ({
@@ -57,15 +64,31 @@ async function clickhouseQuery(data: {
   visitId?: string;
   urlPath?: string;
   eventName?: string;
-  eventData: DynamicData;
+  eventData?: DynamicData;
+  eventBatchData?: Array<DynamicData>;
   createdAt?: string;
 }) {
-  const { websiteId, sessionId, visitId, eventId, urlPath, eventName, eventData, createdAt } = data;
+  const {
+    websiteId,
+    sessionId,
+    visitId,
+    eventId,
+    urlPath,
+    eventName,
+    eventData,
+    eventBatchData,
+    createdAt,
+  } = data;
 
   const { sendMessages, sendMessage } = kafka;
   const { insert, getUTCString } = clickhouse;
 
-  const jsonKeys = flattenJSON(eventData);
+  let jsonKeys: Array<JsonKeyDynamicData> = [];
+  if (eventData) {
+    jsonKeys = flattenJSON(eventData);
+  } else if (eventBatchData) {
+    jsonKeys = eventBatchData.flatMap(d => flattenJSON(d));
+  }
 
   const messages = jsonKeys.map(({ key, value, dataType }) => {
     return {

src/tracker/index.d.ts

@@ -75,6 +75,7 @@ export type EventProperties = {
    */
   name: string;
   data?: EventData;
+  batchData?: EventData[];
 } & WithRequired<TrackedProperties, 'website'>;
 export type PageViewProperties = WithRequired<TrackedProperties, 'website'>;
 export type CustomEventFunction = (
@@ -125,7 +126,7 @@ export type UmamiTracker = {
      * umami.track('signup-button', { name: 'newsletter', id: 123 });
      * ```
      */
-    (eventName: string, obj: EventData): Promise<string>;
+    (eventName: string, obj: EventData | Array<EventData>): Promise<string>;
 
     /**
      * Tracks a page view with custom properties

src/tracker/index.js

@@ -234,10 +234,21 @@
 
   const track = (obj, data) => {
     if (typeof obj === 'string') {
+      let singleData = undefined;
+      let batchData = undefined;
+      if (typeof data === 'object') {
+        if (Array.isArray(data)) {
+          batchData = data;
+        } else {
+          singleData = data;
+        }
+      }
+
       return send({
         ...getPayload(),
         name: obj,
-        data: typeof data === 'object' ? data : undefined,
+        data: singleData,
+        batchData,
       });
     } else if (typeof obj === 'object') {
       return send(obj);