Merge branch 'dev' into jajaja

.eslintrc.json

@@ -33,6 +33,7 @@
     "react/prop-types": "off",
     "import/no-anonymous-default-export": "off",
     "import/no-named-as-default": "off",
+    "css-modules/no-unused-class": "off",
     "@next/next/no-img-element": "off",
     "@typescript-eslint/no-empty-function": "off",
     "@typescript-eslint/no-explicit-any": "off",

next.config.js

@@ -59,15 +59,29 @@ const trackerHeaders = [
   },
 ];
 
+const apiHeaders = [
+  {
+    key: 'Access-Control-Allow-Origin',
+    value: '*'
+  },
+  {
+    key: 'Access-Control-Allow-Headers',
+    value: '*'
+  },
+  {
+    key: 'Access-Control-Allow-Methods',
+    value: 'GET, DELETE, POST, PUT'
+  },
+  {
+    key: 'Access-Control-Max-Age',
+    value: corsMaxAge || '86400'
+  },
+];
+
 const headers = [
   {
     source: '/api/:path*',
-    headers: [
+    headers: apiHeaders
-      { key: 'Access-Control-Allow-Origin', value: '*' },
-      { key: 'Access-Control-Allow-Headers', value: '*' },
-      { key: 'Access-Control-Allow-Methods', value: 'GET, DELETE, POST, PUT' },
-      { key: 'Access-Control-Max-Age', value: corsMaxAge || '86400' },
-    ],
   },
   {
     source: '/:path*',
@@ -89,6 +103,11 @@ if (trackerScriptURL) {
 }
 
 if (collectApiEndpoint) {
+  headers.push({
+    source: collectApiEndpoint,
+    headers: apiHeaders,
+  });
+
   rewrites.push({
     source: collectApiEndpoint,
     destination: '/api/send',

scripts/check-db.js

@@ -82,9 +82,11 @@ async function checkV1Tables() {
 }
 
 async function applyMigration() {
-  console.log(execSync('prisma migrate deploy').toString());
+  if (!process.env.SKIP_DB_MIGRATION) {
+    console.log(execSync('prisma migrate deploy').toString());
 
-  success('Database is up to date.');
+    success('Database is up to date.');
+  }
 }
 
 (async () => {

src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx

@@ -71,9 +71,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
 
     if (__type === TYPE_EVENT) {
       return formatMessage(messages.eventLog, {
-        event: <b>{eventName || formatMessage(labels.unknown)}</b>,
+        event: <b key="b">{eventName || formatMessage(labels.unknown)}</b>,
         url: (
           <a
+            key="a"
             href={`//${website?.domain}${url}`}
             className={styles.link}
             target="_blank"
@@ -100,10 +101,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) {
 
     if (__type === TYPE_SESSION) {
       return formatMessage(messages.visitorLog, {
-        country: <b>{countryNames[country] || formatMessage(labels.unknown)}</b>,
+        country: <b key="country">{countryNames[country] || formatMessage(labels.unknown)}</b>,
-        browser: <b>{BROWSERS[browser]}</b>,
+        browser: <b key="browser">{BROWSERS[browser]}</b>,
-        os: <b>{OS_NAMES[os] || os}</b>,
+        os: <b key="os">{OS_NAMES[os] || os}</b>,
-        device: <b>{formatMessage(labels[device] || labels.unknown)}</b>,
+        device: <b key="device">{formatMessage(labels[device] || labels.unknown)}</b>,
       });
     }
   };

src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx

@@ -62,10 +62,10 @@ export function SessionsWeekly({ websiteId }: { websiteId: string }) {
                 <div className={styles.header}>
                   {format(getDayOfWeekAsDate(index), 'EEE', { locale: dateLocale })}
                 </div>
-                {day?.map((hour: number) => {
+                {day?.map((hour: number, j) => {
                   const pct = hour / max;
                   return (
-                    <div key={hour} className={classNames(styles.cell)}>
+                    <div key={j} className={classNames(styles.cell)}>
                       {hour > 0 && (
                         <TooltipPopup
                           label={`${formatMessage(labels.visitors)}: ${hour}`}

src/app/api/batch/route.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod';
+import * as send from '@/app/api/send/route';
+import { parseRequest } from '@/lib/request';
+import { json, serverError } from '@/lib/response';
+
+const schema = z.array(z.object({}).passthrough());
+
+export async function POST(request: Request) {
+  try {
+    const { body, error } = await parseRequest(request, schema, { skipAuth: true });
+
+    if (error) {
+      return error();
+    }
+
+    const errors = [];
+
+    let index = 0;
+    for (const data of body) {
+      const newRequest = new Request(request, { body: JSON.stringify(data) });
+      const response = await send.POST(newRequest);
+
+      if (!response.ok) {
+        errors.push({ index, response: await response.json() });
+      }
+
+      index++;
+    }
+
+    return json({
+      size: body.length,
+      processed: body.length - errors.length,
+      errors: errors.length,
+      details: errors,
+    });
+  } catch (e) {
+    return serverError(e);
+  }
+}

src/app/api/send/route.ts

@@ -1,31 +1,33 @@
 import { z } from 'zod';
 import { isbot } from 'isbot';
-import { createToken, parseToken } from '@/lib/jwt';
+import { startOfHour, startOfMonth } from 'date-fns';
 import clickhouse from '@/lib/clickhouse';
 import { parseRequest } from '@/lib/request';
 import { badRequest, json, forbidden, serverError } from '@/lib/response';
 import { fetchSession, fetchWebsite } from '@/lib/load';
 import { getClientInfo, hasBlockedIp } from '@/lib/detect';
-import { secret, uuid, visitSalt } from '@/lib/crypto';
+import { createToken, parseToken } from '@/lib/jwt';
-import { COLLECTION_TYPE, DOMAIN_REGEX } from '@/lib/constants';
+import { secret, uuid, hash } from '@/lib/crypto';
+import { COLLECTION_TYPE } from '@/lib/constants';
+import { anyObjectParam, urlOrPathParam } from '@/lib/schema';
 import { createSession, saveEvent, saveSessionData } from '@/queries';
-import { urlOrPathParam } from '@/lib/schema';
 
 const schema = z.object({
   type: z.enum(['event', 'identify']),
   payload: z.object({
     website: z.string().uuid(),
-    data: z.object({}).passthrough().optional(),
+    data: anyObjectParam.optional(),
-    hostname: z.string().regex(DOMAIN_REGEX).max(100).optional(),
+    hostname: z.string().max(100).optional(),
     language: z.string().max(35).optional(),
     referrer: urlOrPathParam.optional(),
     screen: z.string().max(11).optional(),
     title: z.string().optional(),
-    url: urlOrPathParam,
+    url: urlOrPathParam.optional(),
     name: z.string().max(50).optional(),
     tag: z.string().max(50).optional(),
     ip: z.string().ip().optional(),
     userAgent: z.string().optional(),
+    timestamp: z.coerce.number().int().optional(),
   }),
 });
 
@@ -55,6 +57,7 @@ export async function POST(request: Request) {
       data,
       title,
       tag,
+      timestamp,
     } = payload;
 
     // Cache check
@@ -87,7 +90,13 @@ export async function POST(request: Request) {
       return forbidden();
     }
 
-    const sessionId = uuid(websiteId, ip, userAgent);
+    const createdAt = timestamp ? new Date(timestamp * 1000) : new Date();
+    const now = Math.floor(new Date().getTime() / 1000);
+
+    const sessionSalt = hash(startOfMonth(createdAt).toUTCString());
+    const visitSalt = hash(startOfHour(createdAt).toUTCString());
+
+    const sessionId = uuid(websiteId, ip, userAgent, sessionSalt);
 
     // Find session
     if (!clickhouse.enabled && !cache?.sessionId) {
@@ -119,13 +128,12 @@ export async function POST(request: Request) {
     }
 
     // Visit info
-    const now = Math.floor(new Date().getTime() / 1000);
+    let visitId = cache?.visitId || uuid(sessionId, visitSalt);
-    let visitId = cache?.visitId || uuid(sessionId, visitSalt());
     let iat = cache?.iat || now;
 
     // Expire visit after 30 minutes
-    if (now - iat > 1800) {
+    if (!timestamp && now - iat > 1800) {
-      visitId = uuid(sessionId, visitSalt());
+      visitId = uuid(sessionId, visitSalt);
       iat = now;
     }
 
@@ -179,6 +187,7 @@ export async function POST(request: Request) {
         subdivision2,
         city,
         tag,
+        createdAt,
       });
     }
 
@@ -191,12 +200,13 @@ export async function POST(request: Request) {
         websiteId,
         sessionId,
         sessionData: data,
+        createdAt,
       });
     }
 
     const token = createToken({ websiteId, sessionId, visitId, iat }, secret());
 
-    return json({ cache: token });
+    return json({ cache: token, sessionId, visitId });
   } catch (e) {
     return serverError(e);
   }

src/app/api/users/route.ts

@@ -8,6 +8,7 @@ import { createUser, getUserByUsername } from '@/queries';
 
 export async function POST(request: Request) {
   const schema = z.object({
+    id: z.string().uuid().optional(),
     username: z.string().max(255),
     password: z.string(),
     role: z.string().regex(/admin|user|view-only/i),
@@ -23,7 +24,7 @@ export async function POST(request: Request) {
     return unauthorized();
   }
 
-  const { username, password, role } = body;
+  const { id, username, password, role } = body;
 
   const existingUser = await getUserByUsername(username, { showDeleted: true });
 
@@ -32,7 +33,7 @@ export async function POST(request: Request) {
   }
 
   const user = await createUser({
-    id: uuid(),
+    id: id || uuid(),
     username,
     password: hashPassword(password),
     role: role ?? ROLES.user,

src/lib/crypto.ts

@@ -1,5 +1,4 @@
 import crypto from 'crypto';
-import { startOfHour, startOfMonth } from 'date-fns';
 import prand from 'pure-rand';
 import { v4, v5 } from 'uuid';
 
@@ -77,20 +76,8 @@ export function secret() {
   return hash(process.env.APP_SECRET || process.env.DATABASE_URL);
 }
 
-export function salt() {
-  const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString());
-
-  return hash(secret(), ROTATING_SALT);
-}
-
-export function visitSalt() {
-  const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString());
-
-  return hash(secret(), ROTATING_SALT);
-}
-
 export function uuid(...args: any) {
   if (!args.length) return v4();
 
-  return v5(hash(...args, salt()), v5.DNS);
+  return v5(hash(...args, secret()), v5.DNS);
 }

src/lib/prisma.ts

@@ -192,7 +192,9 @@ async function parseFilters(
   options: QueryOptions = {},
 ) {
   const website = await fetchWebsite(websiteId);
-  const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key));
+  const joinSession = Object.keys(filters).find(key =>
+    ['referrer', ...SESSION_COLUMNS].includes(key),
+  );
 
   return {
     joinSession:

src/lib/request.ts

@@ -1,4 +1,4 @@
-import { ZodObject } from 'zod';
+import { ZodSchema } from 'zod';
 import { FILTER_COLUMNS } from '@/lib/constants';
 import { badRequest, unauthorized } from '@/lib/response';
 import { getAllowedUnits, getMinimumUnit } from '@/lib/date';
@@ -15,7 +15,7 @@ export async function getJsonBody(request: Request) {
 
 export async function parseRequest(
   request: Request,
-  schema?: ZodObject<any>,
+  schema?: ZodSchema,
   options?: { skipAuth: boolean },
 ): Promise<any> {
   const url = new URL(request.url);

src/lib/schema.ts

@@ -36,6 +36,8 @@ export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value),
 
 export const roleParam = z.enum(['team-member', 'team-view-only', 'team-manager']);
 
+export const anyObjectParam = z.object({}).passthrough();
+
 export const urlOrPathParam = z.string().refine(
   value => {
     try {

src/queries/sql/events/saveEvent.ts

@@ -29,6 +29,7 @@ export async function saveEvent(args: {
   subdivision2?: string;
   city?: string;
   tag?: string;
+  createdAt?: Date;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(args),
@@ -49,6 +50,7 @@ async function relationalQuery(data: {
   eventName?: string;
   eventData?: any;
   tag?: string;
+  createdAt?: Date;
 }) {
   const {
     websiteId,
@@ -63,6 +65,7 @@ async function relationalQuery(data: {
     eventData,
     pageTitle,
     tag,
+    createdAt,
   } = data;
   const websiteEventId = uuid();
 
@@ -81,6 +84,7 @@ async function relationalQuery(data: {
       eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
       eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
       tag,
+      createdAt,
     },
   });
 
@@ -92,6 +96,7 @@ async function relationalQuery(data: {
       urlPath: urlPath?.substring(0, URL_LENGTH),
       eventName: eventName?.substring(0, EVENT_NAME_LENGTH),
       eventData,
+      createdAt,
     });
   }
 
@@ -121,6 +126,7 @@ async function clickhouseQuery(data: {
   subdivision2?: string;
   city?: string;
   tag?: string;
+  createdAt?: Date;
 }) {
   const {
     websiteId,
@@ -139,12 +145,12 @@ async function clickhouseQuery(data: {
     subdivision2,
     city,
     tag,
+    createdAt,
     ...args
   } = data;
   const { insert, getUTCString } = clickhouse;
   const { sendMessage } = kafka;
   const eventId = uuid();
-  const createdAt = getUTCString();
 
   const message = {
     ...args,
@@ -170,7 +176,7 @@ async function clickhouseQuery(data: {
     event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
     event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
     tag: tag,
-    created_at: createdAt,
+    created_at: getUTCString(createdAt),
   };
 
   if (kafka.enabled) {

src/queries/sql/events/saveEventData.ts

@@ -15,7 +15,7 @@ export async function saveEventData(data: {
   urlPath?: string;
   eventName?: string;
   eventData: DynamicData;
-  createdAt?: string;
+  createdAt?: Date;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(data),
@@ -27,8 +27,9 @@ async function relationalQuery(data: {
   websiteId: string;
   eventId: string;
   eventData: DynamicData;
+  createdAt?: Date;
 }): Promise<Prisma.BatchPayload> {
-  const { websiteId, eventId, eventData } = data;
+  const { websiteId, eventId, eventData, createdAt } = data;
 
   const jsonKeys = flattenJSON(eventData);
 
@@ -42,6 +43,7 @@ async function relationalQuery(data: {
     numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
     dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
     dataType: a.dataType,
+    createdAt,
   }));
 
   return prisma.client.eventData.createMany({
@@ -56,7 +58,7 @@ async function clickhouseQuery(data: {
   urlPath?: string;
   eventName?: string;
   eventData: DynamicData;
-  createdAt?: string;
+  createdAt?: Date;
 }) {
   const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
 
@@ -77,7 +79,7 @@ async function clickhouseQuery(data: {
       string_value: getStringValue(value, dataType),
       number_value: dataType === DATA_TYPE.number ? value : null,
       date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
-      created_at: createdAt,
+      created_at: getUTCString(createdAt),
     };
   });
 

src/queries/sql/sessions/saveSessionData.ts

@@ -11,6 +11,7 @@ export async function saveSessionData(data: {
   websiteId: string;
   sessionId: string;
   sessionData: DynamicData;
+  createdAt?: Date;
 }) {
   return runQuery({
     [PRISMA]: () => relationalQuery(data),
@@ -22,9 +23,10 @@ export async function relationalQuery(data: {
   websiteId: string;
   sessionId: string;
   sessionData: DynamicData;
+  createdAt?: Date;
 }) {
   const { client } = prisma;
-  const { websiteId, sessionId, sessionData } = data;
+  const { websiteId, sessionId, sessionData, createdAt } = data;
 
   const jsonKeys = flattenJSON(sessionData);
 
@@ -37,6 +39,7 @@ export async function relationalQuery(data: {
     numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
     dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
     dataType: a.dataType,
+    createdAt,
   }));
 
   const existing = await client.sessionData.findMany({
@@ -77,12 +80,12 @@ async function clickhouseQuery(data: {
   websiteId: string;
   sessionId: string;
   sessionData: DynamicData;
+  createdAt?: Date;
 }) {
-  const { websiteId, sessionId, sessionData } = data;
+  const { websiteId, sessionId, sessionData, createdAt } = data;
 
   const { insert, getUTCString } = clickhouse;
   const { sendMessage } = kafka;
-  const createdAt = getUTCString();
 
   const jsonKeys = flattenJSON(sessionData);
 
@@ -95,7 +98,7 @@ async function clickhouseQuery(data: {
       string_value: getStringValue(value, dataType),
       number_value: dataType === DATA_TYPE.number ? value : null,
       date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null,
-      created_at: createdAt,
+      created_at: getUTCString(createdAt),
     };
   });
 

src/tracker/index.js

@@ -1,11 +1,12 @@
 (window => {
   const {
     screen: { width, height },
-    navigator: { language },
+    navigator: { language, doNotTrack: ndnt, msDoNotTrack: msdnt },
     location,
     document,
     history,
     top,
+    doNotTrack,
   } = window;
   const { hostname, href, origin } = location;
   const { currentScript, referrer } = document;
@@ -21,6 +22,7 @@
   const hostUrl = attr(_data + 'host-url');
   const tag = attr(_data + 'tag');
   const autoTrack = attr(_data + 'auto-track') !== _false;
+  const dnt = attr(_data + 'do-not-track') === _true;
   const excludeSearch = attr(_data + 'exclude-search') === _true;
   const excludeHash = attr(_data + 'exclude-hash') === _true;
   const domain = attr(_data + 'domains') || '';
@@ -46,6 +48,11 @@
     tag: tag ? tag : undefined,
   });
 
+  const hasDoNotTrack = () => {
+    const dnt = doNotTrack || ndnt || msdnt;
+    return dnt === 1 || dnt === '1' || dnt === 'yes';
+  };
+
   /* Event handlers */
 
   const handlePush = (state, title, url) => {
@@ -182,7 +189,8 @@
     disabled ||
     !website ||
     (localStorage && localStorage.getItem('umami.disabled')) ||
-    (domain && !domains.includes(hostname));
+    (domain && !domains.includes(hostname)) ||
+    (dnt && hasDoNotTrack());
 
   const send = async (payload, type = 'event') => {
     if (trackingDisabled()) return;