Merge branch 'dev' into analytics

Dockerfile

@@ -41,7 +41,7 @@ RUN set -x \
     && apk add --no-cache curl
 
 # Script dependencies
-RUN pnpm add npm-run-all dotenv prisma@6.1.0
+RUN pnpm add npm-run-all dotenv prisma@6.7.0
 
 # Permissions for prisma
 RUN chown -R nextjs:nodejs node_modules/.pnpm/

README.md

@@ -64,10 +64,10 @@ mysql://username:mypassword@localhost:3306/mydb
 ### Build the Application
 
 ```bash
-npm build
+npm run build
 ```
 
-*The build step will create tables in your database if you are installing for the first time. It will also create a login user with username **admin** and password **umami**.*
+_The build step will create tables in your database if you are installing for the first time. It will also create a login user with username **admin** and password **umami**._
 
 ### Start the Application
 
@@ -75,7 +75,7 @@ npm build
 npm run start
 ```
 
-*By default, this will launch the application on `http://localhost:3000`. You will need to either [proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) requests from your web server or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly.*
+_By default, this will launch the application on `http://localhost:3000`. You will need to either [proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/) requests from your web server or change the [port](https://nextjs.org/docs/api-reference/cli#production) to serve the application directly._
 
 ---
 

package.json

@@ -162,7 +162,6 @@
     "postcss-flexbugs-fixes": "^5.0.2",
     "postcss-import": "^15.1.0",
     "postcss-preset-env": "7.8.3",
-    "postcss-rtlcss": "^4.0.1",
     "prettier": "^2.6.2",
     "prompts": "2.4.2",
     "rollup": "^3.28.0",

pnpm-lock.yaml

@@ -291,9 +291,6 @@ importers:
       postcss-preset-env:
         specifier: 7.8.3
         version: 7.8.3(postcss@8.5.3)
-      postcss-rtlcss:
-        specifier: ^4.0.1
-        version: 4.0.9(postcss@8.5.3)
       prettier:
         specifier: ^2.6.2
         version: 2.8.8
@@ -346,8 +343,6 @@ importers:
         specifier: ^5.5.3
         version: 5.8.3
 
-  src/generated/prisma: {}
-
 packages:
 
   '@ampproject/remapping@2.3.0':
@@ -5513,12 +5508,6 @@ packages:
   postcss-resolve-nested-selector@0.1.6:
     resolution: {integrity: sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==}
 
-  postcss-rtlcss@4.0.9:
-    resolution: {integrity: sha512-dCNKEf+FgTv+EA3XI8ysg2RnpS5s3/iZmU+9qpCNFxHU/BhK+4hz7jyCsCAfo0CLnDrMPtaQENhwb+EGm1wh7Q==}
-    engines: {node: '>=18.0.0'}
-    peerDependencies:
-      postcss: ^8.4.21
-
   postcss-safe-parser@6.0.0:
     resolution: {integrity: sha512-FARHN8pwH+WiS2OPCxJI8FuRJpTVnn6ZNFiqAM2aeW2LwTHWWmWgIyKC6cUo0L8aeKiF/14MNvnpls6R2PBeMQ==}
     engines: {node: '>=12.0'}
@@ -5898,11 +5887,6 @@ packages:
     engines: {node: '>=14.18.0', npm: '>=8.0.0'}
     hasBin: true
 
-  rtlcss@4.1.1:
-    resolution: {integrity: sha512-/oVHgBtnPNcggP2aVXQjSy6N1mMAfHg4GSag0QtZBlD5bdDgAHwr4pydqJGd+SUCu9260+Pjqbjwtvu7EMH1KQ==}
-    engines: {node: '>=12.0.0'}
-    hasBin: true
-
   run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
 
@@ -12588,11 +12572,6 @@ snapshots:
 
   postcss-resolve-nested-selector@0.1.6: {}
 
-  postcss-rtlcss@4.0.9(postcss@8.5.3):
-    dependencies:
-      postcss: 8.5.3
-      rtlcss: 4.1.1
-
   postcss-safe-parser@6.0.0(postcss@8.5.3):
     dependencies:
       postcss: 8.5.3
@@ -13008,13 +12987,6 @@ snapshots:
     optionalDependencies:
       fsevents: 2.3.3
 
-  rtlcss@4.1.1:
-    dependencies:
-      escalade: 3.2.0
-      picocolors: 1.1.1
-      postcss: 8.5.3
-      strip-json-comments: 3.1.1
-
   run-parallel@1.2.0:
     dependencies:
       queue-microtask: 1.2.3

scripts/set-routes-manifest.js

@@ -7,8 +7,10 @@ const routesManifestPath = path.resolve(__dirname, '../.next/routes-manifest.jso
 const originalPath = path.resolve(__dirname, '../.next/routes-manifest-orig.json');
 const originalManifest = require(originalPath);
 
-const API_PATH = '/api/:path*';
-const TRACKER_SCRIPT = '/script.js';
+const basePath = originalManifest.basePath;
+
+const API_PATH = basePath + '/api/:path*';
+const TRACKER_SCRIPT = basePath + '/script.js';
 
 const collectApiEndpoint = process.env.COLLECT_API_ENDPOINT;
 const trackerScriptName = process.env.TRACKER_SCRIPT_NAME;
@@ -20,14 +22,16 @@ if (collectApiEndpoint) {
   const apiRoute = originalManifest.headers.find(route => route.source === API_PATH);
   const routeRegex = new RegExp(apiRoute.regex);
 
+  const normalizedSource = basePath + collectApiEndpoint;
+
   rewrites.push({
-    source: collectApiEndpoint,
-    destination: '/api/send',
+    source: normalizedSource,
+    destination: basePath + '/api/send',
   });
 
-  if (!routeRegex.test(collectApiEndpoint)) {
+  if (!routeRegex.test(normalizedSource)) {
     headers.push({
-      source: collectApiEndpoint,
+      source: normalizedSource,
       headers: apiRoute.headers,
     });
   }
@@ -40,7 +44,7 @@ if (trackerScriptName) {
 
   if (names) {
     names.forEach(name => {
-      const normalizedSource = `/${name.replace(/^\/+/, '')}`;
+      const normalizedSource = `${basePath}/${name.replace(/^\/+/, '')}`;
 
       rewrites.push({
         source: normalizedSource,

src/app/api/send/route.ts

@@ -187,26 +187,19 @@ export async function POST(request: Request) {
         websiteId,
         sessionId,
         visitId,
+        createdAt,
+
+        // Page
+        pageTitle: safeDecodeURIComponent(title),
+        hostname: hostname || urlDomain,
         urlPath: safeDecodeURI(urlPath),
         urlQuery,
-        utmSource,
-        utmMedium,
-        utmCampaign,
-        utmContent,
-        utmTerm,
         referrerPath: safeDecodeURI(referrerPath),
         referrerQuery,
         referrerDomain,
-        pageTitle: safeDecodeURIComponent(title),
-        gclid,
-        fbclid,
-        msclkid,
-        ttclid,
-        lifatid,
-        twclid,
-        eventName: name,
-        eventData: data,
-        hostname: hostname || urlDomain,
+
+        // Session
+        distinctId: id,
         browser,
         os,
         device,
@@ -215,24 +208,39 @@ export async function POST(request: Request) {
         country,
         region,
         city,
+
+        // Events
+        eventName: name,
+        eventData: data,
         tag,
-        distinctId: id,
-        createdAt,
+
+        // UTM
+        utmSource,
+        utmMedium,
+        utmCampaign,
+        utmContent,
+        utmTerm,
+
+        // Click IDs
+        gclid,
+        fbclid,
+        msclkid,
+        ttclid,
+        lifatid,
+        twclid,
       });
     }
 
     if (type === COLLECTION_TYPE.identify) {
-      if (!data) {
-        return badRequest('Data required.');
+      if (data) {
+        await saveSessionData({
+          websiteId,
+          sessionId,
+          sessionData: data,
+          distinctId: id,
+          createdAt,
+        });
       }
-
-      await saveSessionData({
-        websiteId,
-        sessionId,
-        sessionData: data,
-        distinctId: id,
-        createdAt,
-      });
     }
 
     const token = createToken({ websiteId, sessionId, visitId, iat }, secret());

src/lib/charts.ts

@@ -11,7 +11,7 @@ export function renderDateLabels(unit: string, locale: string) {
 
     switch (unit) {
       case 'minute':
-        return formatDate(d, 'p', locale).split(' ')[0];
+        return formatDate(d, 'h:mm', locale);
       case 'hour':
         return formatDate(d, 'p', locale);
       case 'day':

src/queries/sql/events/saveEvent.ts

@@ -6,30 +6,23 @@ import prisma from '@/lib/prisma';
 import { uuid } from '@/lib/crypto';
 import { saveEventData } from './saveEventData';
 
-export async function saveEvent(args: {
+export interface SaveEventArgs {
   websiteId: string;
   sessionId: string;
   visitId: string;
+  createdAt?: Date;
+
+  // Page
+  pageTitle?: string;
+  hostname?: string;
   urlPath: string;
   urlQuery?: string;
-  utmSource?: string;
-  utmMedium?: string;
-  utmCampaign?: string;
-  utmContent?: string;
-  utmTerm?: string;
   referrerPath?: string;
   referrerQuery?: string;
   referrerDomain?: string;
-  pageTitle?: string;
-  gclid?: string;
-  fbclid?: string;
-  msclkid?: string;
-  ttclid?: string;
-  lifatid?: string;
-  twclid?: string;
-  eventName?: string;
-  eventData?: any;
-  hostname?: string;
+
+  // Session
+  distinctId?: string;
   browser?: string;
   os?: string;
   device?: string;
@@ -38,73 +31,65 @@ export async function saveEvent(args: {
   country?: string;
   region?: string;
   city?: string;
-  tag?: string;
-  distinctId?: string;
-  createdAt?: Date;
-}) {
-  return runQuery({
-    [PRISMA]: () => relationalQuery(args),
-    [CLICKHOUSE]: () => clickhouseQuery(args),
-  });
-}
 
-async function relationalQuery(data: {
-  websiteId: string;
-  sessionId: string;
-  visitId: string;
-  urlPath: string;
-  urlQuery?: string;
+  // Events
+  eventName?: string;
+  eventData?: any;
+  tag?: string;
+
+  // UTM
   utmSource?: string;
   utmMedium?: string;
   utmCampaign?: string;
   utmContent?: string;
   utmTerm?: string;
-  referrerPath?: string;
-  referrerQuery?: string;
-  referrerDomain?: string;
+
+  // Click IDs
   gclid?: string;
   fbclid?: string;
   msclkid?: string;
   ttclid?: string;
   lifatid?: string;
   twclid?: string;
-  pageTitle?: string;
-  eventName?: string;
-  eventData?: any;
-  tag?: string;
-  hostname?: string;
-  createdAt?: Date;
-}) {
-  const {
-    websiteId,
-    sessionId,
-    visitId,
-    urlPath,
-    urlQuery,
-    utmSource,
-    utmMedium,
-    utmCampaign,
-    utmContent,
-    utmTerm,
-    referrerPath,
-    referrerQuery,
-    referrerDomain,
-    eventName,
-    eventData,
-    pageTitle,
-    gclid,
-    fbclid,
-    msclkid,
-    ttclid,
-    lifatid,
-    twclid,
-    tag,
-    hostname,
-    createdAt,
-  } = data;
+}
+
+export async function saveEvent(args: SaveEventArgs) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(args),
+    [CLICKHOUSE]: () => clickhouseQuery(args),
+  });
+}
+
+async function relationalQuery({
+  websiteId,
+  sessionId,
+  visitId,
+  createdAt,
+  pageTitle,
+  tag,
+  hostname,
+  urlPath,
+  urlQuery,
+  referrerPath,
+  referrerQuery,
+  referrerDomain,
+  eventName,
+  eventData,
+  utmSource,
+  utmMedium,
+  utmCampaign,
+  utmContent,
+  utmTerm,
+  gclid,
+  fbclid,
+  msclkid,
+  ttclid,
+  lifatid,
+  twclid,
+}: SaveEventArgs) {
   const websiteEventId = uuid();
 
-  const websiteEvent = prisma.client.websiteEvent.create({
+  await prisma.client.websiteEvent.create({
     data: {
       id: websiteEventId,
       websiteId,
@@ -146,83 +131,49 @@ async function relationalQuery(data: {
       createdAt,
     });
   }
-
-  return websiteEvent;
 }
 
-async function clickhouseQuery(data: {
-  websiteId: string;
-  sessionId: string;
-  visitId: string;
-  urlPath: string;
-  urlQuery?: string;
-  utmSource?: string;
-  utmMedium?: string;
-  utmCampaign?: string;
-  utmContent?: string;
-  utmTerm?: string;
-  referrerPath?: string;
-  referrerQuery?: string;
-  referrerDomain?: string;
-  pageTitle?: string;
-  gclid?: string;
-  fbclid?: string;
-  msclkid?: string;
-  ttclid?: string;
-  lifatid?: string;
-  twclid?: string;
-  eventName?: string;
-  eventData?: any;
-  hostname?: string;
-  browser?: string;
-  os?: string;
-  device?: string;
-  screen?: string;
-  language?: string;
-  country?: string;
-  region?: string;
-  city?: string;
-  tag?: string;
-  distinctId?: string;
-  createdAt?: Date;
-}) {
-  const {
-    websiteId,
-    sessionId,
-    visitId,
-    urlPath,
-    urlQuery,
-    utmSource,
-    utmMedium,
-    utmCampaign,
-    utmContent,
-    utmTerm,
-    referrerPath,
-    referrerQuery,
-    referrerDomain,
-    gclid,
-    fbclid,
-    msclkid,
-    ttclid,
-    lifatid,
-    twclid,
-    pageTitle,
-    eventName,
-    eventData,
-    country,
-    region,
-    city,
-    tag,
-    distinctId,
-    createdAt,
-    ...args
-  } = data;
+async function clickhouseQuery({
+  websiteId,
+  sessionId,
+  visitId,
+  distinctId,
+  createdAt,
+  pageTitle,
+  browser,
+  os,
+  device,
+  screen,
+  language,
+  country,
+  region,
+  city,
+  tag,
+  hostname,
+  urlPath,
+  urlQuery,
+  referrerPath,
+  referrerQuery,
+  referrerDomain,
+  eventName,
+  eventData,
+  utmSource,
+  utmMedium,
+  utmCampaign,
+  utmContent,
+  utmTerm,
+  gclid,
+  fbclid,
+  msclkid,
+  ttclid,
+  lifatid,
+  twclid,
+}: SaveEventArgs) {
   const { insert, getUTCString } = clickhouse;
   const { sendMessage } = kafka;
   const eventId = uuid();
 
   const message = {
-    ...args,
     website_id: websiteId,
     session_id: sessionId,
     visit_id: visitId,
@@ -252,6 +203,12 @@ async function clickhouseQuery(data: {
     tag: tag,
     distinct_id: distinctId,
     created_at: getUTCString(createdAt),
+    browser,
+    os,
+    device,
+    screen,
+    language,
+    hostname,
   };
 
   if (kafka.enabled) {
@@ -271,6 +228,4 @@ async function clickhouseQuery(data: {
       createdAt,
     });
   }
-
-  return data;
 }

src/queries/sql/events/saveEventData.ts

@@ -1,4 +1,3 @@
-import { Prisma } from '@prisma/client';
 import { DATA_TYPE } from '@/lib/constants';
 import { uuid } from '@/lib/crypto';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
@@ -8,7 +7,7 @@ import kafka from '@/lib/kafka';
 import prisma from '@/lib/prisma';
 import { DynamicData } from '@/lib/types';
 
-export async function saveEventData(data: {
+export interface SaveEventDataArgs {
   websiteId: string;
   eventId: string;
   sessionId?: string;
@@ -16,19 +15,16 @@ export async function saveEventData(data: {
   eventName?: string;
   eventData: DynamicData;
   createdAt?: Date;
-}) {
+}
+
+export async function saveEventData(data: SaveEventDataArgs) {
   return runQuery({
     [PRISMA]: () => relationalQuery(data),
     [CLICKHOUSE]: () => clickhouseQuery(data),
   });
 }
 
-async function relationalQuery(data: {
-  websiteId: string;
-  eventId: string;
-  eventData: DynamicData;
-  createdAt?: Date;
-}): Promise<Prisma.BatchPayload> {
+async function relationalQuery(data: SaveEventDataArgs) {
   const { websiteId, eventId, eventData, createdAt } = data;
 
   const jsonKeys = flattenJSON(eventData);
@@ -46,20 +42,12 @@ async function relationalQuery(data: {
     createdAt,
   }));
 
-  return prisma.client.eventData.createMany({
+  await prisma.client.eventData.createMany({
     data: flattenedData,
   });
 }
 
-async function clickhouseQuery(data: {
-  websiteId: string;
-  eventId: string;
-  sessionId?: string;
-  urlPath?: string;
-  eventName?: string;
-  eventData: DynamicData;
-  createdAt?: Date;
-}) {
+async function clickhouseQuery(data: SaveEventDataArgs) {
   const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;
 
   const { insert, getUTCString } = clickhouse;
@@ -88,6 +76,4 @@ async function clickhouseQuery(data: {
   } else {
     await insert('event_data', messages);
   }
-
-  return data;
 }

src/queries/sql/getChannelMetrics.ts

@@ -18,7 +18,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     `
     select
       referrer_domain as domain,
-      referrer_query as query,
+      url_query as query,
       count(distinct session_id) as visitors
     from website_event
     where website_id = {{websiteId::uuid}}
@@ -41,7 +41,7 @@ async function clickhouseQuery(
   const sql = `
     select
       referrer_domain as domain,
-      referrer_query as query,
+      url_query as query,
       uniq(session_id) as visitors
     from website_event
     where website_id = {websiteId:UUID}

src/queries/sql/getRealtimeActivity.ts

@@ -32,7 +32,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     where website_event.website_id = {{websiteId::uuid}}
     ${filterQuery}
     ${dateQuery}
-    order by website_event.created_at asc
+    order by website_event.created_at desc
     limit 100
     `,
     params,
@@ -59,7 +59,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters): Promis
         where website_id = {websiteId:UUID}
         ${filterQuery}
         ${dateQuery}
-        order by createdAt asc
+        order by createdAt desc
         limit 100
     `,
     params,

src/queries/sql/getRealtimeData.ts

@@ -24,7 +24,7 @@ export async function getRealtimeData(
 
   const uniques = new Set();
 
-  const { countries, urls, referrers, events } = activity.reduce(
+  const { countries, urls, referrers, events } = activity.reverse().reduce(
     (
       obj: { countries: any; urls: any; referrers: any; events: any },
       event: {

src/queries/sql/reports/getAttribution.ts

@@ -84,7 +84,7 @@ async function relationalQuery(
               sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) value
           from event_data ed
           join website_event we
-          on we.event_id = ed.website_event_Id
+          on we.event_id = ed.website_event_id
             and we.website_id = ed.website_id
           join (select website_event_id
                 from event_data
@@ -395,7 +395,7 @@ async function clickhouseQuery(
                    fbclid != '', 'Facebook / Meta', 
                    msclkid != '', 'Microsoft Ads', 
                    ttclid != '', 'TikTok Ads', 
-                   li_fat_id != '', '	LinkedIn Ads', 
+                   li_fat_id != '', 'LinkedIn Ads', 
                    twclid != '', 'Twitter Ads (X)','') name,
         ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value
     from model m

src/queries/sql/sessions/saveSessionData.ts

@@ -7,28 +7,29 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import kafka from '@/lib/kafka';
 import clickhouse from '@/lib/clickhouse';
 
-export async function saveSessionData(data: {
+export interface SaveSessionDataArgs {
   websiteId: string;
   sessionId: string;
   sessionData: DynamicData;
   distinctId?: string;
   createdAt?: Date;
-}) {
+}
+
+export async function saveSessionData(data: SaveSessionDataArgs) {
   return runQuery({
     [PRISMA]: () => relationalQuery(data),
     [CLICKHOUSE]: () => clickhouseQuery(data),
   });
 }
 
-export async function relationalQuery(data: {
-  websiteId: string;
-  sessionId: string;
-  sessionData: DynamicData;
-  distinctId?: string;
-  createdAt?: Date;
-}) {
+export async function relationalQuery({
+  websiteId,
+  sessionId,
+  sessionData,
+  distinctId,
+  createdAt,
+}: SaveSessionDataArgs) {
   const { client } = prisma;
-  const { websiteId, sessionId, sessionData, distinctId, createdAt } = data;
 
   const jsonKeys = flattenJSON(sessionData);
 
@@ -75,19 +76,15 @@ export async function relationalQuery(data: {
       });
     }
   }
-
-  return flattenedData;
 }
 
-async function clickhouseQuery(data: {
-  websiteId: string;
-  sessionId: string;
-  sessionData: DynamicData;
-  distinctId?: string;
-  createdAt?: Date;
-}) {
-  const { websiteId, sessionId, sessionData, distinctId, createdAt } = data;
-
+async function clickhouseQuery({
+  websiteId,
+  sessionId,
+  sessionData,
+  distinctId,
+  createdAt,
+}: SaveSessionDataArgs) {
   const { insert, getUTCString } = clickhouse;
   const { sendMessage } = kafka;
 
@@ -112,6 +109,4 @@ async function clickhouseQuery(data: {
   } else {
     await insert('session_data', messages);
   }
-
-  return data;
 }