From cd532fd53df1c93afed9689207413ad9740ca469 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 8 May 2025 09:05:44 -0700 Subject: [PATCH 01/19] fix minute x-axis date labels --- src/lib/charts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/charts.ts b/src/lib/charts.ts index 957d4962d..3eab7562c 100644 --- a/src/lib/charts.ts +++ b/src/lib/charts.ts @@ -11,7 +11,7 @@ export function renderDateLabels(unit: string, locale: string) { switch (unit) { case 'minute': - return formatDate(d, 'p', locale).split(' ')[0]; + return formatDate(d, 'h:mm', locale).split(' ')[0]; case 'hour': return formatDate(d, 'p', locale); case 'day': From 3b24ea31751a1bc7b61baf91afd8e1ebe77d3015 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 8 May 2025 09:06:27 -0700 Subject: [PATCH 02/19] remove split code --- src/lib/charts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/charts.ts b/src/lib/charts.ts index 3eab7562c..7d4208e2a 100644 --- a/src/lib/charts.ts +++ b/src/lib/charts.ts @@ -11,7 +11,7 @@ export function renderDateLabels(unit: string, locale: string) { switch (unit) { case 'minute': - return formatDate(d, 'h:mm', locale).split(' ')[0]; + return formatDate(d, 'h:mm', locale); case 'hour': return formatDate(d, 'p', locale); case 'day': From 98eb26fd9ee1b30f0aa0493999c4d4f766dcffe5 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 8 May 2025 09:15:27 -0700 Subject: [PATCH 03/19] fixed getAttribution table alias typo --- pnpm-lock.yaml | 2 -- src/queries/sql/reports/getAttribution.ts | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20f2ff557..dfca7de5c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -346,8 +346,6 @@ importers: specifier: ^5.5.3 version: 5.8.3 - src/generated/prisma: {} - packages: '@ampproject/remapping@2.3.0': diff --git a/src/queries/sql/reports/getAttribution.ts b/src/queries/sql/reports/getAttribution.ts index e690d50f0..0c3d447a3 100644 --- a/src/queries/sql/reports/getAttribution.ts +++ b/src/queries/sql/reports/getAttribution.ts @@ -398,7 +398,7 @@ async function clickhouseQuery( li_fat_id != '', 'LinkedIn Ads', twclid != '', 'Twitter Ads (X)','') name, ${currency ? 'sum(e.value)' : 'uniqExact(we.session_id)'} value - from model fm + from model m join website_event we on we.created_at = m.created_at and we.session_id = m.session_id From d2a3ed732a07d586ceb559b358214200c0adb1c0 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 8 May 2025 10:04:21 -0700 Subject: [PATCH 04/19] fix realtime activity ordering --- src/queries/sql/getRealtimeActivity.ts | 4 ++-- src/queries/sql/getRealtimeData.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/queries/sql/getRealtimeActivity.ts b/src/queries/sql/getRealtimeActivity.ts index c2f58ad81..5f6646f60 100644 --- a/src/queries/sql/getRealtimeActivity.ts +++ b/src/queries/sql/getRealtimeActivity.ts @@ -32,7 +32,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { where website_event.website_id = {{websiteId::uuid}} ${filterQuery} ${dateQuery} - order by website_event.created_at asc + order by website_event.created_at desc limit 100 `, params, @@ -59,7 +59,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters): Promis where website_id = {websiteId:UUID} ${filterQuery} ${dateQuery} - order by createdAt asc + order by createdAt desc limit 100 `, params, diff --git a/src/queries/sql/getRealtimeData.ts b/src/queries/sql/getRealtimeData.ts index a9b06814c..1b52f0c54 100644 --- a/src/queries/sql/getRealtimeData.ts +++ b/src/queries/sql/getRealtimeData.ts @@ -64,7 +64,7 @@ export async function getRealtimeData( countries, urls, referrers, - events: events.reverse(), + events: events, series: { views: pageviews, visitors: sessions, From edbe52611e0fd56689c40758be6027b4cc6f9b52 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 8 May 2025 10:31:25 -0700 Subject: [PATCH 05/19] fix session activity order --- src/queries/sql/getRealtimeData.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queries/sql/getRealtimeData.ts b/src/queries/sql/getRealtimeData.ts index 1b52f0c54..c4229e6a8 100644 --- a/src/queries/sql/getRealtimeData.ts +++ b/src/queries/sql/getRealtimeData.ts @@ -24,7 +24,7 @@ export async function getRealtimeData( const uniques = new Set(); - const { countries, urls, referrers, events } = activity.reduce( + const { countries, urls, referrers, events } = activity.reverse().reduce( ( obj: { countries: any; urls: any; referrers: any; events: any }, event: { @@ -64,7 +64,7 @@ export async function getRealtimeData( countries, urls, referrers, - events: events, + events: events.reverse(), series: { views: pageviews, visitors: sessions, From c23f08fd2398092b2889a51d911e446e8100086d Mon Sep 17 00:00:00 2001 From: tsykin Date: Fri, 9 May 2025 14:37:27 +0200 Subject: [PATCH 06/19] changed sql query for channels metrics due to missing referrer_query data --- src/queries/sql/getChannelMetrics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index 9141290e4..433c5e81c 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -18,7 +18,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { ` select referrer_domain as domain, - referrer_query as query, + url_query as query, count(distinct session_id) as visitors from website_event where website_id = {{websiteId::uuid}} @@ -41,7 +41,7 @@ async function clickhouseQuery( const sql = ` select referrer_domain as domain, - referrer_query as query, + url_query as query, uniq(session_id) as visitors from website_event where website_id = {websiteId:UUID} From ed8cac4a3b5b4c10546a25ce64d931e6ec50620b Mon Sep 17 00:00:00 2001 From: Maxime-J Date: Fri, 9 May 2025 14:20:21 +0200 Subject: [PATCH 07/19] Consider base path in set-routes-manifest --- scripts/set-routes-manifest.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/set-routes-manifest.js b/scripts/set-routes-manifest.js index a3ac49562..2dfb71716 100644 --- a/scripts/set-routes-manifest.js +++ b/scripts/set-routes-manifest.js @@ -7,8 +7,10 @@ const routesManifestPath = path.resolve(__dirname, '../.next/routes-manifest.jso const originalPath = path.resolve(__dirname, '../.next/routes-manifest-orig.json'); const originalManifest = require(originalPath); -const API_PATH = '/api/:path*'; -const TRACKER_SCRIPT = '/script.js'; +const basePath = originalManifest.basePath; + +const API_PATH = basePath + '/api/:path*'; +const TRACKER_SCRIPT = basePath + '/script.js'; const collectApiEndpoint = process.env.COLLECT_API_ENDPOINT; const trackerScriptName = process.env.TRACKER_SCRIPT_NAME; @@ -20,14 +22,16 @@ if (collectApiEndpoint) { const apiRoute = originalManifest.headers.find(route => route.source === API_PATH); const routeRegex = new RegExp(apiRoute.regex); + const normalizedSource = basePath + collectApiEndpoint; + rewrites.push({ - source: collectApiEndpoint, - destination: '/api/send', + source: normalizedSource, + destination: basePath + '/api/send', }); - if (!routeRegex.test(collectApiEndpoint)) { + if (!routeRegex.test(normalizedSource)) { headers.push({ - source: collectApiEndpoint, + source: normalizedSource, headers: apiRoute.headers, }); } @@ -40,7 +44,7 @@ if (trackerScriptName) { if (names) { names.forEach(name => { - const normalizedSource = `/${name.replace(/^\/+/, '')}`; + const normalizedSource = `${basePath}/${name.replace(/^\/+/, '')}`; rewrites.push({ source: normalizedSource, From 3f1ecf4c1b9b334360baebe52a6aa7a59fe0a2fd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 9 May 2025 16:54:49 -0700 Subject: [PATCH 08/19] Fixed identify. Closes #3409 --- src/app/api/send/route.ts | 62 ++--- src/queries/sql/events/saveEvent.ts | 237 ++++++++------------ src/queries/sql/events/saveEventData.ts | 28 +-- src/queries/sql/sessions/saveSessionData.ts | 41 ++-- 4 files changed, 156 insertions(+), 212 deletions(-) diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index b35a67b7e..60d6f7af0 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -187,26 +187,19 @@ export async function POST(request: Request) { websiteId, sessionId, visitId, + createdAt, + + // Page + pageTitle: safeDecodeURIComponent(title), + hostname: hostname || urlDomain, urlPath: safeDecodeURI(urlPath), urlQuery, - utmSource, - utmMedium, - utmCampaign, - utmContent, - utmTerm, referrerPath: safeDecodeURI(referrerPath), referrerQuery, referrerDomain, - pageTitle: safeDecodeURIComponent(title), - gclid, - fbclid, - msclkid, - ttclid, - lifatid, - twclid, - eventName: name, - eventData: data, - hostname: hostname || urlDomain, + + // Session + distinctId: id, browser, os, device, @@ -215,24 +208,39 @@ export async function POST(request: Request) { country, region, city, + + // Events + eventName: name, + eventData: data, tag, - distinctId: id, - createdAt, + + // UTM + utmSource, + utmMedium, + utmCampaign, + utmContent, + utmTerm, + + // Click IDs + gclid, + fbclid, + msclkid, + ttclid, + lifatid, + twclid, }); } if (type === COLLECTION_TYPE.identify) { - if (!data) { - return badRequest('Data required.'); + if (data) { + await saveSessionData({ + websiteId, + sessionId, + sessionData: data, + distinctId: id, + createdAt, + }); } - - await saveSessionData({ - websiteId, - sessionId, - sessionData: data, - distinctId: id, - createdAt, - }); } const token = createToken({ websiteId, sessionId, visitId, iat }, secret()); diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index e9b736358..737b9b9f9 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -6,30 +6,23 @@ import prisma from '@/lib/prisma'; import { uuid } from '@/lib/crypto'; import { saveEventData } from './saveEventData'; -export async function saveEvent(args: { +export interface SaveEventArgs { websiteId: string; sessionId: string; visitId: string; + createdAt?: Date; + + // Page + pageTitle?: string; + hostname?: string; urlPath: string; urlQuery?: string; - utmSource?: string; - utmMedium?: string; - utmCampaign?: string; - utmContent?: string; - utmTerm?: string; referrerPath?: string; referrerQuery?: string; referrerDomain?: string; - pageTitle?: string; - gclid?: string; - fbclid?: string; - msclkid?: string; - ttclid?: string; - lifatid?: string; - twclid?: string; - eventName?: string; - eventData?: any; - hostname?: string; + + // Session + distinctId?: string; browser?: string; os?: string; device?: string; @@ -38,73 +31,65 @@ export async function saveEvent(args: { country?: string; region?: string; city?: string; - tag?: string; - distinctId?: string; - createdAt?: Date; -}) { - return runQuery({ - [PRISMA]: () => relationalQuery(args), - [CLICKHOUSE]: () => clickhouseQuery(args), - }); -} -async function relationalQuery(data: { - websiteId: string; - sessionId: string; - visitId: string; - urlPath: string; - urlQuery?: string; + // Events + eventName?: string; + eventData?: any; + tag?: string; + + // UTM utmSource?: string; utmMedium?: string; utmCampaign?: string; utmContent?: string; utmTerm?: string; - referrerPath?: string; - referrerQuery?: string; - referrerDomain?: string; + + // Click IDs gclid?: string; fbclid?: string; msclkid?: string; ttclid?: string; lifatid?: string; twclid?: string; - pageTitle?: string; - eventName?: string; - eventData?: any; - tag?: string; - hostname?: string; - createdAt?: Date; -}) { - const { - websiteId, - sessionId, - visitId, - urlPath, - urlQuery, - utmSource, - utmMedium, - utmCampaign, - utmContent, - utmTerm, - referrerPath, - referrerQuery, - referrerDomain, - eventName, - eventData, - pageTitle, - gclid, - fbclid, - msclkid, - ttclid, - lifatid, - twclid, - tag, - hostname, - createdAt, - } = data; +} + +export async function saveEvent(args: SaveEventArgs) { + return runQuery({ + [PRISMA]: () => relationalQuery(args), + [CLICKHOUSE]: () => clickhouseQuery(args), + }); +} + +async function relationalQuery({ + websiteId, + sessionId, + visitId, + createdAt, + pageTitle, + tag, + hostname, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + eventName, + eventData, + utmSource, + utmMedium, + utmCampaign, + utmContent, + utmTerm, + gclid, + fbclid, + msclkid, + ttclid, + lifatid, + twclid, +}: SaveEventArgs) { const websiteEventId = uuid(); - const websiteEvent = prisma.client.websiteEvent.create({ + await prisma.client.websiteEvent.create({ data: { id: websiteEventId, websiteId, @@ -146,83 +131,49 @@ async function relationalQuery(data: { createdAt, }); } - - return websiteEvent; } -async function clickhouseQuery(data: { - websiteId: string; - sessionId: string; - visitId: string; - urlPath: string; - urlQuery?: string; - utmSource?: string; - utmMedium?: string; - utmCampaign?: string; - utmContent?: string; - utmTerm?: string; - referrerPath?: string; - referrerQuery?: string; - referrerDomain?: string; - pageTitle?: string; - gclid?: string; - fbclid?: string; - msclkid?: string; - ttclid?: string; - lifatid?: string; - twclid?: string; - eventName?: string; - eventData?: any; - hostname?: string; - browser?: string; - os?: string; - device?: string; - screen?: string; - language?: string; - country?: string; - region?: string; - city?: string; - tag?: string; - distinctId?: string; - createdAt?: Date; -}) { - const { - websiteId, - sessionId, - visitId, - urlPath, - urlQuery, - utmSource, - utmMedium, - utmCampaign, - utmContent, - utmTerm, - referrerPath, - referrerQuery, - referrerDomain, - gclid, - fbclid, - msclkid, - ttclid, - lifatid, - twclid, - pageTitle, - eventName, - eventData, - country, - region, - city, - tag, - distinctId, - createdAt, - ...args - } = data; +async function clickhouseQuery({ + websiteId, + sessionId, + visitId, + distinctId, + createdAt, + pageTitle, + browser, + os, + device, + screen, + language, + country, + region, + city, + tag, + hostname, + urlPath, + urlQuery, + referrerPath, + referrerQuery, + referrerDomain, + eventName, + eventData, + utmSource, + utmMedium, + utmCampaign, + utmContent, + utmTerm, + gclid, + fbclid, + msclkid, + ttclid, + lifatid, + twclid, +}: SaveEventArgs) { const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; const eventId = uuid(); const message = { - ...args, website_id: websiteId, session_id: sessionId, visit_id: visitId, @@ -252,6 +203,12 @@ async function clickhouseQuery(data: { tag: tag, distinct_id: distinctId, created_at: getUTCString(createdAt), + browser, + os, + device, + screen, + language, + hostname, }; if (kafka.enabled) { @@ -271,6 +228,4 @@ async function clickhouseQuery(data: { createdAt, }); } - - return data; } diff --git a/src/queries/sql/events/saveEventData.ts b/src/queries/sql/events/saveEventData.ts index 16a5cab10..0fb869376 100644 --- a/src/queries/sql/events/saveEventData.ts +++ b/src/queries/sql/events/saveEventData.ts @@ -1,4 +1,3 @@ -import { Prisma } from '@prisma/client'; import { DATA_TYPE } from '@/lib/constants'; import { uuid } from '@/lib/crypto'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; @@ -8,7 +7,7 @@ import kafka from '@/lib/kafka'; import prisma from '@/lib/prisma'; import { DynamicData } from '@/lib/types'; -export async function saveEventData(data: { +export interface SaveEventDataArgs { websiteId: string; eventId: string; sessionId?: string; @@ -16,19 +15,16 @@ export async function saveEventData(data: { eventName?: string; eventData: DynamicData; createdAt?: Date; -}) { +} + +export async function saveEventData(data: SaveEventDataArgs) { return runQuery({ [PRISMA]: () => relationalQuery(data), [CLICKHOUSE]: () => clickhouseQuery(data), }); } -async function relationalQuery(data: { - websiteId: string; - eventId: string; - eventData: DynamicData; - createdAt?: Date; -}): Promise { +async function relationalQuery(data: SaveEventDataArgs) { const { websiteId, eventId, eventData, createdAt } = data; const jsonKeys = flattenJSON(eventData); @@ -46,20 +42,12 @@ async function relationalQuery(data: { createdAt, })); - return prisma.client.eventData.createMany({ + await prisma.client.eventData.createMany({ data: flattenedData, }); } -async function clickhouseQuery(data: { - websiteId: string; - eventId: string; - sessionId?: string; - urlPath?: string; - eventName?: string; - eventData: DynamicData; - createdAt?: Date; -}) { +async function clickhouseQuery(data: SaveEventDataArgs) { const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data; const { insert, getUTCString } = clickhouse; @@ -88,6 +76,4 @@ async function clickhouseQuery(data: { } else { await insert('event_data', messages); } - - return data; } diff --git a/src/queries/sql/sessions/saveSessionData.ts b/src/queries/sql/sessions/saveSessionData.ts index 46ad3cf6f..cffe8e6e5 100644 --- a/src/queries/sql/sessions/saveSessionData.ts +++ b/src/queries/sql/sessions/saveSessionData.ts @@ -7,28 +7,29 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import kafka from '@/lib/kafka'; import clickhouse from '@/lib/clickhouse'; -export async function saveSessionData(data: { +export interface SaveSessionDataArgs { websiteId: string; sessionId: string; sessionData: DynamicData; distinctId?: string; createdAt?: Date; -}) { +} + +export async function saveSessionData(data: SaveSessionDataArgs) { return runQuery({ [PRISMA]: () => relationalQuery(data), [CLICKHOUSE]: () => clickhouseQuery(data), }); } -export async function relationalQuery(data: { - websiteId: string; - sessionId: string; - sessionData: DynamicData; - distinctId?: string; - createdAt?: Date; -}) { +export async function relationalQuery({ + websiteId, + sessionId, + sessionData, + distinctId, + createdAt, +}: SaveSessionDataArgs) { const { client } = prisma; - const { websiteId, sessionId, sessionData, distinctId, createdAt } = data; const jsonKeys = flattenJSON(sessionData); @@ -75,19 +76,15 @@ export async function relationalQuery(data: { }); } } - - return flattenedData; } -async function clickhouseQuery(data: { - websiteId: string; - sessionId: string; - sessionData: DynamicData; - distinctId?: string; - createdAt?: Date; -}) { - const { websiteId, sessionId, sessionData, distinctId, createdAt } = data; - +async function clickhouseQuery({ + websiteId, + sessionId, + sessionData, + distinctId, + createdAt, +}: SaveSessionDataArgs) { const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; @@ -112,6 +109,4 @@ async function clickhouseQuery(data: { } else { await insert('session_data', messages); } - - return data; } From 97381f300a8cf86ca8f19a1d9949b250284eafcd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 9 May 2025 17:16:24 -0700 Subject: [PATCH 09/19] Fixed wrong Prisma version in Docker. --- Dockerfile | 2 +- package.json | 1 - pnpm-lock.yaml | 26 -------------------------- 3 files changed, 1 insertion(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index f17410859..4b156643b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN set -x \ && apk add --no-cache curl # Script dependencies -RUN pnpm add npm-run-all dotenv prisma@6.1.0 +RUN pnpm add npm-run-all dotenv prisma@6.7.0 # Permissions for prisma RUN chown -R nextjs:nodejs node_modules/.pnpm/ diff --git a/package.json b/package.json index 21c226bca..2d5d5485c 100644 --- a/package.json +++ b/package.json @@ -162,7 +162,6 @@ "postcss-flexbugs-fixes": "^5.0.2", "postcss-import": "^15.1.0", "postcss-preset-env": "7.8.3", - "postcss-rtlcss": "^4.0.1", "prettier": "^2.6.2", "prompts": "2.4.2", "rollup": "^3.28.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dfca7de5c..4496287c0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -291,9 +291,6 @@ importers: postcss-preset-env: specifier: 7.8.3 version: 7.8.3(postcss@8.5.3) - postcss-rtlcss: - specifier: ^4.0.1 - version: 4.0.9(postcss@8.5.3) prettier: specifier: ^2.6.2 version: 2.8.8 @@ -5511,12 +5508,6 @@ packages: postcss-resolve-nested-selector@0.1.6: resolution: {integrity: sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==} - postcss-rtlcss@4.0.9: - resolution: {integrity: sha512-dCNKEf+FgTv+EA3XI8ysg2RnpS5s3/iZmU+9qpCNFxHU/BhK+4hz7jyCsCAfo0CLnDrMPtaQENhwb+EGm1wh7Q==} - engines: {node: '>=18.0.0'} - peerDependencies: - postcss: ^8.4.21 - postcss-safe-parser@6.0.0: resolution: {integrity: sha512-FARHN8pwH+WiS2OPCxJI8FuRJpTVnn6ZNFiqAM2aeW2LwTHWWmWgIyKC6cUo0L8aeKiF/14MNvnpls6R2PBeMQ==} engines: {node: '>=12.0'} @@ -5896,11 +5887,6 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true - rtlcss@4.1.1: - resolution: {integrity: sha512-/oVHgBtnPNcggP2aVXQjSy6N1mMAfHg4GSag0QtZBlD5bdDgAHwr4pydqJGd+SUCu9260+Pjqbjwtvu7EMH1KQ==} - engines: {node: '>=12.0.0'} - hasBin: true - run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -12586,11 +12572,6 @@ snapshots: postcss-resolve-nested-selector@0.1.6: {} - postcss-rtlcss@4.0.9(postcss@8.5.3): - dependencies: - postcss: 8.5.3 - rtlcss: 4.1.1 - postcss-safe-parser@6.0.0(postcss@8.5.3): dependencies: postcss: 8.5.3 @@ -13006,13 +12987,6 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - rtlcss@4.1.1: - dependencies: - escalade: 3.2.0 - picocolors: 1.1.1 - postcss: 8.5.3 - strip-json-comments: 3.1.1 - run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 From 75d4614a9dd363463b62ee385fe56c0cd233dfdd Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 9 May 2025 20:11:06 -0700 Subject: [PATCH 10/19] Ignore generated folder used by Prisma. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b11f45093..70a1e1931 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ node_modules /.next/ /out/ /prisma/ +/src/generated/ # production /build From b93bcecd7b86b9e3ebe30f9af80fc4f1a5096b70 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 9 May 2025 21:47:44 -0700 Subject: [PATCH 11/19] Fixed data migration script. --- .../convert-utm-clid-columns.sql | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/data-migrations/convert-utm-clid-columns.sql b/scripts/data-migrations/convert-utm-clid-columns.sql index 5a4ac4ca1..cf85a156c 100644 --- a/scripts/data-migrations/convert-utm-clid-columns.sql +++ b/scripts/data-migrations/convert-utm-clid-columns.sql @@ -1,18 +1,18 @@ ----------------------------------------------------- --- postgreSQL +-- PostgreSQL ----------------------------------------------------- UPDATE "website_event" we -SET fbclid = url.fbclid, - gclid = url.gclid, - li_fat_id = url.li_fat_id, - msclkid = url.msclkid, - ttclid = url.ttclid, - twclid = url.twclid, - utm_campaign = url.utm_campaign, - utm_content = url.utm_content, - utm_medium = url.utm_medium, - utm_source = url.utm_source, - utm_term = url.utm_term +SET fbclid = LEFT(url.fbclid, 255), + gclid = LEFT(url.gclid, 255), + li_fat_id = LEFT(url.li_fat_id, 255), + msclkid = LEFT(url.msclkid, 255), + ttclid = LEFT(url.ttclid, 255), + twclid = LEFT(url.twclid, 255), + utm_campaign = LEFT(url.utm_campaign, 255), + utm_content = LEFT(url.utm_content, 255), + utm_medium = LEFT(url.utm_medium, 255), + utm_source = LEFT(url.utm_source, 255), + utm_term = LEFT(url.utm_term, 255) FROM (SELECT event_id, website_id, session_id, (regexp_matches(url_query, '(?:[&?]|^)fbclid=([^&]+)', 'i'))[1] AS fbclid, (regexp_matches(url_query, '(?:[&?]|^)gclid=([^&]+)', 'i'))[1] AS gclid, @@ -31,18 +31,18 @@ WHERE we.event_id = url.event_id and we.website_id = url.website_id; ----------------------------------------------------- --- mySQL +-- MySQL ----------------------------------------------------- UPDATE `website_event` -SET fbclid = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)fbclid=[^&]+'), '=', -1), '&', 1), - gclid = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)gclid=[^&]+'), '=', -1), '&', 1), - li_fat_id = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)li_fat_id=[^&]+'), '=', -1), '&', 1), - msclkid = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)msclkid=[^&]+'), '=', -1), '&', 1), - ttclid = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)ttclid=[^&]+'), '=', -1), '&', 1), - twclid = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)twclid=[^&]+'), '=', -1), '&', 1), - utm_campaign = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_campaign=[^&]+'), '=', -1), '&', 1), - utm_content = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_content=[^&]+'), '=', -1), '&', 1), - utm_medium = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_medium=[^&]+'), '=', -1), '&', 1), - utm_source = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_source=[^&]+'), '=', -1), '&', 1), - utm_term = SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_term=[^&]+'), '=', -1), '&', 1) -WHERE 1 = 1; \ No newline at end of file +SET fbclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)fbclid=[^&]+'), '=', -1), '&', 1), 255), + gclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)gclid=[^&]+'), '=', -1), '&', 1), 255), + li_fat_id = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)li_fat_id=[^&]+'), '=', -1), '&', 1), 255), + msclkid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)msclkid=[^&]+'), '=', -1), '&', 1), 255), + ttclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)ttclid=[^&]+'), '=', -1), '&', 1), 255), + twclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)twclid=[^&]+'), '=', -1), '&', 1), 255), + utm_campaign = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_campaign=[^&]+'), '=', -1), '&', 1), 255), + utm_content = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_content=[^&]+'), '=', -1), '&', 1), 255), + utm_medium = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_medium=[^&]+'), '=', -1), '&', 1), 255), + utm_source = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_source=[^&]+'), '=', -1), '&', 1), 255), + utm_term = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_term=[^&]+'), '=', -1), '&', 1), 255) +WHERE 1 = 1; From 42be91b736602e8d9a2d2a614cdfd6b0b2029b07 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Fri, 9 May 2025 23:03:16 -0700 Subject: [PATCH 12/19] Bump version v2.18.1. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2d5d5485c..b3def0f9f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "umami", - "version": "2.18.0", + "version": "2.18.1", "description": "A modern, privacy-focused alternative to Google Analytics.", "author": "Umami Software, Inc. ", "license": "MIT", From 7aa0c83e66733877d84408dcdd6f7c699736ce1d Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Sun, 11 May 2025 23:54:22 -0700 Subject: [PATCH 13/19] Updated migration script. --- scripts/data-migrations/convert-utm-clid-columns.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data-migrations/convert-utm-clid-columns.sql b/scripts/data-migrations/convert-utm-clid-columns.sql index cf85a156c..7211c2026 100644 --- a/scripts/data-migrations/convert-utm-clid-columns.sql +++ b/scripts/data-migrations/convert-utm-clid-columns.sql @@ -45,4 +45,4 @@ SET fbclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[ utm_medium = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_medium=[^&]+'), '=', -1), '&', 1), 255), utm_source = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_source=[^&]+'), '=', -1), '&', 1), 255), utm_term = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_term=[^&]+'), '=', -1), '&', 1), 255) -WHERE 1 = 1; +WHERE url_query IS NOT NULL; From 907a6850dbac98dbe7f9996301421fa8e4d57c7a Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Mon, 12 May 2025 00:01:24 -0700 Subject: [PATCH 14/19] Update migration script. --- scripts/data-migrations/convert-utm-clid-columns.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/data-migrations/convert-utm-clid-columns.sql b/scripts/data-migrations/convert-utm-clid-columns.sql index cf85a156c..fffe1ddb0 100644 --- a/scripts/data-migrations/convert-utm-clid-columns.sql +++ b/scripts/data-migrations/convert-utm-clid-columns.sql @@ -25,7 +25,8 @@ FROM (SELECT event_id, website_id, session_id, (regexp_matches(url_query, '(?:[&?]|^)utm_medium=([^&]+)', 'i'))[1] AS utm_medium, (regexp_matches(url_query, '(?:[&?]|^)utm_source=([^&]+)', 'i'))[1] AS utm_source, (regexp_matches(url_query, '(?:[&?]|^)utm_term=([^&]+)', 'i'))[1] AS utm_term - FROM "website_event") url + FROM "website_event" + WHERE url_query IS NOT NULL) url WHERE we.event_id = url.event_id and we.session_id = url.session_id and we.website_id = url.website_id; @@ -45,4 +46,4 @@ SET fbclid = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[ utm_medium = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_medium=[^&]+'), '=', -1), '&', 1), 255), utm_source = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_source=[^&]+'), '=', -1), '&', 1), 255), utm_term = LEFT(SUBSTRING_INDEX(SUBSTRING_INDEX(REGEXP_SUBSTR(url_query, '(?:[&?]|^)utm_term=[^&]+'), '=', -1), '&', 1), 255) -WHERE 1 = 1; +WHERE url_query IS NOT NULL; From 68fab48ab729e7d6ac4d925ab8c35dbedd9885ab Mon Sep 17 00:00:00 2001 From: 360 Date: Thu, 15 May 2025 10:13:07 +0100 Subject: [PATCH 15/19] fix: correct example description for Umami version in bug report template --- .github/ISSUE_TEMPLATE/1.bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/1.bug_report.yml b/.github/ISSUE_TEMPLATE/1.bug_report.yml index 711468f25..2404918b8 100644 --- a/.github/ISSUE_TEMPLATE/1.bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1.bug_report.yml @@ -25,7 +25,7 @@ body: - type: input attributes: label: Which Umami version are you using? (if relevant) - description: 'For example: Chrome, Edge, Firefox, etc' + description: 'For example: 2.18.0, 2.15.1, 1.39.0, etc' - type: input attributes: label: Which browser are you using? (if relevant) From 8d483d92830cda0efa15d5c54d42a26168004077 Mon Sep 17 00:00:00 2001 From: Eritque arcus Date: Thu, 22 May 2025 00:19:50 -0500 Subject: [PATCH 16/19] fix: hash is not included in record --- src/app/api/send/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 60d6f7af0..9d41a68f6 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -145,7 +145,7 @@ export async function POST(request: Request) { const base = hostname ? `https://${hostname}` : 'https://localhost'; const currentUrl = new URL(url, base); - let urlPath = currentUrl.pathname === '/undefined' ? '' : currentUrl.pathname; + let urlPath = currentUrl.pathname === '/undefined' ? '' : currentUrl.pathname + currentUrl.hash; const urlQuery = currentUrl.search.substring(1); const urlDomain = currentUrl.hostname.replace(/^www./, ''); From 33110a44ec1c04896e4b6f06ba7773156901b6d6 Mon Sep 17 00:00:00 2001 From: Eritque arcus Date: Thu, 22 May 2025 00:24:48 -0500 Subject: [PATCH 17/19] fix: fix remove trailing slash regex --- src/app/api/send/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/api/send/route.ts b/src/app/api/send/route.ts index 9d41a68f6..65c88a285 100644 --- a/src/app/api/send/route.ts +++ b/src/app/api/send/route.ts @@ -169,7 +169,7 @@ export async function POST(request: Request) { const twclid = currentUrl.searchParams.get('twclid'); if (process.env.REMOVE_TRAILING_SLASH) { - urlPath = urlPath.replace(/(.+)\/$/, '$1'); + urlPath = urlPath.replace(/\/(?=(#.*)?$)/, ''); } if (referrer) { From e8f166cc690c595f21a1ebf6143bbaf61b343848 Mon Sep 17 00:00:00 2001 From: Ru Chern Chong <10343662+ruchernchong@users.noreply.github.com> Date: Thu, 22 May 2025 15:19:34 +0800 Subject: [PATCH 18/19] Update ci.yml Only run the CI if it belongs to the original repository --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 314c6944b..835407b41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ env: jobs: build: + # Only run the CI if it belongs to the original repository + if: github.repository == 'umami-software/umami' runs-on: ubuntu-latest strategy: From 19ccfa0745410c4e501a6ce7a577ab51893fbb44 Mon Sep 17 00:00:00 2001 From: Matt Harrington Date: Fri, 13 Jun 2025 12:17:18 -0700 Subject: [PATCH 19/19] fixing the clickhouse schema file --- db/clickhouse/schema.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index fef600e05..1080b4d11 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -122,7 +122,7 @@ CREATE TABLE umami.website_event_stats_hourly min_time SimpleAggregateFunction(min, DateTime('UTC')), max_time SimpleAggregateFunction(max, DateTime('UTC')), tag SimpleAggregateFunction(groupArrayArray, Array(String)), - distinct_id, + distinct_id String, created_at Datetime('UTC') ) ENGINE = AggregatingMergeTree @@ -213,7 +213,7 @@ FROM (SELECT min(created_at) min_time, max(created_at) max_time, arrayFilter(x -> x != '', groupArray(tag)) tag, - distinct_id String, + distinct_id, toStartOfHour(created_at) timestamp FROM umami.website_event GROUP BY website_id,