diff --git a/.eslintrc.json b/.eslintrc.json index 82f6a122..324e291c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -33,6 +33,7 @@ "react/prop-types": "off", "import/no-anonymous-default-export": "off", "import/no-named-as-default": "off", + "css-modules/no-unused-class": "off", "@next/next/no-img-element": "off", "@typescript-eslint/no-empty-function": "off", "@typescript-eslint/no-explicit-any": "off", diff --git a/next.config.js b/next.config.js index c767a784..fefe8bd4 100644 --- a/next.config.js +++ b/next.config.js @@ -59,15 +59,29 @@ const trackerHeaders = [ }, ]; +const apiHeaders = [ + { + key: 'Access-Control-Allow-Origin', + value: '*' + }, + { + key: 'Access-Control-Allow-Headers', + value: '*' + }, + { + key: 'Access-Control-Allow-Methods', + value: 'GET, DELETE, POST, PUT' + }, + { + key: 'Access-Control-Max-Age', + value: corsMaxAge || '86400' + }, +]; + const headers = [ { source: '/api/:path*', - headers: [ - { key: 'Access-Control-Allow-Origin', value: '*' }, - { key: 'Access-Control-Allow-Headers', value: '*' }, - { key: 'Access-Control-Allow-Methods', value: 'GET, DELETE, POST, PUT' }, - { key: 'Access-Control-Max-Age', value: corsMaxAge || '86400' }, - ], + headers: apiHeaders }, { source: '/:path*', @@ -89,6 +103,11 @@ if (trackerScriptURL) { } if (collectApiEndpoint) { + headers.push({ + source: collectApiEndpoint, + headers: apiHeaders, + }); + rewrites.push({ source: collectApiEndpoint, destination: '/api/send', diff --git a/scripts/check-db.js b/scripts/check-db.js index cdfeafa3..ca0fca31 100644 --- a/scripts/check-db.js +++ b/scripts/check-db.js @@ -82,9 +82,11 @@ async function checkV1Tables() { } async function applyMigration() { - console.log(execSync('prisma migrate deploy').toString()); + if (!process.env.SKIP_DB_MIGRATION) { + console.log(execSync('prisma migrate deploy').toString()); - success('Database is up to date.'); + success('Database is up to date.'); + } } (async () => { diff --git a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx index e42f86fd..05cfc001 100644 --- a/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx +++ b/src/app/(main)/websites/[websiteId]/realtime/RealtimeLog.tsx @@ -71,9 +71,10 @@ export function RealtimeLog({ data }: { data: RealtimeData }) { if (__type === TYPE_EVENT) { return formatMessage(messages.eventLog, { - event: {eventName || formatMessage(labels.unknown)}, + event: {eventName || formatMessage(labels.unknown)}, url: ( {countryNames[country] || formatMessage(labels.unknown)}, - browser: {BROWSERS[browser]}, - os: {OS_NAMES[os] || os}, - device: {formatMessage(labels[device] || labels.unknown)}, + country: {countryNames[country] || formatMessage(labels.unknown)}, + browser: {BROWSERS[browser]}, + os: {OS_NAMES[os] || os}, + device: {formatMessage(labels[device] || labels.unknown)}, }); } }; diff --git a/src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx b/src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx index 872e8b20..58e1c1a0 100644 --- a/src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx +++ b/src/app/(main)/websites/[websiteId]/sessions/SessionsWeekly.tsx @@ -62,10 +62,10 @@ export function SessionsWeekly({ websiteId }: { websiteId: string }) {
{format(getDayOfWeekAsDate(index), 'EEE', { locale: dateLocale })}
- {day?.map((hour: number) => { + {day?.map((hour: number, j) => { const pct = hour / max; return ( -
+
{hour > 0 && ( 1800) { - visitId = uuid(sessionId, visitSalt()); + if (!timestamp && now - iat > 1800) { + visitId = uuid(sessionId, visitSalt); iat = now; } @@ -179,6 +187,7 @@ export async function POST(request: Request) { subdivision2, city, tag, + createdAt, }); } @@ -191,12 +200,13 @@ export async function POST(request: Request) { websiteId, sessionId, sessionData: data, + createdAt, }); } const token = createToken({ websiteId, sessionId, visitId, iat }, secret()); - return json({ cache: token }); + return json({ cache: token, sessionId, visitId }); } catch (e) { return serverError(e); } diff --git a/src/app/api/users/route.ts b/src/app/api/users/route.ts index f6b32fe7..c5896f89 100644 --- a/src/app/api/users/route.ts +++ b/src/app/api/users/route.ts @@ -8,6 +8,7 @@ import { createUser, getUserByUsername } from '@/queries'; export async function POST(request: Request) { const schema = z.object({ + id: z.string().uuid().optional(), username: z.string().max(255), password: z.string(), role: z.string().regex(/admin|user|view-only/i), @@ -23,7 +24,7 @@ export async function POST(request: Request) { return unauthorized(); } - const { username, password, role } = body; + const { id, username, password, role } = body; const existingUser = await getUserByUsername(username, { showDeleted: true }); @@ -32,7 +33,7 @@ export async function POST(request: Request) { } const user = await createUser({ - id: uuid(), + id: id || uuid(), username, password: hashPassword(password), role: role ?? ROLES.user, diff --git a/src/lib/crypto.ts b/src/lib/crypto.ts index a4ff3a52..d22bad09 100644 --- a/src/lib/crypto.ts +++ b/src/lib/crypto.ts @@ -1,5 +1,4 @@ import crypto from 'crypto'; -import { startOfHour, startOfMonth } from 'date-fns'; import prand from 'pure-rand'; import { v4, v5 } from 'uuid'; @@ -77,20 +76,8 @@ export function secret() { return hash(process.env.APP_SECRET || process.env.DATABASE_URL); } -export function salt() { - const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString()); - - return hash(secret(), ROTATING_SALT); -} - -export function visitSalt() { - const ROTATING_SALT = hash(startOfHour(new Date()).toUTCString()); - - return hash(secret(), ROTATING_SALT); -} - export function uuid(...args: any) { if (!args.length) return v4(); - return v5(hash(...args, salt()), v5.DNS); + return v5(hash(...args, secret()), v5.DNS); } diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts index e2f50a6c..c8286082 100644 --- a/src/lib/prisma.ts +++ b/src/lib/prisma.ts @@ -192,7 +192,9 @@ async function parseFilters( options: QueryOptions = {}, ) { const website = await fetchWebsite(websiteId); - const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key)); + const joinSession = Object.keys(filters).find(key => + ['referrer', ...SESSION_COLUMNS].includes(key), + ); return { joinSession: diff --git a/src/lib/request.ts b/src/lib/request.ts index 9d32f89b..0c71537a 100644 --- a/src/lib/request.ts +++ b/src/lib/request.ts @@ -1,4 +1,4 @@ -import { ZodObject } from 'zod'; +import { ZodSchema } from 'zod'; import { FILTER_COLUMNS } from '@/lib/constants'; import { badRequest, unauthorized } from '@/lib/response'; import { getAllowedUnits, getMinimumUnit } from '@/lib/date'; @@ -15,7 +15,7 @@ export async function getJsonBody(request: Request) { export async function parseRequest( request: Request, - schema?: ZodObject, + schema?: ZodSchema, options?: { skipAuth: boolean }, ): Promise { const url = new URL(request.url); diff --git a/src/lib/schema.ts b/src/lib/schema.ts index 8df7be9f..4e2b3e4a 100644 --- a/src/lib/schema.ts +++ b/src/lib/schema.ts @@ -36,6 +36,8 @@ export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value), export const roleParam = z.enum(['team-member', 'team-view-only', 'team-manager']); +export const anyObjectParam = z.object({}).passthrough(); + export const urlOrPathParam = z.string().refine( value => { try { diff --git a/src/queries/sql/events/saveEvent.ts b/src/queries/sql/events/saveEvent.ts index 65ee1175..148b03f3 100644 --- a/src/queries/sql/events/saveEvent.ts +++ b/src/queries/sql/events/saveEvent.ts @@ -29,6 +29,7 @@ export async function saveEvent(args: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(args), @@ -49,6 +50,7 @@ async function relationalQuery(data: { eventName?: string; eventData?: any; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -63,6 +65,7 @@ async function relationalQuery(data: { eventData, pageTitle, tag, + createdAt, } = data; const websiteEventId = uuid(); @@ -81,6 +84,7 @@ async function relationalQuery(data: { eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, tag, + createdAt, }, }); @@ -92,6 +96,7 @@ async function relationalQuery(data: { urlPath: urlPath?.substring(0, URL_LENGTH), eventName: eventName?.substring(0, EVENT_NAME_LENGTH), eventData, + createdAt, }); } @@ -121,6 +126,7 @@ async function clickhouseQuery(data: { subdivision2?: string; city?: string; tag?: string; + createdAt?: Date; }) { const { websiteId, @@ -139,12 +145,12 @@ async function clickhouseQuery(data: { subdivision2, city, tag, + createdAt, ...args } = data; const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; const eventId = uuid(); - const createdAt = getUTCString(); const message = { ...args, @@ -170,7 +176,7 @@ async function clickhouseQuery(data: { event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null, tag: tag, - created_at: createdAt, + created_at: getUTCString(createdAt), }; if (kafka.enabled) { diff --git a/src/queries/sql/events/saveEventData.ts b/src/queries/sql/events/saveEventData.ts index 7c158da4..16a5cab1 100644 --- a/src/queries/sql/events/saveEventData.ts +++ b/src/queries/sql/events/saveEventData.ts @@ -15,7 +15,7 @@ export async function saveEventData(data: { urlPath?: string; eventName?: string; eventData: DynamicData; - createdAt?: string; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(data), @@ -27,8 +27,9 @@ async function relationalQuery(data: { websiteId: string; eventId: string; eventData: DynamicData; + createdAt?: Date; }): Promise { - const { websiteId, eventId, eventData } = data; + const { websiteId, eventId, eventData, createdAt } = data; const jsonKeys = flattenJSON(eventData); @@ -42,6 +43,7 @@ async function relationalQuery(data: { numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, + createdAt, })); return prisma.client.eventData.createMany({ @@ -56,7 +58,7 @@ async function clickhouseQuery(data: { urlPath?: string; eventName?: string; eventData: DynamicData; - createdAt?: string; + createdAt?: Date; }) { const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data; @@ -77,7 +79,7 @@ async function clickhouseQuery(data: { string_value: getStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, - created_at: createdAt, + created_at: getUTCString(createdAt), }; }); diff --git a/src/queries/sql/sessions/saveSessionData.ts b/src/queries/sql/sessions/saveSessionData.ts index 35f0c712..a060e9a8 100644 --- a/src/queries/sql/sessions/saveSessionData.ts +++ b/src/queries/sql/sessions/saveSessionData.ts @@ -11,6 +11,7 @@ export async function saveSessionData(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { return runQuery({ [PRISMA]: () => relationalQuery(data), @@ -22,9 +23,10 @@ export async function relationalQuery(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { const { client } = prisma; - const { websiteId, sessionId, sessionData } = data; + const { websiteId, sessionId, sessionData, createdAt } = data; const jsonKeys = flattenJSON(sessionData); @@ -37,6 +39,7 @@ export async function relationalQuery(data: { numberValue: a.dataType === DATA_TYPE.number ? a.value : null, dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dataType, + createdAt, })); const existing = await client.sessionData.findMany({ @@ -77,12 +80,12 @@ async function clickhouseQuery(data: { websiteId: string; sessionId: string; sessionData: DynamicData; + createdAt?: Date; }) { - const { websiteId, sessionId, sessionData } = data; + const { websiteId, sessionId, sessionData, createdAt } = data; const { insert, getUTCString } = clickhouse; const { sendMessage } = kafka; - const createdAt = getUTCString(); const jsonKeys = flattenJSON(sessionData); @@ -95,7 +98,7 @@ async function clickhouseQuery(data: { string_value: getStringValue(value, dataType), number_value: dataType === DATA_TYPE.number ? value : null, date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, - created_at: createdAt, + created_at: getUTCString(createdAt), }; }); diff --git a/src/tracker/index.js b/src/tracker/index.js index dbd47b7c..c423a66b 100644 --- a/src/tracker/index.js +++ b/src/tracker/index.js @@ -1,11 +1,12 @@ (window => { const { screen: { width, height }, - navigator: { language }, + navigator: { language, doNotTrack: ndnt, msDoNotTrack: msdnt }, location, document, history, top, + doNotTrack, } = window; const { hostname, href, origin } = location; const { currentScript, referrer } = document; @@ -21,6 +22,7 @@ const hostUrl = attr(_data + 'host-url'); const tag = attr(_data + 'tag'); const autoTrack = attr(_data + 'auto-track') !== _false; + const dnt = attr(_data + 'do-not-track') === _true; const excludeSearch = attr(_data + 'exclude-search') === _true; const excludeHash = attr(_data + 'exclude-hash') === _true; const domain = attr(_data + 'domains') || ''; @@ -46,6 +48,11 @@ tag: tag ? tag : undefined, }); + const hasDoNotTrack = () => { + const dnt = doNotTrack || ndnt || msdnt; + return dnt === 1 || dnt === '1' || dnt === 'yes'; + }; + /* Event handlers */ const handlePush = (state, title, url) => { @@ -182,7 +189,8 @@ disabled || !website || (localStorage && localStorage.getItem('umami.disabled')) || - (domain && !domains.includes(hostname)); + (domain && !domains.includes(hostname)) || + (dnt && hasDoNotTrack()); const send = async (payload, type = 'event') => { if (trackingDisabled()) return;