From fe03e5f85ad60a072187cc4be74fd1134f21b2fe Mon Sep 17 00:00:00 2001 From: YiOrainy Date: Wed, 4 Jun 2025 15:49:12 +0300 Subject: [PATCH] R --- Dockerfile | 2 + docker-compose.yml | 7 ++- .../api/websites/[websiteId]/active/route.ts | 10 +++- .../api/websites/[websiteId]/metrics/route.ts | 4 +- .../[websiteId]/sessions/stats/route.ts | 3 + .../api/websites/[websiteId]/stats/route.ts | 5 +- src/lib/types.ts | 1 + src/queries/sql/getActiveVisitors.ts | 60 ++++++++++++++----- src/queries/sql/getWebsiteStats.ts | 54 ++++++++++++++--- .../sql/pageviews/getPageviewMetrics.ts | 31 ++++++++-- src/queries/sql/sessions/getSessionMetrics.ts | 25 ++++++++ .../sql/sessions/getWebsiteSessionStats.ts | 44 ++++++++++---- 12 files changed, 200 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b156643b..31bc450fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,11 @@ COPY . . ARG DATABASE_TYPE ARG BASE_PATH +ARG DATABASE_URL ENV DATABASE_TYPE=$DATABASE_TYPE ENV BASE_PATH=$BASE_PATH +ENV DATABASE_URL=$DATABASE_URL ENV NEXT_TELEMETRY_DISABLED=1 diff --git a/docker-compose.yml b/docker-compose.yml index 7b51db66c..236ad4174 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,12 @@ --- services: umami: - image: ghcr.io/umami-software/umami:postgresql-latest + build: + context: . + dockerfile: Dockerfile + args: + DATABASE_URL: postgresql://umami:umami@db:5432/umami + image: umami-local:dev ports: - "3000:3000" environment: diff --git a/src/app/api/websites/[websiteId]/active/route.ts b/src/app/api/websites/[websiteId]/active/route.ts index 88c0fd178..06cdc68e4 100644 --- a/src/app/api/websites/[websiteId]/active/route.ts +++ b/src/app/api/websites/[websiteId]/active/route.ts @@ -2,24 +2,30 @@ import { canViewWebsite } from '@/lib/auth'; import { json, unauthorized } from '@/lib/response'; import { getActiveVisitors } from '@/queries'; import { parseRequest } from '@/lib/request'; +import { z } from 'zod'; export async function GET( request: Request, { params }: { params: Promise<{ websiteId: string }> }, ) { - const { auth, error } = await parseRequest(request); + const schema = z.object({ + pathPrefix: z.string().optional(), + host: z.string().optional(), + }); + const { auth, query, error } = await parseRequest(request, schema); if (error) { return error(); } const { websiteId } = await params; + const { pathPrefix, host } = query; if (!(await canViewWebsite(auth, websiteId))) { return unauthorized(); } - const result = await getActiveVisitors(websiteId); + const result = await getActiveVisitors(websiteId, pathPrefix, host); return json(result); } diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 854339041..06c04e646 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -29,6 +29,7 @@ export async function GET( limit: z.coerce.number().optional(), offset: z.coerce.number().optional(), search: z.string().optional(), + pathPrefix: z.string().optional(), ...filterParams, }); @@ -39,7 +40,7 @@ export async function GET( } const { websiteId } = await params; - const { type, limit, offset, search } = query; + const { type, limit, offset, search, pathPrefix } = query; if (!(await canViewWebsite(auth, websiteId))) { return unauthorized(); @@ -51,6 +52,7 @@ export async function GET( ...getRequestFilters(query), startDate, endDate, + pathPrefix, }; if (search) { diff --git a/src/app/api/websites/[websiteId]/sessions/stats/route.ts b/src/app/api/websites/[websiteId]/sessions/stats/route.ts index e8e8e6c86..ae53acebd 100644 --- a/src/app/api/websites/[websiteId]/sessions/stats/route.ts +++ b/src/app/api/websites/[websiteId]/sessions/stats/route.ts @@ -12,6 +12,7 @@ export async function GET( const schema = z.object({ startAt: z.coerce.number().int(), endAt: z.coerce.number().int(), + pathPrefix: z.string().optional(), ...filterParams, }); @@ -22,6 +23,7 @@ export async function GET( } const { websiteId } = await params; + const { pathPrefix } = query; if (!(await canViewWebsite(auth, websiteId))) { return unauthorized(); @@ -35,6 +37,7 @@ export async function GET( ...filters, startDate, endDate, + pathPrefix, }); const data = Object.keys(metrics[0]).reduce((obj, key) => { diff --git a/src/app/api/websites/[websiteId]/stats/route.ts b/src/app/api/websites/[websiteId]/stats/route.ts index c146271f9..c55ff2ac6 100644 --- a/src/app/api/websites/[websiteId]/stats/route.ts +++ b/src/app/api/websites/[websiteId]/stats/route.ts @@ -14,6 +14,7 @@ export async function GET( startAt: z.coerce.number().int(), endAt: z.coerce.number().int(), compare: z.string().optional(), + pathPrefix: z.string().optional(), ...filterParams, }); @@ -24,7 +25,7 @@ export async function GET( } const { websiteId } = await params; - const { compare } = query; + const { compare, pathPrefix } = query; if (!(await canViewWebsite(auth, websiteId))) { return unauthorized(); @@ -43,12 +44,14 @@ export async function GET( ...filters, startDate, endDate, + pathPrefix, }); const prevPeriod = await getWebsiteStats(websiteId, { ...filters, startDate: compareStartDate, endDate: compareEndDate, + pathPrefix, }); const stats = Object.keys(metrics[0]).reduce((obj, key) => { diff --git a/src/lib/types.ts b/src/lib/types.ts index 00b6eec0c..6b903fe13 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -158,6 +158,7 @@ export interface QueryFilters { event?: string; search?: string; tag?: string; + pathPrefix?: string; } export interface QueryOptions { diff --git a/src/queries/sql/getActiveVisitors.ts b/src/queries/sql/getActiveVisitors.ts index e0225f3aa..29e78ead1 100644 --- a/src/queries/sql/getActiveVisitors.ts +++ b/src/queries/sql/getActiveVisitors.ts @@ -3,42 +3,70 @@ import prisma from '@/lib/prisma'; import clickhouse from '@/lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db'; -export async function getActiveVisitors(...args: [websiteId: string]) { +export async function getActiveVisitors(websiteId: string, pathPrefix?: string, host?: string) { return runQuery({ - [PRISMA]: () => relationalQuery(...args), - [CLICKHOUSE]: () => clickhouseQuery(...args), + [PRISMA]: () => relationalQuery(websiteId, pathPrefix, host), + [CLICKHOUSE]: () => clickhouseQuery(websiteId, pathPrefix, host), }); } -async function relationalQuery(websiteId: string) { +async function relationalQuery(websiteId: string, pathPrefix?: string, host?: string) { const { rawQuery } = prisma; - const result = await rawQuery( - ` + let sql = ` select count(distinct session_id) as visitors from website_event where website_id = {{websiteId::uuid}} and created_at >= {{startDate}} - `, - { websiteId, startDate: subMinutes(new Date(), 5) }, - ); + `; + const params: any = { websiteId, startDate: subMinutes(new Date(), 5) }; + if (pathPrefix) { + sql += ` + and ( + url_path LIKE {{pathPrefix}} + or url_path LIKE {{pathPrefixWithLang}} + )`; + params.pathPrefix = `${pathPrefix}%`; + params.pathPrefixWithLang = `%/en${pathPrefix}%`; + } + + if (host) { + sql += ` and hostname LIKE {{host}}`; + params.host = `%${host}%`; + } + + const result = await rawQuery(sql, params); return result[0] ?? null; } -async function clickhouseQuery(websiteId: string): Promise<{ x: number }> { +async function clickhouseQuery(websiteId: string, pathPrefix?: string, host?: string): Promise<{ x: number }> { const { rawQuery } = clickhouse; - const result = await rawQuery( - ` + let sql = ` select - count(distinct session_id) as "visitors" + count(distinct session_id) as \'visitors\' from website_event where website_id = {websiteId:UUID} and created_at >= {startDate:DateTime64} - `, - { websiteId, startDate: subMinutes(new Date(), 5) }, - ); + `; + const params: any = { websiteId, startDate: subMinutes(new Date(), 5) }; + if (pathPrefix) { + sql += ` + and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${pathPrefix}%`; + params.pathPrefixWithLang = `%/en${pathPrefix}%`; + } + + if (host) { + sql += ` and hostname LIKE {host:String}`; + params.host = `%${host}%`; + } + + const result = await rawQuery(sql, params); return result[0] ?? null; } diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts index 80f1d5789..7a0df0da3 100644 --- a/src/queries/sql/getWebsiteStats.ts +++ b/src/queries/sql/getWebsiteStats.ts @@ -6,7 +6,7 @@ import { QueryFilters } from '@/lib/types'; import { EVENT_COLUMNS } from '@/lib/constants'; export async function getWebsiteStats( - ...args: [websiteId: string, filters: QueryFilters] + ...args: [websiteId: string, filters: QueryFilters & { pathPrefix?: string }] ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { @@ -18,7 +18,7 @@ export async function getWebsiteStats( async function relationalQuery( websiteId: string, - filters: QueryFilters, + filters: QueryFilters & { pathPrefix?: string }, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { @@ -28,8 +28,7 @@ async function relationalQuery( eventType: EVENT_TYPE.pageView, }); - return rawQuery( - ` + let sql = ` select sum(t.c) as "pageviews", count(distinct t.session_id) as "visitors", @@ -49,16 +48,29 @@ async function relationalQuery( and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${filterQuery} + `; + + if (filters.pathPrefix) { + sql += ` + and ( + website_event.url_path LIKE {{pathPrefix}} + or website_event.url_path LIKE {{pathPrefixWithLang}} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + sql += ` group by 1, 2 ) as t - `, - params, - ); + `; + + return rawQuery(sql, params); } async function clickhouseQuery( websiteId: string, - filters: QueryFilters, + filters: QueryFilters & { pathPrefix?: string }, ): Promise< { pageviews: number; visitors: number; visits: number; bounces: number; totaltime: number }[] > { @@ -90,6 +102,19 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} + `; + + if (filters.pathPrefix) { + sql += ` + and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + sql += ` group by session_id, visit_id ) as t; `; @@ -112,6 +137,19 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${filterQuery} + `; + + if (filters.pathPrefix) { + sql += ` + and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + sql += ` group by session_id, visit_id ) as t; `; diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index 0053b4ad0..049f06d98 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -39,6 +39,16 @@ async function relationalQuery( let entryExitQuery = ''; let excludeDomain = ''; + let pathPrefixQuery = ''; + + if (filters.pathPrefix) { + pathPrefixQuery = `and ( + website_event.url_path LIKE {{pathPrefix}} + or website_event.url_path LIKE {{pathPrefixWithLang}} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } if (column === 'referrer_domain') { excludeDomain = `and website_event.referrer_domain != website_event.hostname @@ -74,6 +84,7 @@ async function relationalQuery( and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${excludeDomain} + ${pathPrefixQuery} ${filterQuery} group by 1 order by 2 desc @@ -100,14 +111,24 @@ async function clickhouseQuery( let sql = ''; let excludeDomain = ''; + let pathPrefixQuery = ''; + + if (filters.pathPrefix) { + pathPrefixQuery = `and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + } if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { let entryExitQuery = ''; - if (column === 'referrer_domain') { - excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; - } - if (type === 'entry' || type === 'exit') { const aggregrate = type === 'entry' ? 'min' : 'max'; @@ -132,6 +153,7 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${excludeDomain} + ${pathPrefixQuery} ${filterQuery} group by x order by y desc @@ -169,6 +191,7 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} ${excludeDomain} + ${pathPrefixQuery} ${filterQuery} ${groupByQuery}) as g group by x diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts index ef787920e..ac81cc24b 100644 --- a/src/queries/sql/sessions/getSessionMetrics.ts +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -40,6 +40,17 @@ async function relationalQuery( ); const includeCountry = column === 'city' || column === 'region'; + let pathPrefixQuery = ''; + + if (filters.pathPrefix) { + pathPrefixQuery = `and ( + website_event.url_path LIKE {{pathPrefix}} + or website_event.url_path LIKE {{pathPrefixWithLang}} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + return rawQuery( ` select @@ -51,6 +62,7 @@ async function relationalQuery( where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} and website_event.event_type = {{eventType}} + ${pathPrefixQuery} ${filterQuery} group by 1 ${includeCountry ? ', 3' : ''} @@ -77,6 +89,17 @@ async function clickhouseQuery( }); const includeCountry = column === 'city' || column === 'region'; + let pathPrefixQuery = ''; + + if (filters.pathPrefix) { + pathPrefixQuery = `and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + let sql = ''; if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { @@ -89,6 +112,7 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} + ${pathPrefixQuery} ${filterQuery} group by x ${includeCountry ? ', country' : ''} @@ -106,6 +130,7 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} + ${pathPrefixQuery} ${filterQuery} group by x ${includeCountry ? ', country' : ''} diff --git a/src/queries/sql/sessions/getWebsiteSessionStats.ts b/src/queries/sql/sessions/getWebsiteSessionStats.ts index 2463b7adf..174f437f3 100644 --- a/src/queries/sql/sessions/getWebsiteSessionStats.ts +++ b/src/queries/sql/sessions/getWebsiteSessionStats.ts @@ -4,7 +4,7 @@ import prisma from '@/lib/prisma'; import { QueryFilters } from '@/lib/types'; export async function getWebsiteSessionStats( - ...args: [websiteId: string, filters: QueryFilters] + ...args: [websiteId: string, filters: QueryFilters & { pathPrefix?: string }] ): Promise< { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] > { @@ -16,7 +16,7 @@ export async function getWebsiteSessionStats( async function relationalQuery( websiteId: string, - filters: QueryFilters, + filters: QueryFilters & { pathPrefix?: string }, ): Promise< { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] > { @@ -25,8 +25,7 @@ async function relationalQuery( ...filters, }); - return rawQuery( - ` + let sql = ` select count(*) as "pageviews", count(distinct website_event.session_id) as "visitors", @@ -38,14 +37,24 @@ async function relationalQuery( where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} - `, - params, - ); + `; + + if (filters.pathPrefix) { + sql += ` + and ( + website_event.url_path LIKE {{pathPrefix}} + or website_event.url_path LIKE {{pathPrefixWithLang}} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + return rawQuery(sql, params); } async function clickhouseQuery( websiteId: string, - filters: QueryFilters, + filters: QueryFilters & { pathPrefix?: string }, ): Promise< { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] > { @@ -54,8 +63,7 @@ async function clickhouseQuery( ...filters, }); - return rawQuery( - ` + let sql = ` select sum(views) as "pageviews", uniq(session_id) as "visitors", @@ -66,7 +74,17 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} ${filterQuery} - `, - params, - ); + `; + + if (filters.pathPrefix) { + sql += ` + and ( + url_path LIKE {pathPrefix:String} + or url_path LIKE {pathPrefixWithLang:String} + )`; + params.pathPrefix = `${filters.pathPrefix}%`; + params.pathPrefixWithLang = `%/en${filters.pathPrefix}%`; + } + + return rawQuery(sql, params); }