Merge remote-tracking branch 'origin/dev' into dev

This commit is contained in:
Mike Cao 2025-04-25 03:20:00 -07:00
commit ffa8d8dd88
19 changed files with 11252 additions and 89 deletions

View file

@ -0,0 +1,122 @@
-- drop projections
ALTER TABLE umami.website_event DROP PROJECTION website_event_url_path_projection;
ALTER TABLE umami.website_event DROP PROJECTION website_event_referrer_domain_projection;
--drop view
DROP TABLE umami.website_event_stats_hourly_mv;
-- rename columns
ALTER TABLE umami.website_event RENAME COLUMN "subdivision1" TO "region";
ALTER TABLE umami.website_event_stats_hourly RENAME COLUMN "subdivision1" TO "region";
-- drop columns
ALTER TABLE umami.website_event DROP COLUMN "subdivision2";
-- recreate projections
ALTER TABLE umami.website_event
ADD PROJECTION website_event_url_path_projection (
SELECT * ORDER BY toStartOfDay(created_at), website_id, url_path, created_at
);
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_url_path_projection;
ALTER TABLE umami.website_event
ADD PROJECTION website_event_referrer_domain_projection (
SELECT * ORDER BY toStartOfDay(created_at), website_id, referrer_domain, created_at
);
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_referrer_domain_projection;
-- recreate view
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
timestamp);

View file

@ -13,8 +13,7 @@ CREATE TABLE umami.website_event
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
subdivision2 LowCardinality(String),
region LowCardinality(String),
city String,
--pageviews
url_path String,
@ -96,7 +95,7 @@ CREATE TABLE umami.website_event_stats_hourly
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
region LowCardinality(String),
city String,
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
@ -148,7 +147,7 @@ SELECT
screen,
language,
country,
subdivision1,
region,
city,
entry_url,
exit_url,
@ -185,7 +184,7 @@ FROM (SELECT
screen,
language,
country,
subdivision1,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
@ -222,7 +221,7 @@ GROUP BY website_id,
screen,
language,
country,
subdivision1,
region,
city,
event_type,
timestamp);

View file

@ -0,0 +1,22 @@
-- AlterTable
ALTER TABLE `website_event` ADD COLUMN `hostname` VARCHAR(100) NULL;
-- DataMigration
UPDATE `website_event` w
JOIN `session` s
ON s.website_id = w.website_id
and s.session_id = w.session_id
SET w.hostname = s.hostname;
-- DropIndex
DROP INDEX `session_website_id_created_at_hostname_idx` ON `session`;
DROP INDEX `session_website_id_created_at_subdivision1_idx` ON `session`;
-- AlterTable
ALTER TABLE `session` RENAME COLUMN `subdivision1` TO `region`;
ALTER TABLE `session` DROP COLUMN `subdivision2`;
ALTER TABLE `session` DROP COLUMN `hostname`;
-- CreateIndex
CREATE INDEX `website_event_website_id_created_at_hostname_idx` ON `website_event`(`website_id`, `created_at`, `hostname`);
CREATE INDEX `session_website_id_created_at_region_idx` ON `session`(`website_id`, `created_at`, `region`);

View file

@ -31,15 +31,13 @@ model User {
model Session {
id String @id @unique @map("session_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
hostname String? @db.VarChar(100)
browser String? @db.VarChar(20)
os String? @db.VarChar(20)
device String? @db.VarChar(20)
screen String? @db.VarChar(11)
language String? @db.VarChar(35)
country String? @db.Char(2)
subdivision1 String? @db.Char(20)
subdivision2 String? @db.VarChar(50)
region String? @db.Char(20)
city String? @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
@ -49,14 +47,13 @@ model Session {
@@index([createdAt])
@@index([websiteId])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, hostname])
@@index([websiteId, createdAt, browser])
@@index([websiteId, createdAt, os])
@@index([websiteId, createdAt, device])
@@index([websiteId, createdAt, screen])
@@index([websiteId, createdAt, language])
@@index([websiteId, createdAt, country])
@@index([websiteId, createdAt, subdivision1])
@@index([websiteId, createdAt, region])
@@index([websiteId, createdAt, city])
@@map("session")
}
@ -115,6 +112,7 @@ model WebsiteEvent {
eventType Int @default(1) @map("event_type") @db.UnsignedInt
eventName String? @map("event_name") @db.VarChar(50)
tag String? @db.VarChar(50)
hostname String? @db.VarChar(100)
eventData EventData[]
session Session @relation(fields: [sessionId], references: [id])
@ -132,6 +130,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, tag])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@index([websiteId, createdAt, hostname])
@@map("website_event")
}

View file

@ -0,0 +1,25 @@
-- AlterTable
ALTER TABLE "website_event" ADD COLUMN "hostname" VARCHAR(100);
-- DataMigration
UPDATE "website_event" w
SET hostname = s.hostname
FROM "session" s
WHERE s.website_id = w.website_id
and s.session_id = w.session_id;
-- DropIndex
DROP INDEX IF EXISTS "session_website_id_created_at_hostname_idx";
DROP INDEX IF EXISTS "session_website_id_created_at_subdivision1_idx";
-- AlterTable
ALTER TABLE "session" RENAME COLUMN "subdivision1" TO "region";
ALTER TABLE "session" DROP COLUMN "subdivision2";
ALTER TABLE "session" DROP COLUMN "hostname";
-- CreateIndex
CREATE INDEX "website_event_website_id_created_at_hostname_idx" ON "website_event"("website_id", "created_at", "hostname");
CREATE INDEX "session_website_id_created_at_region_idx" ON "session"("website_id", "created_at", "region");

View file

@ -31,15 +31,13 @@ model User {
model Session {
id String @id @unique @map("session_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
hostname String? @db.VarChar(100)
browser String? @db.VarChar(20)
os String? @db.VarChar(20)
device String? @db.VarChar(20)
screen String? @db.VarChar(11)
language String? @db.VarChar(35)
country String? @db.Char(2)
subdivision1 String? @db.VarChar(20)
subdivision2 String? @db.VarChar(50)
region String? @db.VarChar(20)
city String? @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
@ -49,14 +47,13 @@ model Session {
@@index([createdAt])
@@index([websiteId])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, hostname])
@@index([websiteId, createdAt, browser])
@@index([websiteId, createdAt, os])
@@index([websiteId, createdAt, device])
@@index([websiteId, createdAt, screen])
@@index([websiteId, createdAt, language])
@@index([websiteId, createdAt, country])
@@index([websiteId, createdAt, subdivision1])
@@index([websiteId, createdAt, region])
@@index([websiteId, createdAt, city])
@@map("session")
}
@ -115,6 +112,7 @@ model WebsiteEvent {
eventType Int @default(1) @map("event_type") @db.Integer
eventName String? @map("event_name") @db.VarChar(50)
tag String? @db.VarChar(50)
hostname String? @db.VarChar(100)
eventData EventData[]
session Session @relation(fields: [sessionId], references: [id])
@ -132,6 +130,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, tag])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@index([websiteId, createdAt, hostname])
@@map("website_event")
}

View file

@ -36,7 +36,7 @@ export default function SessionInfo({ data }) {
<Icon>
<Icons.Location />
</Icon>
{getRegionName(data?.subdivision1)}
{getRegionName(data?.region)}
</dd>
<dt>{formatMessage(labels.city)}</dt>

View file

@ -80,8 +80,10 @@ export async function POST(request: Request) {
}
// Client info
const { ip, userAgent, device, browser, os, country, subdivision1, subdivision2, city } =
await getClientInfo(request, payload);
const { ip, userAgent, device, browser, os, country, region, city } = await getClientInfo(
request,
payload,
);
// Bot check
if (!process.env.DISABLE_BOT_CHECK && isbot(userAgent)) {
@ -111,15 +113,13 @@ export async function POST(request: Request) {
await createSession({
id: sessionId,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
region,
city,
});
} catch (e: any) {
@ -212,8 +212,7 @@ export async function POST(request: Request) {
screen,
language,
country,
subdivision1,
subdivision2,
region,
city,
tag,
createdAt,

View file

@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];
export const EVENT_COLUMNS = [
'url',
'entry',
'exit',
'referrer',
'title',
'query',
'event',
'tag',
'region',
];
export const SESSION_COLUMNS = [
'browser',
@ -42,7 +52,6 @@ export const SESSION_COLUMNS = [
'screen',
'language',
'country',
'region',
'city',
'host',
];
@ -59,7 +68,7 @@ export const FILTER_COLUMNS = {
browser: 'browser',
device: 'device',
country: 'country',
region: 'subdivision1',
region: 'region',
city: 'city',
language: 'language',
event: 'event_name',

View file

@ -96,12 +96,12 @@ export async function getLocation(ip: string = '', headers: Headers, hasPayloadI
// Cloudflare headers
if (headers.get('cf-ipcountry')) {
const country = decodeHeader(headers.get('cf-ipcountry'));
const subdivision1 = decodeHeader(headers.get('cf-region-code'));
const region = decodeHeader(headers.get('cf-region-code'));
const city = decodeHeader(headers.get('cf-ipcity'));
return {
country,
subdivision1: getRegionCode(country, subdivision1),
region: getRegionCode(country, region),
city,
};
}
@ -109,12 +109,12 @@ export async function getLocation(ip: string = '', headers: Headers, hasPayloadI
// Vercel headers
if (headers.get('x-vercel-ip-country')) {
const country = decodeHeader(headers.get('x-vercel-ip-country'));
const subdivision1 = decodeHeader(headers.get('x-vercel-ip-country-region'));
const region = decodeHeader(headers.get('x-vercel-ip-country-region'));
const city = decodeHeader(headers.get('x-vercel-ip-city'));
return {
country,
subdivision1: getRegionCode(country, subdivision1),
region: getRegionCode(country, region),
city,
};
}
@ -131,14 +131,12 @@ export async function getLocation(ip: string = '', headers: Headers, hasPayloadI
if (result) {
const country = result.country?.iso_code ?? result?.registered_country?.iso_code;
const subdivision1 = result.subdivisions?.[0]?.iso_code;
const subdivision2 = result.subdivisions?.[1]?.names?.en;
const region = result.subdivisions?.[0]?.iso_code;
const city = result.city?.names?.en;
return {
country,
subdivision1: getRegionCode(country, subdivision1),
subdivision2,
region: getRegionCode(country, region),
city,
};
}
@ -149,14 +147,13 @@ export async function getClientInfo(request: Request, payload: Record<string, an
const ip = payload?.ip || getIpAddress(request.headers);
const location = await getLocation(ip, request.headers, !!payload?.ip);
const country = location?.country;
const subdivision1 = location?.subdivision1;
const subdivision2 = location?.subdivision2;
const region = location?.region;
const city = location?.city;
const browser = browserName(userAgent);
const os = detectOS(userAgent) as string;
const device = getDevice(payload?.screen, os);
return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };
return { userAgent, browser, os, ip, country, region, city, device };
}
export function hasBlockedIp(clientIp: string) {

View file

@ -151,7 +151,7 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):
if (name === 'referrer') {
arr.push(
`and (website_event.referrer_domain != session.hostname or website_event.referrer_domain is null)`,
`and (website_event.referrer_domain != website_event.hostname or website_event.referrer_domain is null)`,
);
}
}

View file

@ -197,8 +197,7 @@ export interface SessionData {
screen: string;
language: string;
country: string;
subdivision1: string;
subdivision2: string;
region: string;
city: string;
ip?: string;
userAgent?: string;

View file

@ -36,8 +36,7 @@ export async function saveEvent(args: {
screen?: string;
language?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
region?: string;
city?: string;
tag?: string;
createdAt?: Date;
@ -72,6 +71,7 @@ async function relationalQuery(data: {
eventName?: string;
eventData?: any;
tag?: string;
hostname?: string;
createdAt?: Date;
}) {
const {
@ -98,6 +98,7 @@ async function relationalQuery(data: {
lifatid,
twclid,
tag,
hostname,
createdAt,
} = data;
const websiteEventId = uuid();
@ -128,6 +129,7 @@ async function relationalQuery(data: {
eventType: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
eventName: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
tag,
hostname,
createdAt,
},
});
@ -177,8 +179,7 @@ async function clickhouseQuery(data: {
screen?: string;
language?: string;
country?: string;
subdivision1?: string;
subdivision2?: string;
region?: string;
city?: string;
tag?: string;
createdAt?: Date;
@ -207,8 +208,7 @@ async function clickhouseQuery(data: {
eventName,
eventData,
country,
subdivision1,
subdivision2,
region,
city,
tag,
createdAt,
@ -225,13 +225,7 @@ async function clickhouseQuery(data: {
visit_id: visitId,
event_id: eventId,
country: country,
subdivision1:
country && subdivision1
? subdivision1.includes('-')
? subdivision1
: `${country}-${subdivision1}`
: null,
subdivision2: subdivision2,
region: country && region ? (region.includes('-') ? region : `${country}-${region}`) : null,
city: city,
url_path: urlPath?.substring(0, URL_LENGTH),
url_query: urlQuery?.substring(0, URL_LENGTH),

View file

@ -41,7 +41,7 @@ async function relationalQuery(
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain = `and website_event.referrer_domain != session.hostname
excludeDomain = `and website_event.referrer_domain != website_event.hostname
and website_event.referrer_domain != ''`;
}

View file

@ -2,34 +2,19 @@ import { Prisma } from '@prisma/client';
import prisma from '@/lib/prisma';
export async function createSession(data: Prisma.SessionCreateInput) {
const {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
city,
} = data;
const { id, websiteId, browser, os, device, screen, language, country, region, city } = data;
return prisma.client.session.create({
data: {
id,
websiteId,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
subdivision2,
region,
city,
},
});

View file

@ -38,7 +38,7 @@ async function relationalQuery(
joinSession: SESSION_COLUMNS.includes(type),
},
);
const includeCountry = column === 'city' || column === 'subdivision1';
const includeCountry = column === 'city' || column === 'region';
return rawQuery(
`
@ -75,7 +75,7 @@ async function clickhouseQuery(
...filters,
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
const includeCountry = column === 'city' || column === 'region';
let sql = '';

View file

@ -23,7 +23,7 @@ async function relationalQuery(websiteId: string, sessionId: string) {
screen,
language,
country,
subdivision1,
region,
city,
min(min_time) as "firstAt",
max(max_time) as "lastAt",
@ -35,14 +35,14 @@ async function relationalQuery(websiteId: string, sessionId: string) {
session.session_id as id,
website_event.visit_id,
session.website_id,
session.hostname,
website_event.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.region,
session.city,
min(website_event.created_at) as min_time,
max(website_event.created_at) as max_time,
@ -52,8 +52,8 @@ async function relationalQuery(websiteId: string, sessionId: string) {
join website_event on website_event.session_id = session.session_id
where session.website_id = {{websiteId::uuid}}
and session.session_id = {{sessionId::uuid}}
group by session.session_id, visit_id, session.website_id, session.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.subdivision1, session.city) t
group by id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city;
group by session.session_id, visit_id, session.website_id, website_event.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
group by id, website_id, hostname, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
@ -73,7 +73,7 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
screen,
language,
country,
subdivision1,
region,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
@ -92,7 +92,7 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
screen,
language,
country,
subdivision1,
region,
city,
min(min_time) as min_time,
max(max_time) as max_time,
@ -101,8 +101,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
group by session_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city) t
group by id, websiteId, hostname, browser, os, device, screen, language, country, subdivision1, city;
group by session_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, region, city) t
group by id, websiteId, hostname, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);

View file

@ -24,14 +24,14 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
select
session.session_id as "id",
session.website_id as "websiteId",
session.hostname,
website_event.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.region,
session.city,
min(website_event.created_at) as "firstAt",
max(website_event.created_at) as "lastAt",
@ -45,14 +45,14 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
${filterQuery}
group by session.session_id,
session.website_id,
session.hostname,
website_event.hostname,
session.browser,
session.os,
session.device,
session.screen,
session.language,
session.country,
session.subdivision1,
session.region,
session.city
order by max(website_event.created_at) desc
limit 1000)
@ -80,7 +80,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
screen,
language,
country,
subdivision1,
region,
city,
${getDateStringSQL('min(min_time)')} as firstAt,
${getDateStringSQL('max(max_time)')} as lastAt,
@ -91,7 +91,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
where website_id = {websiteId:UUID}
${dateQuery}
${filterQuery}
group by session_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city
group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city
order by lastAt desc
limit 1000)
select * from sessions

11014
yarn.lock Normal file

File diff suppressed because it is too large Load diff