watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-12 00:27:11 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'umami-software:master' into master

Browse source
This commit is contained in:
Adam Stamper 2025-07-28 10:42:03 +01:00 • committed by GitHub
commit 6bacd32b1d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
110 changed files with 4586 additions and 3039 deletions
Download patch file Download diff file Expand all files Collapse all files

253
db/clickhouse/migrations/08_update_hostname_view.sql Normal file
View file

@ -0,0 +1,253 @@
-- create new hourly table
CREATE TABLE umami.website_event_stats_hourly_new
(
website_id UUID,
session_id UUID,
visit_id UUID,
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
region LowCardinality(String),
city String,
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),
utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),
referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
event_type UInt32,
event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
views SimpleAggregateFunction(sum, UInt64),
min_time SimpleAggregateFunction(min, DateTime('UTC')),
max_time SimpleAggregateFunction(max, DateTime('UTC')),
tag SimpleAggregateFunction(groupArrayArray, Array(String)),
distinct_id String,
created_at Datetime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (
website_id,
event_type,
toStartOfHour(created_at),
cityHash64(visit_id),
visit_id
)
SAMPLE BY cityHash64(visit_id);
-- create view
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new
TO umami.website_event_stats_hourly_new
AS
SELECT
website_id,
session_id,
visit_id,
hostnames as hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
distinct_id,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
distinct_id,
timestamp);
-- rename tables
RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;
RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;
-- drop views
DROP TABLE umami.website_event_stats_hourly_mv;
DROP TABLE umami.website_event_stats_hourly_mv_new;
-- recreate view
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
session_id,
visit_id,
hostnames as hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
utm_source,
utm_medium,
utm_campaign,
utm_content,
utm_term,
referrer_domain,
page_title,
gclid,
fbclid,
msclkid,
ttclid,
li_fat_id,
twclid,
event_type,
event_name,
views,
min_time,
max_time,
tag,
distinct_id,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
screen,
language,
country,
region,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
region,
city,
event_type,
distinct_id,
timestamp);

47
db/clickhouse/schema.sql
View file

@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourly
website_id UUID,
session_id UUID,
visit_id UUID,
hostname LowCardinality(String),
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
@ -122,7 +122,7 @@ CREATE TABLE umami.website_event_stats_hourly
min_time SimpleAggregateFunction(min, DateTime('UTC')),
max_time SimpleAggregateFunction(max, DateTime('UTC')),
tag SimpleAggregateFunction(groupArrayArray, Array(String)),
distinct_id,
distinct_id String,
created_at Datetime('UTC')
)
ENGINE = AggregatingMergeTree
@ -143,7 +143,7 @@ SELECT
website_id,
session_id,
visit_id,
hostname,
hostnames as hostname,
browser,
os,
device,
@ -181,7 +181,7 @@ FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
@ -199,7 +199,7 @@ FROM (SELECT
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
@ -213,7 +213,7 @@ FROM (SELECT
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
distinct_id String,
distinct_id,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
@ -246,3 +246,38 @@ SELECT * ORDER BY toStartOfDay(created_at), website_id, referrer_domain, created
);
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_referrer_domain_projection;
-- revenue
CREATE TABLE umami.website_revenue
(
website_id UUID,
session_id UUID,
event_id UUID,
event_name String,
currency String,
revenue DECIMAL(18,4),
created_at DateTime('UTC')
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (website_id, session_id, created_at)
SETTINGS index_granularity = 8192;
CREATE MATERIALIZED VIEW umami.website_revenue_mv
TO umami.website_revenue
AS
SELECT DISTINCT
ed.website_id,
ed.session_id,
ed.event_id,
ed.event_name,
c.currency,
coalesce(toDecimal64(ed.number_value, 2), toDecimal64(ed.string_value, 2)) revenue,
ed.created_at
FROM umami.event_data ed
JOIN (SELECT event_id, string_value as currency
FROM umami.event_data
WHERE positionCaseInsensitive(data_key, 'currency') > 0) c
ON c.event_id = ed.event_id
WHERE positionCaseInsensitive(data_key, 'revenue') > 0;

14
db/mysql/migrations/11_add_segment/migration.sql Normal file
View file

@ -0,0 +1,14 @@
-- CreateTable
CREATE TABLE `segment` (
`segment_id` VARCHAR(36) NOT NULL,
`website_id` VARCHAR(36) NOT NULL,
`type` VARCHAR(200) NOT NULL,
`name` VARCHAR(200) NOT NULL,
`parameters` JSON NOT NULL,
`created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
`updated_at` TIMESTAMP(0) NULL,
UNIQUE INDEX `segment_segment_id_key`(`segment_id`),
INDEX `segment_website_id_idx`(`website_id`),
PRIMARY KEY (`segment_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

2
db/mysql/migrations/12_update_report_parameter/migration.sql Normal file
View file

@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE `report` MODIFY `parameters` JSON NOT NULL;

18
db/mysql/migrations/13_add_revenue/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- CreateTable
CREATE TABLE `revenue` (
`revenue_id` VARCHAR(36) NOT NULL,
`website_id` VARCHAR(36) NOT NULL,
`session_id` VARCHAR(36) NOT NULL,
`event_id` VARCHAR(36) NOT NULL,
`event_name` VARCHAR(50) NOT NULL,
`currency` VARCHAR(100) NOT NULL,
`revenue` DECIMAL(19, 4) NULL,
`created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
UNIQUE INDEX `revenue_revenue_id_key`(`revenue_id`),
INDEX `revenue_website_id_idx`(`website_id`),
INDEX `revenue_session_id_idx`(`session_id`),
INDEX `revenue_website_id_created_at_idx`(`website_id`, `created_at`),
INDEX `revenue_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
PRIMARY KEY (`revenue_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

46
db/mysql/schema.prisma
View file

@ -43,6 +43,7 @@ model Session {
websiteEvent WebsiteEvent[]
sessionData SessionData[]
revenue Revenue[]
@@index([createdAt])
@@index([websiteId])
@ -76,7 +77,9 @@ model Website {
team Team? @relation(fields: [teamId], references: [id])
eventData EventData[]
report Report[]
revenue Revenue[]
sessionData SessionData[]
segment Segment[]
@@index([userId])
@@index([teamId])
@ -215,10 +218,10 @@ model Report {
id String @id() @unique() @map("report_id") @db.VarChar(36)
userId String @map("user_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
type String @map("type") @db.VarChar(200)
name String @map("name") @db.VarChar(200)
description String @map("description") @db.VarChar(500)
parameters String @map("parameters") @db.VarChar(6000)
type String @db.VarChar(200)
name String @db.VarChar(200)
description String @db.VarChar(500)
parameters Json
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
@ -231,3 +234,38 @@ model Report {
@@index([name])
@@map("report")
}
model Segment {
id String @id() @unique() @map("segment_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
type String @db.VarChar(200)
name String @db.VarChar(200)
parameters Json
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
website Website @relation(fields: [websiteId], references: [id])
@@index([websiteId])
@@map("segment")
}
model Revenue {
id String @id() @unique() @map("revenue_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
sessionId String @map("session_id") @db.VarChar(36)
eventId String @map("event_id") @db.VarChar(36)
eventName String @map("event_name") @db.VarChar(50)
currency String @db.VarChar(100)
revenue Decimal? @db.Decimal(19, 4)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
website Website @relation(fields: [websiteId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([websiteId])
@@index([sessionId])
@@index([websiteId, createdAt])
@@index([websiteId, sessionId, createdAt])
@@map("revenue")
}

18
db/postgresql/migrations/11_add_segment/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- CreateTable
CREATE TABLE "segment" (
"segment_id" UUID NOT NULL,
"website_id" UUID NOT NULL,
"type" VARCHAR(200) NOT NULL,
"name" VARCHAR(200) NOT NULL,
"parameters" JSONB NOT NULL,
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMPTZ(6),
CONSTRAINT "segment_pkey" PRIMARY KEY ("segment_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "segment_segment_id_key" ON "segment"("segment_id");
-- CreateIndex
CREATE INDEX "segment_website_id_idx" ON "segment"("website_id");

3
db/postgresql/migrations/12_update_report_parameter/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "report"
ALTER COLUMN "parameters" SET DATA TYPE JSONB USING parameters::JSONB;

28
db/postgresql/migrations/13_add_revenue/migration.sql Normal file
View file

@ -0,0 +1,28 @@
-- CreateTable
CREATE TABLE "revenue" (
"revenue_id" UUID NOT NULL,
"website_id" UUID NOT NULL,
"session_id" UUID NOT NULL,
"event_id" UUID NOT NULL,
"event_name" VARCHAR(50) NOT NULL,
"currency" VARCHAR(100) NOT NULL,
"revenue" DECIMAL(19,4),
"created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "revenue_pkey" PRIMARY KEY ("revenue_id")
);
-- CreateIndex
CREATE UNIQUE INDEX "revenue_revenue_id_key" ON "revenue"("revenue_id");
-- CreateIndex
CREATE INDEX "revenue_website_id_idx" ON "revenue"("website_id");
-- CreateIndex
CREATE INDEX "revenue_session_id_idx" ON "revenue"("session_id");
-- CreateIndex
CREATE INDEX "revenue_website_id_created_at_idx" ON "revenue"("website_id", "created_at");
-- CreateIndex
CREATE INDEX "revenue_website_id_session_id_created_at_idx" ON "revenue"("website_id", "session_id", "created_at");

58
db/postgresql/schema.prisma
View file

@ -44,6 +44,7 @@ model Session {
websiteEvent WebsiteEvent[]
sessionData SessionData[]
revenue Revenue[]
@@index([createdAt])
@@index([websiteId])
@ -77,7 +78,9 @@ model Website {
team Team? @relation(fields: [teamId], references: [id])
eventData EventData[]
report Report[]
revenue Revenue[]
sessionData SessionData[]
segment Segment[]
@@index([userId])
@@index([teamId])
@ -104,12 +107,12 @@ model WebsiteEvent {
referrerQuery String? @map("referrer_query") @db.VarChar(500)
referrerDomain String? @map("referrer_domain") @db.VarChar(500)
pageTitle String? @map("page_title") @db.VarChar(500)
gclid String? @map("gclid") @db.VarChar(255)
fbclid String? @map("fbclid") @db.VarChar(255)
msclkid String? @map("msclkid") @db.VarChar(255)
ttclid String? @map("ttclid") @db.VarChar(255)
gclid String? @db.VarChar(255)
fbclid String? @db.VarChar(255)
msclkid String? @db.VarChar(255)
ttclid String? @db.VarChar(255)
lifatid String? @map("li_fat_id") @db.VarChar(255)
twclid String? @map("twclid") @db.VarChar(255)
twclid String? @db.VarChar(255)
eventType Int @default(1) @map("event_type") @db.Integer
eventName String? @map("event_name") @db.VarChar(50)
tag String? @db.VarChar(50)
@ -200,7 +203,7 @@ model TeamUser {
id String @id() @unique() @map("team_user_id") @db.Uuid
teamId String @map("team_id") @db.Uuid
userId String @map("user_id") @db.Uuid
role String @map("role") @db.VarChar(50)
role String @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
@ -216,10 +219,10 @@ model Report {
id String @id() @unique() @map("report_id") @db.Uuid
userId String @map("user_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
type String @map("type") @db.VarChar(200)
name String @map("name") @db.VarChar(200)
description String @map("description") @db.VarChar(500)
parameters String @map("parameters") @db.VarChar(6000)
type String @db.VarChar(200)
name String @db.VarChar(200)
description String @db.VarChar(500)
parameters Json
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
@ -232,3 +235,38 @@ model Report {
@@index([name])
@@map("report")
}
model Segment {
id String @id() @unique() @map("segment_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
type String @db.VarChar(200)
name String @db.VarChar(200)
parameters Json
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
@@index([websiteId])
@@map("segment")
}
model Revenue {
id String @id() @unique() @map("revenue_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
eventId String @map("event_id") @db.Uuid
eventName String @map("event_name") @db.VarChar(50)
currency String @db.VarChar(100)
revenue Decimal? @db.Decimal(19, 4)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@@index([websiteId])
@@index([sessionId])
@@index([websiteId, createdAt])
@@index([websiteId, sessionId, createdAt])
@@map("revenue")
}