watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-19 03:55:37 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

upgrade

Browse source
This commit is contained in:
badrAG 2025-02-24 20:00:01 +00:00
parent bbf91f31db
commit 6a8b804aff
27 changed files with 16152 additions and 2893 deletions
Download patch file Download diff file Expand all files Collapse all files

57
db/clickhouse/migrations/03_session_data.sql Normal file
View file

@ -0,0 +1,57 @@
CREATE TABLE umami.event_data_new
(
website_id UUID,
session_id UUID,
event_id UUID,
url_path String,
event_name String,
data_key String,
string_value Nullable(String),
number_value Nullable(Decimal64(4)),
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC'),
job_id Nullable(UUID)
)
engine = MergeTree
ORDER BY (website_id, event_id, data_key, created_at)
SETTINGS index_granularity = 8192;
INSERT INTO umami.event_data_new
SELECT website_id,
session_id,
event_id,
url_path,
event_name,
event_key,
string_value,
number_value,
date_value,
data_type,
created_at,
NULL
FROM umami.event_data;
CREATE TABLE umami.session_data
(
website_id UUID,
session_id UUID,
data_key String,
string_value Nullable(String),
number_value Nullable(Decimal64(4)),
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC'),
job_id Nullable(UUID)
)
engine = MergeTree
ORDER BY (website_id, session_id, data_key, created_at)
SETTINGS index_granularity = 8192;
RENAME TABLE umami.event_data TO umami.event_data_old;
RENAME TABLE umami.event_data_new TO umami.event_data;
/*
DROP TABLE umami.event_data_old
*/

77
db/clickhouse/migrations/04_add_tag.sql Normal file
View file

@ -0,0 +1,77 @@
-- add tag column
ALTER TABLE umami.website_event ADD COLUMN "tag" String AFTER "event_name";
ALTER TABLE umami.website_event_stats_hourly ADD COLUMN "tag" SimpleAggregateFunction(groupArrayArray, Array(String)) AFTER "max_time";
-- update materialized view
DROP TABLE umami.website_event_stats_hourly_mv;
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
referrer_domain,
page_title,
event_type,
event_name,
views,
min_time,
max_time,
tag,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
event_type,
timestamp);

165
db/clickhouse/schema.sql
View file

@ -26,12 +26,15 @@ CREATE TABLE umami.website_event
--events
event_type UInt32,
event_name String,
tag String,
created_at DateTime('UTC'),
job_id UUID
job_id Nullable(UUID)
)
engine = MergeTree
ORDER BY (website_id, session_id, created_at)
SETTINGS index_granularity = 8192;
ENGINE = MergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (toStartOfHour(created_at), website_id, session_id, visit_id, created_at)
PRIMARY KEY (toStartOfHour(created_at), website_id, session_id, visit_id)
SETTINGS index_granularity = 8192;
CREATE TABLE umami.event_data
(
@ -40,14 +43,156 @@ CREATE TABLE umami.event_data
event_id UUID,
url_path String,
event_name String,
event_key String,
data_key String,
string_value Nullable(String),
number_value Nullable(Decimal64(4)), --922337203685477.5625
number_value Nullable(Decimal64(4)),
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC'),
job_id UUID
job_id Nullable(UUID)
)
engine = MergeTree
ORDER BY (website_id, event_id, event_key, created_at)
SETTINGS index_granularity = 8192;
ENGINE = MergeTree
ORDER BY (website_id, event_id, data_key, created_at)
SETTINGS index_granularity = 8192;
CREATE TABLE umami.session_data
(
website_id UUID,
session_id UUID,
data_key String,
string_value Nullable(String),
number_value Nullable(Decimal64(4)),
date_value Nullable(DateTime('UTC')),
data_type UInt32,
created_at DateTime('UTC'),
job_id Nullable(UUID)
)
ENGINE = ReplacingMergeTree
ORDER BY (website_id, session_id, data_key)
SETTINGS index_granularity = 8192;
-- stats hourly
CREATE TABLE umami.website_event_stats_hourly
(
website_id UUID,
session_id UUID,
visit_id UUID,
hostname LowCardinality(String),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
screen LowCardinality(String),
language LowCardinality(String),
country LowCardinality(String),
subdivision1 LowCardinality(String),
city String,
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
event_type UInt32,
event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
views SimpleAggregateFunction(sum, UInt64),
min_time SimpleAggregateFunction(min, DateTime('UTC')),
max_time SimpleAggregateFunction(max, DateTime('UTC')),
tag SimpleAggregateFunction(groupArrayArray, Array(String)),
created_at Datetime('UTC')
)
ENGINE = AggregatingMergeTree
PARTITION BY toYYYYMM(created_at)
ORDER BY (
website_id,
event_type,
toStartOfHour(created_at),
cityHash64(visit_id),
visit_id
)
SAMPLE BY cityHash64(visit_id);
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
TO umami.website_event_stats_hourly
AS
SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
entry_url,
exit_url,
url_paths as url_path,
url_query,
referrer_domain,
page_title,
event_type,
event_name,
views,
min_time,
max_time,
tag,
timestamp as created_at
FROM (SELECT
website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
argMinState(url_path, created_at) entry_url,
argMaxState(url_path, created_at) exit_url,
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
event_type,
if(event_type = 2, groupArray(event_name), []) event_name,
sumIf(1, event_type = 1) views,
min(created_at) min_time,
max(created_at) max_time,
arrayFilter(x -> x != '', groupArray(tag)) tag,
toStartOfHour(created_at) timestamp
FROM umami.website_event
GROUP BY website_id,
session_id,
visit_id,
hostname,
browser,
os,
device,
screen,
language,
country,
subdivision1,
city,
event_type,
timestamp);
-- projections
ALTER TABLE umami.website_event
ADD PROJECTION website_event_url_path_projection (
SELECT * ORDER BY toStartOfDay(created_at), website_id, url_path, created_at
);
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_url_path_projection;
ALTER TABLE umami.website_event
ADD PROJECTION website_event_referrer_domain_projection (
SELECT * ORDER BY toStartOfDay(created_at), website_id, referrer_domain, created_at
);
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_referrer_domain_projection;

16
db/mysql/migrations/01_init/migration.sql
View file

@ -11,7 +11,7 @@ CREATE TABLE `user` (
UNIQUE INDEX `user_user_id_key`(`user_id`),
UNIQUE INDEX `user_username_key`(`username`),
PRIMARY KEY (`user_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `session` (
@ -33,7 +33,7 @@ CREATE TABLE `session` (
INDEX `session_created_at_idx`(`created_at`),
INDEX `session_website_id_idx`(`website_id`),
PRIMARY KEY (`session_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `website` (
@ -53,7 +53,7 @@ CREATE TABLE `website` (
INDEX `website_created_at_idx`(`created_at`),
INDEX `website_share_id_idx`(`share_id`),
PRIMARY KEY (`website_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `website_event` (
@ -76,7 +76,7 @@ CREATE TABLE `website_event` (
INDEX `website_event_website_id_created_at_idx`(`website_id`, `created_at`),
INDEX `website_event_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`),
PRIMARY KEY (`event_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `event_data` (
@ -95,7 +95,7 @@ CREATE TABLE `event_data` (
INDEX `event_data_website_event_id_idx`(`website_event_id`),
INDEX `event_data_website_id_website_event_id_created_at_idx`(`website_id`, `website_event_id`, `created_at`),
PRIMARY KEY (`event_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `team` (
@ -109,7 +109,7 @@ CREATE TABLE `team` (
UNIQUE INDEX `team_access_code_key`(`access_code`),
INDEX `team_access_code_idx`(`access_code`),
PRIMARY KEY (`team_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `team_user` (
@ -124,7 +124,7 @@ CREATE TABLE `team_user` (
INDEX `team_user_team_id_idx`(`team_id`),
INDEX `team_user_user_id_idx`(`user_id`),
PRIMARY KEY (`team_user_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `team_website` (
@ -137,7 +137,7 @@ CREATE TABLE `team_website` (
INDEX `team_website_team_id_idx`(`team_id`),
INDEX `team_website_website_id_idx`(`website_id`),
PRIMARY KEY (`team_website_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- AddSystemUser
INSERT INTO user (user_id, username, role, password) VALUES ('41e2b680-648e-4b09-bcd7-3e2b10c06264' , 'admin', 'admin', '$2b$10$BUli0c.muyCW1ErNJc3jL.vFRFtFJWrT8/GcR4A.sUdCznaXiqFXa');

4
db/mysql/migrations/02_report_schema_session_data/migration.sql
View file

@ -21,7 +21,7 @@ CREATE TABLE `session_data` (
INDEX `session_data_website_id_idx`(`website_id`),
INDEX `session_data_session_id_idx`(`session_id`),
PRIMARY KEY (`session_data_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- CreateTable
CREATE TABLE `report` (
@ -41,7 +41,7 @@ CREATE TABLE `report` (
INDEX `report_type_idx`(`type`),
INDEX `report_name_idx`(`name`),
PRIMARY KEY (`report_id`)
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-- EventData migration
UPDATE event_data

20
db/mysql/migrations/06_session_data/migration.sql Normal file
View file

@ -0,0 +1,20 @@
-- DropIndex
DROP INDEX `event_data_website_id_created_at_event_key_idx` ON `event_data`;
-- DropIndex
DROP INDEX `event_data_website_id_website_event_id_created_at_idx` ON `event_data`;
-- AlterTable
ALTER TABLE `event_data` RENAME COLUMN `event_key` TO `data_key`;
-- AlterTable
ALTER TABLE `session_data` RENAME COLUMN `event_key` TO `data_key`;
-- CreateIndex
CREATE INDEX `event_data_website_id_created_at_data_key_idx` ON `event_data`(`website_id`, `created_at`, `data_key`);
-- CreateIndex
CREATE INDEX `session_data_session_id_created_at_idx` ON `session_data`(`session_id`, `created_at`);
-- CreateIndex
CREATE INDEX `session_data_website_id_created_at_data_key_idx` ON `session_data`(`website_id`, `created_at`, `data_key`);

5
db/mysql/migrations/07_add_tag/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE `website_event` ADD COLUMN `tag` VARCHAR(50) NULL;
-- CreateIndex
CREATE INDEX `website_event_website_id_created_at_tag_idx` ON `website_event`(`website_id`, `created_at`, `tag`);

26
db/mysql/schema.prisma
View file

@ -1,5 +1,6 @@
generator client {
provider = "prisma-client-js"
provider = "prisma-client-js"
binaryTargets = ["native"]
}
datasource db {
@ -19,10 +20,10 @@ model User {
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
websiteUser Website[] @relation("user")
websiteCreateUser Website[] @relation("createUser")
teamUser TeamUser[]
report Report[]
websiteUser Website[] @relation("user")
websiteCreateUser Website[] @relation("createUser")
teamUser TeamUser[]
report Report[]
@@map("user")
}
@ -102,6 +103,7 @@ model WebsiteEvent {
pageTitle String? @map("page_title") @db.VarChar(500)
eventType Int @default(1) @map("event_type") @db.UnsignedInt
eventName String? @map("event_name") @db.VarChar(50)
tag String? @db.VarChar(50)
eventData EventData[]
session Session @relation(fields: [sessionId], references: [id])
@ -116,6 +118,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, referrerDomain])
@@index([websiteId, createdAt, pageTitle])
@@index([websiteId, createdAt, eventName])
@@index([websiteId, createdAt, tag])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@map("website_event")
@ -125,7 +128,7 @@ model EventData {
id String @id() @map("event_data_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
websiteEventId String @map("website_event_id") @db.VarChar(36)
eventKey String @map("event_key") @db.VarChar(500)
dataKey String @map("data_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numberValue Decimal? @map("number_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamp(0)
@ -138,9 +141,8 @@ model EventData {
@@index([createdAt])
@@index([websiteId])
@@index([websiteEventId])
@@index([websiteId, websiteEventId, createdAt])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, eventKey])
@@index([websiteId, createdAt, dataKey])
@@map("event_data")
}
@ -148,7 +150,7 @@ model SessionData {
id String @id() @map("session_data_id") @db.VarChar(36)
websiteId String @map("website_id") @db.VarChar(36)
sessionId String @map("session_id") @db.VarChar(36)
eventKey String @map("event_key") @db.VarChar(500)
dataKey String @map("data_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numberValue Decimal? @map("number_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamp(0)
@ -161,6 +163,8 @@ model SessionData {
@@index([createdAt])
@@index([websiteId])
@@index([sessionId])
@@index([sessionId, createdAt])
@@index([websiteId, createdAt, dataKey])
@@map("session_data")
}
@ -173,8 +177,8 @@ model Team {
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)
deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)
website Website[]
teamUser TeamUser[]
website Website[]
teamUser TeamUser[]
@@index([accessCode])
@@map("team")

18
db/postgresql/migrations/06_session_data/migration.sql Normal file
View file

@ -0,0 +1,18 @@
-- DropIndex
DROP INDEX IF EXISTS "event_data_website_id_created_at_event_key_idx";
-- AlterTable
ALTER TABLE "event_data" RENAME COLUMN "event_key" TO "data_key";
-- AlterTable
ALTER TABLE "session_data" DROP COLUMN "deleted_at";
ALTER TABLE "session_data" RENAME COLUMN "session_key" TO "data_key";
-- CreateIndex
CREATE INDEX "event_data_website_id_created_at_data_key_idx" ON "event_data"("website_id", "created_at", "data_key");
-- CreateIndex
CREATE INDEX "session_data_session_id_created_at_idx" ON "session_data"("session_id", "created_at");
-- CreateIndex
CREATE INDEX "session_data_website_id_created_at_data_key_idx" ON "session_data"("website_id", "created_at", "data_key");

5
db/postgresql/migrations/07_add_tag/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "website_event" ADD COLUMN "tag" VARCHAR(50);
-- CreateIndex
CREATE INDEX "website_event_website_id_created_at_tag_idx" ON "website_event"("website_id", "created_at", "tag");

18
db/postgresql/schema.prisma
View file

@ -1,5 +1,6 @@
generator client {
provider = "prisma-client-js"
provider = "prisma-client-js"
binaryTargets = ["native"]
}
datasource db {
@ -19,8 +20,8 @@ model User {
updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6)
websiteUser Website[] @relation("user")
websiteCreateUser Website[] @relation("createUser")
websiteUser Website[] @relation("user")
websiteCreateUser Website[] @relation("createUser")
teamUser TeamUser[]
report Report[]
@ -102,6 +103,7 @@ model WebsiteEvent {
pageTitle String? @map("page_title") @db.VarChar(500)
eventType Int @default(1) @map("event_type") @db.Integer
eventName String? @map("event_name") @db.VarChar(50)
tag String? @db.VarChar(50)
eventData EventData[]
session Session @relation(fields: [sessionId], references: [id])
@ -116,6 +118,7 @@ model WebsiteEvent {
@@index([websiteId, createdAt, referrerDomain])
@@index([websiteId, createdAt, pageTitle])
@@index([websiteId, createdAt, eventName])
@@index([websiteId, createdAt, tag])
@@index([websiteId, sessionId, createdAt])
@@index([websiteId, visitId, createdAt])
@@map("website_event")
@ -125,7 +128,7 @@ model EventData {
id String @id() @map("event_data_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
websiteEventId String @map("website_event_id") @db.Uuid
eventKey String @map("event_key") @db.VarChar(500)
dataKey String @map("data_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numberValue Decimal? @map("number_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamptz(6)
@ -139,7 +142,7 @@ model EventData {
@@index([websiteId])
@@index([websiteEventId])
@@index([websiteId, createdAt])
@@index([websiteId, createdAt, eventKey])
@@index([websiteId, createdAt, dataKey])
@@map("event_data")
}
@ -147,13 +150,12 @@ model SessionData {
id String @id() @map("session_data_id") @db.Uuid
websiteId String @map("website_id") @db.Uuid
sessionId String @map("session_id") @db.Uuid
sessionKey String @map("session_key") @db.VarChar(500)
dataKey String @map("data_key") @db.VarChar(500)
stringValue String? @map("string_value") @db.VarChar(500)
numberValue Decimal? @map("number_value") @db.Decimal(19, 4)
dateValue DateTime? @map("date_value") @db.Timestamptz(6)
dataType Int @map("data_type") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
deletedAt DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6)
website Website @relation(fields: [websiteId], references: [id])
session Session @relation(fields: [sessionId], references: [id])
@ -161,6 +163,8 @@ model SessionData {
@@index([createdAt])
@@index([websiteId])
@@index([sessionId])
@@index([sessionId, createdAt])
@@index([websiteId, createdAt, dataKey])
@@map("session_data")
}