diff --git a/db/clickhouse/migrations/07_add_distinct_id.sql b/db/clickhouse/migrations/07_add_distinct_id.sql new file mode 100644 index 00000000..e2356940 --- /dev/null +++ b/db/clickhouse/migrations/07_add_distinct_id.sql @@ -0,0 +1,103 @@ +-- add tag column +ALTER TABLE umami.website_event ADD COLUMN "distinct_id" String AFTER "tag"; +ALTER TABLE umami.website_event_stats_hourly ADD COLUMN "distinct_id" String AFTER "tag"; +ALTER TABLE umami.session_data ADD COLUMN "distinct_id" String AFTER "data_type"; + +-- update materialized view +DROP TABLE umami.website_event_stats_hourly_mv; + +CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv +TO umami.website_event_stats_hourly +AS +SELECT + website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + region, + city, + entry_url, + exit_url, + url_paths as url_path, + url_query, + utm_source, + utm_medium, + utm_campaign, + utm_content, + utm_term, + referrer_domain, + page_title, + gclid, + fbclid, + msclkid, + ttclid, + li_fat_id, + twclid, + event_type, + event_name, + views, + min_time, + max_time, + tag, + distinct_id, + timestamp as created_at +FROM (SELECT + website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + region, + city, + argMinState(url_path, created_at) entry_url, + argMaxState(url_path, created_at) exit_url, + arrayFilter(x -> x != '', groupArray(url_path)) as url_paths, + arrayFilter(x -> x != '', groupArray(url_query)) url_query, + arrayFilter(x -> x != '', groupArray(utm_source)) utm_source, + arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium, + arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign, + arrayFilter(x -> x != '', groupArray(utm_content)) utm_content, + arrayFilter(x -> x != '', groupArray(utm_term)) utm_term, + arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain, + arrayFilter(x -> x != '', groupArray(page_title)) page_title, + arrayFilter(x -> x != '', groupArray(gclid)) gclid, + arrayFilter(x -> x != '', groupArray(fbclid)) fbclid, + arrayFilter(x -> x != '', groupArray(msclkid)) msclkid, + arrayFilter(x -> x != '', groupArray(ttclid)) ttclid, + arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id, + arrayFilter(x -> x != '', groupArray(twclid)) twclid, + event_type, + if(event_type = 2, groupArray(event_name), []) event_name, + sumIf(1, event_type = 1) views, + min(created_at) min_time, + max(created_at) max_time, + arrayFilter(x -> x != '', groupArray(tag)) tag, + distinct_id, + toStartOfHour(created_at) timestamp +FROM umami.website_event +GROUP BY website_id, + session_id, + visit_id, + hostname, + browser, + os, + device, + screen, + language, + country, + region, + city, + event_type, + distinct_id, + timestamp); \ No newline at end of file diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 40268762..fef600e0 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -38,6 +38,7 @@ CREATE TABLE umami.website_event event_type UInt32, event_name String, tag String, + distinct_id String, created_at DateTime('UTC'), job_id Nullable(UUID) ) @@ -75,6 +76,7 @@ CREATE TABLE umami.session_data number_value Nullable(Decimal64(4)), date_value Nullable(DateTime('UTC')), data_type UInt32, + distinct_id String, created_at DateTime('UTC'), job_id Nullable(UUID) ) @@ -120,6 +122,7 @@ CREATE TABLE umami.website_event_stats_hourly min_time SimpleAggregateFunction(min, DateTime('UTC')), max_time SimpleAggregateFunction(max, DateTime('UTC')), tag SimpleAggregateFunction(groupArrayArray, Array(String)), + distinct_id, created_at Datetime('UTC') ) ENGINE = AggregatingMergeTree @@ -172,6 +175,7 @@ SELECT min_time, max_time, tag, + distinct_id, timestamp as created_at FROM (SELECT website_id, @@ -209,6 +213,7 @@ FROM (SELECT min(created_at) min_time, max(created_at) max_time, arrayFilter(x -> x != '', groupArray(tag)) tag, + distinct_id String, toStartOfHour(created_at) timestamp FROM umami.website_event GROUP BY website_id, @@ -224,6 +229,7 @@ GROUP BY website_id, region, city, event_type, + distinct_id, timestamp); -- projections diff --git a/db/mysql/migrations/10_add_distinct_id/migration.sql b/db/mysql/migrations/10_add_distinct_id/migration.sql new file mode 100644 index 00000000..fa3942e4 --- /dev/null +++ b/db/mysql/migrations/10_add_distinct_id/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE `session` ADD COLUMN `distinct_id` VARCHAR(50) NULL; + +-- AlterTable +ALTER TABLE `session_data` ADD COLUMN `distinct_id` VARCHAR(50) NULL; diff --git a/db/mysql/migrations/migration_lock.toml b/db/mysql/migrations/migration_lock.toml index 8a21669a..592fc0b3 100644 --- a/db/mysql/migrations/migration_lock.toml +++ b/db/mysql/migrations/migration_lock.toml @@ -1,3 +1,3 @@ # Please do not edit this file manually # It should be added in your version-control system (e.g., Git) -provider = "mysql" \ No newline at end of file +provider = "mysql" diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prisma index 5f7d6f36..89b455b2 100644 --- a/db/mysql/schema.prisma +++ b/db/mysql/schema.prisma @@ -30,17 +30,18 @@ model User { } model Session { - id String @id @unique @map("session_id") @db.VarChar(36) - websiteId String @map("website_id") @db.VarChar(36) - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - device String? @db.VarChar(20) - screen String? @db.VarChar(11) - language String? @db.VarChar(35) - country String? @db.Char(2) - region String? @db.Char(20) - city String? @db.VarChar(50) - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) + id String @id @unique @map("session_id") @db.VarChar(36) + websiteId String @map("website_id") @db.VarChar(36) + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + device String? @db.VarChar(20) + screen String? @db.VarChar(11) + language String? @db.VarChar(35) + country String? @db.Char(2) + region String? @db.Char(20) + city String? @db.VarChar(50) + distinct_id String? @db.VarChar(50) + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) websiteEvent WebsiteEvent[] sessionData SessionData[] @@ -166,6 +167,7 @@ model SessionData { numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamp(0) dataType Int @map("data_type") @db.UnsignedInt + distinct_id String? @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0) website Website @relation(fields: [websiteId], references: [id]) diff --git a/db/postgresql/migrations/10_add_distinct_id/migration.sql b/db/postgresql/migrations/10_add_distinct_id/migration.sql new file mode 100644 index 00000000..654c23b0 --- /dev/null +++ b/db/postgresql/migrations/10_add_distinct_id/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "session" ADD COLUMN "distinct_id" VARCHAR(50); + +-- AlterTable +ALTER TABLE "session_data" ADD COLUMN "distinct_id" VARCHAR(50); diff --git a/db/postgresql/migrations/migration_lock.toml b/db/postgresql/migrations/migration_lock.toml index 648c57fd..044d57cd 100644 --- a/db/postgresql/migrations/migration_lock.toml +++ b/db/postgresql/migrations/migration_lock.toml @@ -1,3 +1,3 @@ # Please do not edit this file manually # It should be added in your version-control system (e.g., Git) -provider = "postgresql" \ No newline at end of file +provider = "postgresql" diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma index d7c5de08..0ea6d955 100644 --- a/db/postgresql/schema.prisma +++ b/db/postgresql/schema.prisma @@ -30,17 +30,18 @@ model User { } model Session { - id String @id @unique @map("session_id") @db.Uuid - websiteId String @map("website_id") @db.Uuid - browser String? @db.VarChar(20) - os String? @db.VarChar(20) - device String? @db.VarChar(20) - screen String? @db.VarChar(11) - language String? @db.VarChar(35) - country String? @db.Char(2) - region String? @db.VarChar(20) - city String? @db.VarChar(50) - createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) + id String @id @unique @map("session_id") @db.Uuid + websiteId String @map("website_id") @db.Uuid + browser String? @db.VarChar(20) + os String? @db.VarChar(20) + device String? @db.VarChar(20) + screen String? @db.VarChar(11) + language String? @db.VarChar(35) + country String? @db.Char(2) + region String? @db.VarChar(20) + city String? @db.VarChar(50) + distinct_id String? @db.VarChar(50) + createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) websiteEvent WebsiteEvent[] sessionData SessionData[] @@ -166,6 +167,7 @@ model SessionData { numberValue Decimal? @map("number_value") @db.Decimal(19, 4) dateValue DateTime? @map("date_value") @db.Timestamptz(6) dataType Int @map("data_type") @db.Integer + distinct_id String? @db.VarChar(50) createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6) website Website @relation(fields: [websiteId], references: [id]) diff --git a/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx b/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx index 56e0df62..0f193a97 100644 --- a/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx +++ b/src/app/(main)/websites/[websiteId]/sessions/SessionsDataTable.tsx @@ -14,7 +14,7 @@ export default function SessionsDataTable({ const queryResult = useWebsiteSessions(websiteId); return ( - children}> + children}> {({ data }) => } ); diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 7e7e606b..ccbd0044 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -33,17 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers'; export const FILTER_PAGES = 'filter-pages'; export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute']; -export const EVENT_COLUMNS = [ - 'url', - 'entry', - 'exit', - 'referrer', - 'title', - 'query', - 'event', - 'tag', - 'region', -]; +export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag']; export const SESSION_COLUMNS = [ 'browser', @@ -53,6 +43,7 @@ export const SESSION_COLUMNS = [ 'language', 'country', 'city', + 'region', 'host', ]; diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.ts index ca11700b..ff702537 100644 --- a/src/queries/sql/sessions/getWebsiteSessions.ts +++ b/src/queries/sql/sessions/getWebsiteSessions.ts @@ -1,5 +1,5 @@ import clickhouse from '@/lib/clickhouse'; -import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { CLICKHOUSE, getDatabaseType, POSTGRESQL, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import { PageParams, QueryFilters } from '@/lib/types'; @@ -14,10 +14,14 @@ export async function getWebsiteSessions( async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams: PageParams) { const { pagedRawQuery, parseFilters } = prisma; + const { search } = pageParams; const { filterQuery, params } = await parseFilters(websiteId, { ...filters, }); + const db = getDatabaseType(); + const like = db === POSTGRESQL ? 'ilike' : 'like'; + return pagedRawQuery( ` with sessions as ( @@ -43,6 +47,15 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} + ${ + search + ? `and (distinct_id ${like} {{search}} + or city ${like} {{search}} + or browser ${like} {{search}} + or os ${like} {{search}} + or device ${like} {{search}})` + : '' + } group by session.session_id, session.website_id, website_event.hostname, @@ -58,7 +71,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar limit 1000) select * from sessions `, - params, + { ...params, search: `%${search}%` }, pageParams, ); } @@ -66,6 +79,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) { const { pagedQuery, parseFilters, getDateStringSQL } = clickhouse; const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters); + const { search } = pageParams; return pagedQuery( ` @@ -91,12 +105,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar where website_id = {websiteId:UUID} ${dateQuery} ${filterQuery} + ${ + search + ? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0) + or (positionCaseInsensitive(city, {search:String}) > 0) + or (positionCaseInsensitive(browser, {search:String}) > 0) + or (positionCaseInsensitive(os, {search:String}) > 0) + or (positionCaseInsensitive(device, {search:String}) > 0))` + : '' + } group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city order by lastAt desc limit 1000) select * from sessions `, - params, + { ...params, search }, pageParams, ); }