diff --git a/src/queries/sql/events/saveEventData.ts b/src/queries/sql/events/saveEventData.ts new file mode 100644 index 000000000..7c158da40 --- /dev/null +++ b/src/queries/sql/events/saveEventData.ts @@ -0,0 +1,91 @@ +import { Prisma } from '@prisma/client'; +import { DATA_TYPE } from '@/lib/constants'; +import { uuid } from '@/lib/crypto'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { flattenJSON, getStringValue } from '@/lib/data'; +import clickhouse from '@/lib/clickhouse'; +import kafka from '@/lib/kafka'; +import prisma from '@/lib/prisma'; +import { DynamicData } from '@/lib/types'; + +export async function saveEventData(data: { + websiteId: string; + eventId: string; + sessionId?: string; + urlPath?: string; + eventName?: string; + eventData: DynamicData; + createdAt?: string; +}) { + return runQuery({ + [PRISMA]: () => relationalQuery(data), + [CLICKHOUSE]: () => clickhouseQuery(data), + }); +} + +async function relationalQuery(data: { + websiteId: string; + eventId: string; + eventData: DynamicData; +}): Promise { + const { websiteId, eventId, eventData } = data; + + const jsonKeys = flattenJSON(eventData); + + // id, websiteEventId, eventStringValue + const flattenedData = jsonKeys.map(a => ({ + id: uuid(), + websiteEventId: eventId, + websiteId, + dataKey: a.key, + stringValue: getStringValue(a.value, a.dataType), + numberValue: a.dataType === DATA_TYPE.number ? a.value : null, + dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dataType, + })); + + return prisma.client.eventData.createMany({ + data: flattenedData, + }); +} + +async function clickhouseQuery(data: { + websiteId: string; + eventId: string; + sessionId?: string; + urlPath?: string; + eventName?: string; + eventData: DynamicData; + createdAt?: string; +}) { + const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data; + + const { insert, getUTCString } = clickhouse; + const { sendMessage } = kafka; + + const jsonKeys = flattenJSON(eventData); + + const messages = jsonKeys.map(({ key, value, dataType }) => { + return { + website_id: websiteId, + session_id: sessionId, + event_id: eventId, + url_path: urlPath, + event_name: eventName, + data_key: key, + data_type: dataType, + string_value: getStringValue(value, dataType), + number_value: dataType === DATA_TYPE.number ? value : null, + date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, + created_at: createdAt, + }; + }); + + if (kafka.enabled) { + await sendMessage('event_data', messages); + } else { + await insert('event_data', messages); + } + + return data; +} diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts new file mode 100644 index 000000000..19a9b4673 --- /dev/null +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -0,0 +1,182 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export async function getPageviewMetrics( + ...args: [ + websiteId: string, + type: string, + filters: QueryFilters, + limit?: number | string, + offset?: number | string, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + type: string, + filters: QueryFilters, + limit: number | string = 500, + offset: number | string = 0, +) { + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, joinSession, params } = await parseFilters( + websiteId, + { + ...filters, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }, + { joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' }, + ); + + let entryExitQuery = ''; + let excludeDomain = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and website_event.referrer_domain != session.hostname + and website_event.referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + join ( + select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + group by visit_id + ) x + on x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at + `; + } + + return rawQuery( + ` + select ${column} x, + ${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y + from website_event + ${joinSession} + ${entryExitQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${excludeDomain} + ${filterQuery} + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + type: string, + filters: QueryFilters, + limit: number | string = 500, + offset: number | string = 0, +): Promise<{ x: string; y: number }[]> { + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + }); + + let sql = ''; + let excludeDomain = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + let entryExitQuery = ''; + + if (column === 'referrer_domain') { + excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + } + + if (type === 'entry' || type === 'exit') { + const aggregrate = type === 'entry' ? 'min' : 'max'; + + entryExitQuery = ` + JOIN (select visit_id, + ${aggregrate}(created_at) target_created_at + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + group by visit_id) x + ON x.visit_id = website_event.visit_id + and x.target_created_at = website_event.created_at`; + } + + sql = ` + select ${column} x, + ${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y + from website_event + ${entryExitQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${excludeDomain} + ${filterQuery} + group by x + order by y desc + limit ${limit} + offset ${offset} + `; + } else { + let groupByQuery = ''; + let columnQuery = `arrayJoin(${column})`; + + if (column === 'referrer_domain') { + excludeDomain = `and t != hostname and t != ''`; + columnQuery = `session_id s, arrayJoin(${column})`; + } + + if (type === 'entry') { + columnQuery = `visit_id x, argMinMerge(entry_url)`; + } + + if (type === 'exit') { + columnQuery = `visit_id x, argMaxMerge(exit_url)`; + } + + if (type === 'entry' || type === 'exit') { + groupByQuery = 'group by x'; + } + + sql = ` + select g.t as x, + ${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y + from ( + select ${columnQuery} as t + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${excludeDomain} + ${filterQuery} + ${groupByQuery}) as g + group by x + order by y desc + limit ${limit} + offset ${offset} + `; + } + + return rawQuery(sql, params); +} diff --git a/src/queries/sql/pageviews/getPageviewStats.ts b/src/queries/sql/pageviews/getPageviewStats.ts new file mode 100644 index 000000000..f5ace52cd --- /dev/null +++ b/src/queries/sql/pageviews/getPageviewStats.ts @@ -0,0 +1,92 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants'; +import { QueryFilters } from '@/lib/types'; + +export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; + const { getDateSQL, parseFilters, rawQuery } = prisma; + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + ${getDateSQL('website_event.created_at', unit, timezone)} x, + count(*) y + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + order by 1 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { + const { timezone = 'utc', unit = 'day' } = filters; + const { parseFilters, rawQuery, getDateSQL } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + + let sql = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') { + sql = ` + select + g.t as x, + g.y as y + from ( + select + ${getDateSQL('website_event.created_at', unit, timezone)} as t, + count(*) as y + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } else { + sql = ` + select + g.t as x, + g.y as y + from ( + select + ${getDateSQL('website_event.created_at', unit, timezone)} as t, + sum(views)as y + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } + + return rawQuery(sql, params); +} diff --git a/src/queries/sql/reports/getGoals.ts b/src/queries/sql/reports/getGoals.ts new file mode 100644 index 000000000..eda76050f --- /dev/null +++ b/src/queries/sql/reports/getGoals.ts @@ -0,0 +1,375 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +export async function getGoals( + ...args: [ + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + goals: { type: string; value: string; goal: number; operator?: string }[]; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + goals: { type: string; value: string; goal: number; operator?: string }[]; + }, +): Promise { + const { startDate, endDate, goals } = criteria; + const { rawQuery } = prisma; + + const urls = goals.filter(a => a.type === 'url'); + const events = goals.filter(a => a.type === 'event'); + const eventData = goals.filter(a => a.type === 'event-data'); + + const hasUrl = urls.length > 0; + const hasEvent = events.length > 0; + const hasEventData = eventData.length > 0; + + function getParameters( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlParam = urls.reduce((acc, cv, i) => { + acc[`${cv.type}${i}`] = cv.value; + return acc; + }, {}); + + const eventParam = events.reduce((acc, cv, i) => { + acc[`${cv.type}${i}`] = cv.value; + return acc; + }, {}); + + const eventDataParam = eventData.reduce((acc, cv, i) => { + acc[`eventData${i}`] = cv.value; + acc[`property${i}`] = cv.property; + return acc; + }, {}); + + return { + urls: { ...urlParam, startDate, endDate, websiteId }, + events: { ...eventParam, startDate, endDate, websiteId }, + eventData: { ...eventDataParam, startDate, endDate, websiteId }, + }; + } + + function getColumns( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlColumns = urls + .map((a, i) => `COUNT(CASE WHEN url_path = {{url${i}}} THEN 1 END) AS URL${i},`) + .join('\n') + .slice(0, -1); + const eventColumns = events + .map((a, i) => `COUNT(CASE WHEN event_name = {{event${i}}} THEN 1 END) AS EVENT${i},`) + .join('\n') + .slice(0, -1); + const eventDataColumns = eventData + .map( + (a, i) => + `${ + a.operator === 'average' ? 'avg' : a.operator + }(CASE WHEN event_name = {{eventData${i}}} AND data_key = {{property${i}}} THEN ${ + a.operator === 'count' ? '1' : 'number_value' + } END) AS EVENT_DATA${i},`, + ) + .join('\n') + .slice(0, -1); + + return { urls: urlColumns, events: eventColumns, eventData: eventDataColumns }; + } + + function getWhere( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlWhere = urls.map((a, i) => `{{url${i}}}`).join(','); + const eventWhere = events.map((a, i) => `{{event${i}}}`).join(','); + const eventDataNameWhere = eventData.map((a, i) => `{{eventData${i}}}`).join(','); + const eventDataKeyWhere = eventData.map((a, i) => `{{property${i}}}`).join(','); + + return { + urls: `and url_path in (${urlWhere})`, + events: `and event_name in (${eventWhere})`, + eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`, + }; + } + + const parameters = getParameters(urls, events, eventData); + const columns = getColumns(urls, events, eventData); + const where = getWhere(urls, events, eventData); + + const urlResults = hasUrl + ? await rawQuery( + ` + select + ${columns.urls} + from website_event + where website_id = {{websiteId::uuid}} + ${where.urls} + and created_at between {{startDate}} and {{endDate}} + `, + parameters.urls, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => ({ + ...urls[i], + goal: Number(urls[i].goal), + result: Number(results[key]), + })); + }) + : []; + + const eventResults = hasEvent + ? await rawQuery( + ` + select + ${columns.events} + from website_event + where website_id = {{websiteId::uuid}} + ${where.events} + and created_at between {{startDate}} and {{endDate}} + `, + parameters.events, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => { + return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) }; + }); + }) + : []; + + const eventDataResults = hasEventData + ? await rawQuery( + ` + select + ${columns.eventData} + from website_event w + join event_data d + on d.website_event_id = w.event_id + where w.website_id = {{websiteId::uuid}} + ${where.eventData} + and w.created_at between {{startDate}} and {{endDate}} + `, + parameters.eventData, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => { + return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) }; + }); + }) + : []; + + return [...urlResults, ...eventResults, ...eventDataResults]; +} + +async function clickhouseQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + goals: { type: string; value: string; goal: number; operator?: string; property?: string }[]; + }, +): Promise<{ type: string; value: string; goal: number; result: number }[]> { + const { startDate, endDate, goals } = criteria; + const { rawQuery } = clickhouse; + + const urls = goals.filter(a => a.type === 'url'); + const events = goals.filter(a => a.type === 'event'); + const eventData = goals.filter(a => a.type === 'event-data'); + + const hasUrl = urls.length > 0; + const hasEvent = events.length > 0; + const hasEventData = eventData.length > 0; + + function getParameters( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlParam = urls.reduce((acc, cv, i) => { + acc[`${cv.type}${i}`] = cv.value; + return acc; + }, {}); + + const eventParam = events.reduce((acc, cv, i) => { + acc[`${cv.type}${i}`] = cv.value; + return acc; + }, {}); + + const eventDataParam = eventData.reduce((acc, cv, i) => { + acc[`eventData${i}`] = cv.value; + acc[`property${i}`] = cv.property; + return acc; + }, {}); + + return { + urls: { ...urlParam, startDate, endDate, websiteId }, + events: { ...eventParam, startDate, endDate, websiteId }, + eventData: { ...eventDataParam, startDate, endDate, websiteId }, + }; + } + + function getColumns( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlColumns = urls + .map((a, i) => `countIf(url_path = {url${i}:String}) AS URL${i},`) + .join('\n') + .slice(0, -1); + const eventColumns = events + .map((a, i) => `countIf(event_name = {event${i}:String}) AS EVENT${i},`) + .join('\n') + .slice(0, -1); + const eventDataColumns = eventData + .map( + (a, i) => + `${a.operator === 'average' ? 'avg' : a.operator}If(${ + a.operator !== 'count' ? 'number_value, ' : '' + }event_name = {eventData${i}:String} AND data_key = {property${i}:String}) AS EVENT_DATA${i},`, + ) + .join('\n') + .slice(0, -1); + + return { url: urlColumns, events: eventColumns, eventData: eventDataColumns }; + } + + function getWhere( + urls: { type: string; value: string; goal: number }[], + events: { type: string; value: string; goal: number }[], + eventData: { + type: string; + value: string; + goal: number; + operator?: string; + property?: string; + }[], + ) { + const urlWhere = urls.map((a, i) => `{url${i}:String}`).join(','); + const eventWhere = events.map((a, i) => `{event${i}:String}`).join(','); + const eventDataNameWhere = eventData.map((a, i) => `{eventData${i}:String}`).join(','); + const eventDataKeyWhere = eventData.map((a, i) => `{property${i}:String}`).join(','); + + return { + urls: `and url_path in (${urlWhere})`, + events: `and event_name in (${eventWhere})`, + eventData: `and event_name in (${eventDataNameWhere}) and data_key in (${eventDataKeyWhere})`, + }; + } + + const parameters = getParameters(urls, events, eventData); + const columns = getColumns(urls, events, eventData); + const where = getWhere(urls, events, eventData); + + const urlResults = hasUrl + ? await rawQuery( + ` + select + ${columns.url} + from website_event + where website_id = {websiteId:UUID} + ${where.urls} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + `, + parameters.urls, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => { + return { ...urls[i], goal: Number(urls[i].goal), result: Number(results[key]) }; + }); + }) + : []; + + const eventResults = hasEvent + ? await rawQuery( + ` + select + ${columns.events} + from website_event + where website_id = {websiteId:UUID} + ${where.events} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + `, + parameters.events, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => { + return { ...events[i], goal: Number(events[i].goal), result: Number(results[key]) }; + }); + }) + : []; + + const eventDataResults = hasEventData + ? await rawQuery( + ` + select + ${columns.eventData} + from event_data + where website_id = {websiteId:UUID} + ${where.eventData} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + `, + parameters.eventData, + ).then(a => { + const results = a[0]; + + return Object.keys(results).map((key, i) => { + return { ...eventData[i], goal: Number(eventData[i].goal), result: Number(results[key]) }; + }); + }) + : []; + + return [...urlResults, ...eventResults, ...eventDataResults]; +} diff --git a/src/queries/sql/reports/getInsights.ts b/src/queries/sql/reports/getInsights.ts new file mode 100644 index 000000000..d7cdc283f --- /dev/null +++ b/src/queries/sql/reports/getInsights.ts @@ -0,0 +1,127 @@ +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { QueryFilters } from '@/lib/types'; + +export async function getInsights( + ...args: [websiteId: string, fields: { name: string; type?: string }[], filters: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + fields: { name: string; type?: string }[], + filters: QueryFilters, +): Promise< + { + x: string; + y: number; + }[] +> { + const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma; + const { filterQuery, joinSession, params } = await parseFilters( + websiteId, + { + ...filters, + eventType: EVENT_TYPE.pageView, + }, + { + joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)), + }, + ); + + return rawQuery( + ` + select + sum(t.c) as "views", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", + sum(case when t.c = 1 then 1 else 0 end) as "bounces", + sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime", + ${parseFieldsByName(fields)} + from ( + select + ${parseFields(fields)}, + website_event.session_id, + website_event.visit_id, + count(*) as "c", + min(website_event.created_at) as "min_time", + max(website_event.created_at) as "max_time" + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by ${parseFieldsByName(fields)}, + website_event.session_id, website_event.visit_id + ) as t + group by ${parseFieldsByName(fields)} + order by 1 desc, 2 desc + limit 500 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + fields: { name: string; type?: string }[], + filters: QueryFilters, +): Promise< + { + x: string; + y: number; + }[] +> { + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + sum(t.c) as "views", + count(distinct t.session_id) as "visitors", + count(distinct t.visit_id) as "visits", + sum(if(t.c = 1, 1, 0)) as "bounces", + sum(max_time-min_time) as "totaltime", + ${parseFieldsByName(fields)} + from ( + select + ${parseFields(fields)}, + session_id, + visit_id, + count(*) c, + min(created_at) min_time, + max(created_at) max_time + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by ${parseFieldsByName(fields)}, + session_id, visit_id + ) as t + group by ${parseFieldsByName(fields)} + order by 1 desc, 2 desc + limit 500 + `, + params, + ); +} + +function parseFields(fields: { name: any }[]) { + return fields.map(({ name }) => `${FILTER_COLUMNS[name]} as "${name}"`).join(','); +} + +function parseFieldsByName(fields: { name: any }[]) { + return `${fields.map(({ name }) => name).join(',')}`; +} diff --git a/src/queries/sql/reports/getJourney.ts b/src/queries/sql/reports/getJourney.ts new file mode 100644 index 000000000..4c43cc03d --- /dev/null +++ b/src/queries/sql/reports/getJourney.ts @@ -0,0 +1,272 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +interface JourneyResult { + e1: string; + e2: string; + e3: string; + e4: string; + e5: string; + e6: string; + e7: string; + count: number; +} + +export async function getJourney( + ...args: [ + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + steps: number; + startStep?: string; + endStep?: string; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + steps: number; + startStep?: string; + endStep?: string; + }, +): Promise { + const { startDate, endDate, steps, startStep, endStep } = filters; + const { rawQuery } = prisma; + const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery( + steps, + startStep, + endStep, + ); + + function getJourneyQuery( + steps: number, + startStep?: string, + endStep?: string, + ): { + sequenceQuery: string; + startStepQuery: string; + endStepQuery: string; + params: { [key: string]: string }; + } { + const params = {}; + let sequenceQuery = ''; + let startStepQuery = ''; + let endStepQuery = ''; + + // create sequence query + let selectQuery = ''; + let maxQuery = ''; + let groupByQuery = ''; + + for (let i = 1; i <= steps; i++) { + const endQuery = i < steps ? ',' : ''; + selectQuery += `s.e${i},`; + maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`; + groupByQuery += `s.e${i}${endQuery} `; + } + + sequenceQuery = `\nsequences as ( + select ${selectQuery} + count(*) count + FROM ( + select visit_id, + ${maxQuery} + FROM events + group by visit_id) s + group by ${groupByQuery}) + `; + + // create start Step params query + if (startStep) { + startStepQuery = `and e1 = {{startStep}}`; + params['startStep'] = startStep; + } + + // create end Step params query + if (endStep) { + for (let i = 1; i < steps; i++) { + const startQuery = i === 1 ? 'and (' : '\nor '; + endStepQuery += `${startQuery}(e${i} = {{endStep}} and e${i + 1} is null) `; + } + endStepQuery += `\nor (e${steps} = {{endStep}}))`; + + params['endStep'] = endStep; + } + + return { + sequenceQuery, + startStepQuery, + endStepQuery, + params, + }; + } + + return rawQuery( + ` + WITH events AS ( + select distinct + visit_id, + referrer_path, + coalesce(nullIf(event_name, ''), url_path) event, + row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}}), + ${sequenceQuery} + select * + from sequences + where 1 = 1 + ${startStepQuery} + ${endStepQuery} + order by count desc + limit 100 + `, + { + websiteId, + startDate, + endDate, + ...params, + }, + ).then(parseResult); +} + +async function clickhouseQuery( + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + steps: number; + startStep?: string; + endStep?: string; + }, +): Promise { + const { startDate, endDate, steps, startStep, endStep } = filters; + const { rawQuery } = clickhouse; + const { sequenceQuery, startStepQuery, endStepQuery, params } = getJourneyQuery( + steps, + startStep, + endStep, + ); + + function getJourneyQuery( + steps: number, + startStep?: string, + endStep?: string, + ): { + sequenceQuery: string; + startStepQuery: string; + endStepQuery: string; + params: { [key: string]: string }; + } { + const params = {}; + let sequenceQuery = ''; + let startStepQuery = ''; + let endStepQuery = ''; + + // create sequence query + let selectQuery = ''; + let maxQuery = ''; + let groupByQuery = ''; + + for (let i = 1; i <= steps; i++) { + const endQuery = i < steps ? ',' : ''; + selectQuery += `s.e${i},`; + maxQuery += `\nmax(CASE WHEN event_number = ${i} THEN event ELSE NULL END) AS e${i}${endQuery}`; + groupByQuery += `s.e${i}${endQuery} `; + } + + sequenceQuery = `\nsequences as ( + select ${selectQuery} + count(*) count + FROM ( + select visit_id, + ${maxQuery} + FROM events + group by visit_id) s + group by ${groupByQuery}) + `; + + // create start Step params query + if (startStep) { + startStepQuery = `and e1 = {startStep:String}`; + params['startStep'] = startStep; + } + + // create end Step params query + if (endStep) { + for (let i = 1; i < steps; i++) { + const startQuery = i === 1 ? 'and (' : '\nor '; + endStepQuery += `${startQuery}(e${i} = {endStep:String} and e${i + 1} is null) `; + } + endStepQuery += `\nor (e${steps} = {endStep:String}))`; + + params['endStep'] = endStep; + } + + return { + sequenceQuery, + startStepQuery, + endStepQuery, + params, + }; + } + + return rawQuery( + ` + WITH events AS ( + select distinct + visit_id, + coalesce(nullIf(event_name, ''), url_path) event, + row_number() OVER (PARTITION BY visit_id ORDER BY created_at) AS event_number + from umami.website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64}), + ${sequenceQuery} + select * + from sequences + where 1 = 1 + ${startStepQuery} + ${endStepQuery} + order by count desc + limit 100 + `, + { + websiteId, + startDate, + endDate, + ...params, + }, + ).then(parseResult); +} + +function combineSequentialDuplicates(array: any) { + if (array.length === 0) return array; + + const result = [array[0]]; + + for (let i = 1; i < array.length; i++) { + if (array[i] !== array[i - 1]) { + result.push(array[i]); + } + } + + return result; +} + +function parseResult(data: any) { + return data.map(({ e1, e2, e3, e4, e5, e6, e7, count }) => ({ + items: combineSequentialDuplicates([e1, e2, e3, e4, e5, e6, e7]), + count: +Number(count), + })); +} diff --git a/src/queries/sql/reports/getRevenue.ts b/src/queries/sql/reports/getRevenue.ts new file mode 100644 index 000000000..c9c7b74a4 --- /dev/null +++ b/src/queries/sql/reports/getRevenue.ts @@ -0,0 +1,274 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, getDatabaseType, POSTGRESQL, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +export async function getRevenue( + ...args: [ + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + unit: string; + timezone: string; + currency: string; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + unit: string; + timezone: string; + currency: string; + }, +): Promise<{ + chart: { x: string; t: string; y: number }[]; + country: { name: string; value: number }[]; + total: { sum: number; count: number; unique_count: number }; + table: { + currency: string; + sum: number; + count: number; + unique_count: number; + }[]; +}> { + const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria; + const { getDateSQL, rawQuery } = prisma; + const db = getDatabaseType(); + const like = db === POSTGRESQL ? 'ilike' : 'like'; + + const chartRes = await rawQuery( + ` + select + we.event_name x, + ${getDateSQL('ed.created_at', unit, timezone)} t, + sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) y + from event_data ed + join website_event we + on we.event_id = ed.website_event_id + join (select website_event_id + from event_data + where data_key ${like} '%currency%' + and string_value = {{currency}}) currency + on currency.website_event_id = ed.website_event_id + where ed.website_id = {{websiteId::uuid}} + and ed.created_at between {{startDate}} and {{endDate}} + and ed.data_key ${like} '%revenue%' + group by x, t + order by t + `, + { websiteId, startDate, endDate, unit, timezone, currency }, + ); + + const countryRes = await rawQuery( + ` + select + s.country as name, + sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) value + from event_data ed + join website_event we + on we.event_id = ed.website_event_id + join session s + on s.session_id = we.session_id + join (select website_event_id + from event_data + where data_key ${like} '%currency%' + and string_value = {{currency}}) currency + on currency.website_event_id = ed.website_event_id + where ed.website_id = {{websiteId::uuid}} + and ed.created_at between {{startDate}} and {{endDate}} + and ed.data_key ${like} '%revenue%' + group by s.country + `, + { websiteId, startDate, endDate, currency }, + ); + + const totalRes = await rawQuery( + ` + select + sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum, + count(distinct event_id) as count, + count(distinct session_id) as unique_count + from event_data ed + join website_event we + on we.event_id = ed.website_event_id + join (select website_event_id + from event_data + where data_key ${like} '%currency%' + and string_value = {{currency}}) currency + on currency.website_event_id = ed.website_event_id + where ed.website_id = {{websiteId::uuid}} + and ed.created_at between {{startDate}} and {{endDate}} + and ed.data_key ${like} '%revenue%' + `, + { websiteId, startDate, endDate, currency }, + ).then(result => result?.[0]); + + const tableRes = await rawQuery( + ` + select + c.currency, + sum(coalesce(cast(number_value as decimal(10,2)), cast(string_value as decimal(10,2)))) as sum, + count(distinct ed.website_event_id) as count, + count(distinct we.session_id) as unique_count + from event_data ed + join website_event we + on we.event_id = ed.website_event_id + join (select website_event_id, string_value as currency + from event_data + where data_key ${like} '%currency%') c + on c.website_event_id = ed.website_event_id + where ed.website_id = {{websiteId::uuid}} + and ed.created_at between {{startDate}} and {{endDate}} + and ed.data_key ${like} '%revenue%' + group by c.currency + order by sum desc; + `, + { websiteId, startDate, endDate, unit, timezone, currency }, + ); + + return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes }; +} + +async function clickhouseQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + unit: string; + timezone: string; + currency: string; + }, +): Promise<{ + chart: { x: string; t: string; y: number }[]; + country: { name: string; value: number }[]; + total: { sum: number; count: number; unique_count: number }; + table: { + currency: string; + sum: number; + count: number; + unique_count: number; + }[]; +}> { + const { startDate, endDate, timezone = 'UTC', unit = 'day', currency } = criteria; + const { getDateSQL, rawQuery } = clickhouse; + + const chartRes = await rawQuery< + { + x: string; + t: string; + y: number; + }[] + >( + ` + select + event_name x, + ${getDateSQL('created_at', unit, timezone)} t, + sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) y + from event_data + join (select event_id + from event_data + where positionCaseInsensitive(data_key, 'currency') > 0 + and string_value = {currency:String}) currency + on currency.event_id = event_data.event_id + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and positionCaseInsensitive(data_key, 'revenue') > 0 + group by x, t + order by t + `, + { websiteId, startDate, endDate, unit, timezone, currency }, + ); + + const countryRes = await rawQuery< + { + name: string; + value: number; + }[] + >( + ` + select + s.country as name, + sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as value + from event_data ed + join (select event_id + from event_data + where positionCaseInsensitive(data_key, 'currency') > 0 + and string_value = {currency:String}) c + on c.event_id = ed.event_id + join (select distinct website_id, session_id, country + from website_event_stats_hourly + where website_id = {websiteId:UUID}) s + on ed.website_id = s.website_id + and ed.session_id = s.session_id + where ed.website_id = {websiteId:UUID} + and ed.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and positionCaseInsensitive(ed.data_key, 'revenue') > 0 + group by s.country + `, + { websiteId, startDate, endDate, currency }, + ); + + const totalRes = await rawQuery<{ + sum: number; + avg: number; + count: number; + unique_count: number; + }>( + ` + select + sum(coalesce(toDecimal64(number_value, 2), toDecimal64(string_value, 2))) as sum, + uniqExact(event_id) as count, + uniqExact(session_id) as unique_count + from event_data + join (select event_id + from event_data + where positionCaseInsensitive(data_key, 'currency') > 0 + and string_value = {currency:String}) currency + on currency.event_id = event_data.event_id + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and positionCaseInsensitive(data_key, 'revenue') > 0 + `, + { websiteId, startDate, endDate, currency }, + ).then(result => result?.[0]); + + const tableRes = await rawQuery< + { + currency: string; + sum: number; + avg: number; + count: number; + unique_count: number; + }[] + >( + ` + select + c.currency, + sum(coalesce(toDecimal64(ed.number_value, 2), toDecimal64(ed.string_value, 2))) as sum, + uniqExact(ed.event_id) as count, + uniqExact(ed.session_id) as unique_count + from event_data ed + join (select event_id, string_value as currency + from event_data + where positionCaseInsensitive(data_key, 'currency') > 0) c + on c.event_id = ed.event_id + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and positionCaseInsensitive(data_key, 'revenue') > 0 + group by c.currency + order by sum desc; + `, + { websiteId, startDate, endDate, unit, timezone, currency }, + ); + + return { chart: chartRes, country: countryRes, total: totalRes, table: tableRes }; +} diff --git a/src/queries/sql/reports/getRevenueValues.ts b/src/queries/sql/reports/getRevenueValues.ts new file mode 100644 index 000000000..a46bf0bfd --- /dev/null +++ b/src/queries/sql/reports/getRevenueValues.ts @@ -0,0 +1,75 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { runQuery, CLICKHOUSE, PRISMA, getDatabaseType, POSTGRESQL } from '@/lib/db'; + +export async function getRevenueValues( + ...args: [ + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + }, +) { + const { rawQuery } = prisma; + const { startDate, endDate } = criteria; + + const db = getDatabaseType(); + const like = db === POSTGRESQL ? 'ilike' : 'like'; + + return rawQuery( + ` + select distinct string_value as currency + from event_data + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and data_key ${like} '%currency%' + order by currency + `, + { + websiteId, + startDate, + endDate, + }, + ); +} + +async function clickhouseQuery( + websiteId: string, + criteria: { + startDate: Date; + endDate: Date; + }, +) { + const { rawQuery } = clickhouse; + const { startDate, endDate } = criteria; + + return rawQuery( + ` + select distinct string_value as currency + from event_data + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and positionCaseInsensitive(data_key, 'currency') > 0 + order by currency + `, + { + websiteId, + startDate, + endDate, + }, + ); +} diff --git a/src/queries/sql/reports/getUTM.ts b/src/queries/sql/reports/getUTM.ts new file mode 100644 index 000000000..5463815bb --- /dev/null +++ b/src/queries/sql/reports/getUTM.ts @@ -0,0 +1,102 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +export async function getUTM( + ...args: [ + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, +) { + const { startDate, endDate } = filters; + const { rawQuery } = prisma; + + return rawQuery( + ` + select url_query, count(*) as "num" + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and coalesce(url_query, '') != '' + and event_type = 1 + group by 1 + `, + { + websiteId, + startDate, + endDate, + }, + ).then(result => parseParameters(result as any[])); +} + +async function clickhouseQuery( + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, +) { + const { startDate, endDate } = filters; + const { rawQuery } = clickhouse; + + return rawQuery( + ` + select url_query, count(*) as "num" + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and url_query != '' + and event_type = 1 + group by 1 + `, + { + websiteId, + startDate, + endDate, + }, + ).then(result => parseParameters(result as any[])); +} + +function parseParameters(data: any[]) { + return data.reduce((obj, { url_query, num }) => { + try { + const searchParams = new URLSearchParams(url_query); + + for (const [key, value] of searchParams) { + if (key.match(/^utm_(\w+)$/)) { + const name = value; + if (!obj[key]) { + obj[key] = { [name]: Number(num) }; + } else if (!obj[key][name]) { + obj[key][name] = Number(num); + } else { + obj[key][name] += Number(num); + } + } + } + } catch { + // Ignore + } + + return obj; + }, {}); +} diff --git a/src/queries/sql/sessions/getSessionActivity.ts b/src/queries/sql/sessions/getSessionActivity.ts new file mode 100644 index 000000000..d7e2a4132 --- /dev/null +++ b/src/queries/sql/sessions/getSessionActivity.ts @@ -0,0 +1,59 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; + +export async function getSessionActivity( + ...args: [websiteId: string, sessionId: string, startDate: Date, endDate: Date] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + sessionId: string, + startDate: Date, + endDate: Date, +) { + return prisma.client.websiteEvent.findMany({ + where: { + sessionId, + websiteId, + createdAt: { gte: startDate, lte: endDate }, + }, + take: 500, + orderBy: { createdAt: 'desc' }, + }); +} + +async function clickhouseQuery( + websiteId: string, + sessionId: string, + startDate: Date, + endDate: Date, +) { + const { rawQuery } = clickhouse; + + return rawQuery( + ` + select + created_at as createdAt, + url_path as urlPath, + url_query as urlQuery, + referrer_domain as referrerDomain, + event_id as eventId, + event_type as eventType, + event_name as eventName, + visit_id as visitId + from website_event + where website_id = {websiteId:UUID} + and session_id = {sessionId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + order by created_at desc + limit 500 + `, + { websiteId, sessionId, startDate, endDate }, + ); +} diff --git a/src/queries/sql/sessions/getSessionData.ts b/src/queries/sql/sessions/getSessionData.ts new file mode 100644 index 000000000..a3f1e113b --- /dev/null +++ b/src/queries/sql/sessions/getSessionData.ts @@ -0,0 +1,56 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db'; + +export async function getSessionData(...args: [websiteId: string, sessionId: string]) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, sessionId: string) { + const { rawQuery } = prisma; + + return rawQuery( + ` + select + website_id as "websiteId", + session_id as "sessionId", + data_key as "dataKey", + data_type as "dataType", + replace(string_value, '.0000', '') as "stringValue", + number_value as "numberValue", + date_value as "dateValue", + created_at as "createdAt" + from session_data + where website_id = {{websiteId::uuid}} + and session_id = {{sessionId::uuid}} + order by data_key asc + `, + { websiteId, sessionId }, + ); +} + +async function clickhouseQuery(websiteId: string, sessionId: string) { + const { rawQuery } = clickhouse; + + return rawQuery( + ` + select + website_id as websiteId, + session_id as sessionId, + data_key as dataKey, + data_type as dataType, + replace(string_value, '.0000', '') as stringValue, + number_value as numberValue, + date_value as dateValue, + created_at as createdAt + from session_data final + where website_id = {websiteId:UUID} + and session_id = {sessionId:UUID} + order by data_key asc + `, + { websiteId, sessionId }, + ); +} diff --git a/src/queries/sql/sessions/getSessionDataProperties.ts b/src/queries/sql/sessions/getSessionDataProperties.ts new file mode 100644 index 000000000..20fb11d51 --- /dev/null +++ b/src/queries/sql/sessions/getSessionDataProperties.ts @@ -0,0 +1,70 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { QueryFilters, WebsiteEventData } from '@/lib/types'; + +export async function getSessionDataProperties( + ...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + filters: QueryFilters & { propertyName?: string }, +) { + const { rawQuery, parseFilters } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, filters, { + columns: { propertyName: 'data_key' }, + }); + + return rawQuery( + ` + select + data_key as "propertyName", + count(distinct d.session_id) as "total" + from website_event e + join session_data d + on d.session_id = e.session_id + where e.website_id = {{websiteId::uuid}} + and e.created_at between {{startDate}} and {{endDate}} + ${filterQuery} + group by 1 + order by 2 desc + limit 500 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters & { propertyName?: string }, +): Promise<{ propertyName: string; total: number }[]> { + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, filters, { + columns: { propertyName: 'data_key' }, + }); + + return rawQuery( + ` + select + data_key as propertyName, + count(distinct d.session_id) as total + from website_event e + join session_data d final + on d.session_id = e.session_id + where e.website_id = {websiteId:UUID} + and e.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and d.data_key != '' + ${filterQuery} + group by 1 + order by 2 desc + limit 500 + `, + params, + ); +} diff --git a/src/queries/sql/sessions/getSessionDataValues.ts b/src/queries/sql/sessions/getSessionDataValues.ts new file mode 100644 index 000000000..8cd6a4ab8 --- /dev/null +++ b/src/queries/sql/sessions/getSessionDataValues.ts @@ -0,0 +1,73 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import { QueryFilters, WebsiteEventData } from '@/lib/types'; + +export async function getSessionDataValues( + ...args: [websiteId: string, filters: QueryFilters & { propertyName?: string }] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + filters: QueryFilters & { propertyName?: string }, +) { + const { rawQuery, parseFilters, getDateSQL } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + case + when data_type = 2 then replace(string_value, '.0000', '') + when data_type = 4 then ${getDateSQL('date_value', 'hour')} + else string_value + end as "value", + count(distinct d.session_id) as "total" + from website_event e + join session_data d + on d.session_id = e.session_id + where e.website_id = {{websiteId::uuid}} + and e.created_at between {{startDate}} and {{endDate}} + and d.data_key = {{propertyName}} + ${filterQuery} + group by value + order by 2 desc + limit 100 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters & { propertyName?: string }, +): Promise<{ propertyName: string; dataType: number; propertyValue: string; total: number }[]> { + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), + data_type = 4, toString(date_trunc('hour', date_value)), + string_value) as "value", + uniq(d.session_id) as "total" + from website_event e + join session_data d final + on d.session_id = e.session_id + where e.website_id = {websiteId:UUID} + and e.created_at between {startDate:DateTime64} and {endDate:DateTime64} + and d.data_key = {propertyName:String} + ${filterQuery} + group by value + order by 2 desc + limit 100 + `, + params, + ); +} diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts new file mode 100644 index 000000000..010989b5d --- /dev/null +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -0,0 +1,119 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export async function getSessionMetrics( + ...args: [ + websiteId: string, + type: string, + filters: QueryFilters, + limit?: number | string, + offset?: number | string, + ] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + type: string, + filters: QueryFilters, + limit: number | string = 500, + offset: number | string = 0, +) { + const column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = prisma; + const { filterQuery, joinSession, params } = await parseFilters( + websiteId, + { + ...filters, + eventType: EVENT_TYPE.pageView, + }, + { + joinSession: SESSION_COLUMNS.includes(type), + }, + ); + const includeCountry = column === 'city' || column === 'subdivision1'; + + return rawQuery( + ` + select + ${column} x, + count(distinct website_event.session_id) y + ${includeCountry ? ', country' : ''} + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = {{eventType}} + ${filterQuery} + group by 1 + ${includeCountry ? ', 3' : ''} + order by 2 desc + limit ${limit} + offset ${offset} + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + type: string, + filters: QueryFilters, + limit: number | string = 500, + offset: number | string = 0, +): Promise<{ x: string; y: number }[]> { + const column = FILTER_COLUMNS[type] || type; + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + const includeCountry = column === 'city' || column === 'subdivision1'; + + let sql = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) { + sql = ` + select + ${column} x, + count(distinct session_id) y + ${includeCountry ? ', country' : ''} + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x + ${includeCountry ? ', country' : ''} + order by y desc + limit ${limit} + offset ${offset} + `; + } else { + sql = ` + select + ${column} x, + uniq(session_id) y + ${includeCountry ? ', country' : ''} + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x + ${includeCountry ? ', country' : ''} + order by y desc + limit ${limit} + offset ${offset} + `; + } + + return rawQuery(sql, params); +} diff --git a/src/queries/sql/sessions/getSessionStats.ts b/src/queries/sql/sessions/getSessionStats.ts new file mode 100644 index 000000000..22cc04a73 --- /dev/null +++ b/src/queries/sql/sessions/getSessionStats.ts @@ -0,0 +1,92 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; + const { getDateSQL, parseFilters, rawQuery } = prisma; + const { filterQuery, joinSession, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + + return rawQuery( + ` + select + ${getDateSQL('website_event.created_at', unit, timezone)} x, + count(distinct website_event.session_id) y + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + order by 1 + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; y: number }[]> { + const { timezone = 'utc', unit = 'day' } = filters; + const { parseFilters, rawQuery, getDateSQL } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + eventType: EVENT_TYPE.pageView, + }); + + let sql = ''; + + if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') { + sql = ` + select + g.t as x, + g.y as y + from ( + select + ${getDateSQL('website_event.created_at', unit, timezone)} as t, + count(distinct session_id) as y + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } else { + sql = ` + select + g.t as x, + g.y as y + from ( + select + ${getDateSQL('website_event.created_at', unit, timezone)} as t, + uniq(session_id) as y + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `; + } + + return rawQuery(sql, params); +} diff --git a/src/queries/sql/sessions/getWebsiteSession.ts b/src/queries/sql/sessions/getWebsiteSession.ts new file mode 100644 index 000000000..45e8640a4 --- /dev/null +++ b/src/queries/sql/sessions/getWebsiteSession.ts @@ -0,0 +1,109 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db'; + +export async function getWebsiteSession(...args: [websiteId: string, sessionId: string]) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, sessionId: string) { + const { rawQuery, getTimestampDiffSQL } = prisma; + + return rawQuery( + ` + select id, + website_id as "websiteId", + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + min(min_time) as "firstAt", + max(max_time) as "lastAt", + count(distinct visit_id) as visits, + sum(views) as views, + sum(events) as events, + sum(${getTimestampDiffSQL('min_time', 'max_time')}) as "totaltime" + from (select + session.session_id as id, + website_event.visit_id, + session.website_id, + session.hostname, + session.browser, + session.os, + session.device, + session.screen, + session.language, + session.country, + session.subdivision1, + session.city, + min(website_event.created_at) as min_time, + max(website_event.created_at) as max_time, + sum(case when website_event.event_type = 1 then 1 else 0 end) as views, + sum(case when website_event.event_type = 2 then 1 else 0 end) as events + from session + join website_event on website_event.session_id = session.session_id + where session.website_id = {{websiteId::uuid}} + and session.session_id = {{sessionId::uuid}} + group by session.session_id, visit_id, session.website_id, session.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.subdivision1, session.city) t + group by id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city; + `, + { websiteId, sessionId }, + ).then(result => result?.[0]); +} + +async function clickhouseQuery(websiteId: string, sessionId: string) { + const { rawQuery, getDateStringSQL } = clickhouse; + + return rawQuery( + ` + select id, + websiteId, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + ${getDateStringSQL('min(min_time)')} as firstAt, + ${getDateStringSQL('max(max_time)')} as lastAt, + uniq(visit_id) visits, + sum(views) as views, + sum(events) as events, + sum(max_time-min_time) as totaltime + from (select + session_id as id, + visit_id, + website_id as websiteId, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + min(min_time) as min_time, + max(max_time) as max_time, + sum(views) as views, + length(groupArrayArray(event_name)) as events + from website_event_stats_hourly + where website_id = {websiteId:UUID} + and session_id = {sessionId:UUID} + group by session_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city) t + group by id, websiteId, hostname, browser, os, device, screen, language, country, subdivision1, city; + `, + { websiteId, sessionId }, + ).then(result => result?.[0]); +} diff --git a/src/queries/sql/sessions/getWebsiteSessionStats.ts b/src/queries/sql/sessions/getWebsiteSessionStats.ts new file mode 100644 index 000000000..2463b7adf --- /dev/null +++ b/src/queries/sql/sessions/getWebsiteSessionStats.ts @@ -0,0 +1,72 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export async function getWebsiteSessionStats( + ...args: [websiteId: string, filters: QueryFilters] +): Promise< + { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] +> { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery( + websiteId: string, + filters: QueryFilters, +): Promise< + { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] +> { + const { parseFilters, rawQuery } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + }); + + return rawQuery( + ` + select + count(*) as "pageviews", + count(distinct website_event.session_id) as "visitors", + count(distinct website_event.visit_id) as "visits", + count(distinct session.country) as "countries", + sum(case when website_event.event_type = 2 then 1 else 0 end) as "events" + from website_event + join session on website_event.session_id = session.session_id + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + ${filterQuery} + `, + params, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise< + { pageviews: number; visitors: number; visits: number; countries: number; events: number }[] +> { + const { rawQuery, parseFilters } = clickhouse; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + }); + + return rawQuery( + ` + select + sum(views) as "pageviews", + uniq(session_id) as "visitors", + uniq(visit_id) as "visits", + uniq(country) as "countries", + sum(length(event_name)) as "events" + from umami.website_event_stats_hourly "website_event" + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + ${filterQuery} + `, + params, + ); +} diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.ts new file mode 100644 index 000000000..264a084bf --- /dev/null +++ b/src/queries/sql/sessions/getWebsiteSessions.ts @@ -0,0 +1,102 @@ +import clickhouse from '@/lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { PageParams, QueryFilters } from '@/lib/types'; + +export async function getWebsiteSessions( + ...args: [websiteId: string, filters?: QueryFilters, pageParams?: PageParams] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters, pageParams: PageParams) { + const { pagedRawQuery, parseFilters } = prisma; + const { filterQuery, params } = await parseFilters(websiteId, { + ...filters, + }); + + return pagedRawQuery( + ` + with sessions as ( + select + session.session_id as "id", + session.website_id as "websiteId", + session.hostname, + session.browser, + session.os, + session.device, + session.screen, + session.language, + session.country, + session.subdivision1, + session.city, + min(website_event.created_at) as "firstAt", + max(website_event.created_at) as "lastAt", + count(distinct website_event.visit_id) as "visits", + sum(case when website_event.event_type = 1 then 1 else 0 end) as "views", + max(website_event.created_at) as "createdAt" + from website_event + join session on session.session_id = website_event.session_id + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + ${filterQuery} + group by session.session_id, + session.website_id, + session.hostname, + session.browser, + session.os, + session.device, + session.screen, + session.language, + session.country, + session.subdivision1, + session.city + order by max(website_event.created_at) desc + limit 1000) + select * from sessions + `, + params, + pageParams, + ); +} + +async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParams?: PageParams) { + const { pagedQuery, parseFilters, getDateStringSQL } = clickhouse; + const { params, dateQuery, filterQuery } = await parseFilters(websiteId, filters); + + return pagedQuery( + ` + with sessions as ( + select + session_id as id, + website_id as websiteId, + hostname, + browser, + os, + device, + screen, + language, + country, + subdivision1, + city, + ${getDateStringSQL('min(min_time)')} as firstAt, + ${getDateStringSQL('max(max_time)')} as lastAt, + uniq(visit_id) as visits, + sumIf(views, event_type = 1) as views, + lastAt as createdAt + from website_event_stats_hourly + where website_id = {websiteId:UUID} + ${dateQuery} + ${filterQuery} + group by session_id, website_id, hostname, browser, os, device, screen, language, country, subdivision1, city + order by lastAt desc + limit 1000) + select * from sessions + `, + params, + pageParams, + ); +} diff --git a/src/queries/sql/sessions/getWebsiteSessionsWeekly.ts b/src/queries/sql/sessions/getWebsiteSessionsWeekly.ts new file mode 100644 index 000000000..58f8d6922 --- /dev/null +++ b/src/queries/sql/sessions/getWebsiteSessionsWeekly.ts @@ -0,0 +1,71 @@ +import prisma from '@/lib/prisma'; +import clickhouse from '@/lib/clickhouse'; +import { runQuery, PRISMA, CLICKHOUSE } from '@/lib/db'; +import { QueryFilters } from '@/lib/types'; + +export async function getWebsiteSessionsWeekly( + ...args: [websiteId: string, filters?: QueryFilters] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc' } = filters; + const { rawQuery, getDateWeeklySQL, parseFilters } = prisma; + const { params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + ${getDateWeeklySQL('created_at', timezone)} as time, + count(distinct session_id) as value + from website_event + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + group by time + order by 2 + `, + params, + ).then(formatResults); +} + +async function clickhouseQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc' } = filters; + const { rawQuery, parseFilters } = clickhouse; + const { params } = await parseFilters(websiteId, filters); + + return rawQuery( + ` + select + formatDateTime(toDateTime(created_at, '${timezone}'), '%w:%H') as time, + count(distinct session_id) as value + from website_event_stats_hourly + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + group by time + order by time + `, + params, + ).then(formatResults); +} + +function formatResults(data: any) { + const days = []; + + for (let i = 0; i < 7; i++) { + days.push([]); + + for (let j = 0; j < 24; j++) { + days[i].push( + Number( + data.find(({ time }) => time === `${i}:${j.toString().padStart(2, '0')}`)?.value || 0, + ), + ); + } + } + + return days; +} diff --git a/src/queries/sql/sessions/saveSessionData.ts b/src/queries/sql/sessions/saveSessionData.ts new file mode 100644 index 000000000..35f0c7126 --- /dev/null +++ b/src/queries/sql/sessions/saveSessionData.ts @@ -0,0 +1,109 @@ +import { DATA_TYPE } from '@/lib/constants'; +import { uuid } from '@/lib/crypto'; +import { flattenJSON, getStringValue } from '@/lib/data'; +import prisma from '@/lib/prisma'; +import { DynamicData } from '@/lib/types'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import kafka from '@/lib/kafka'; +import clickhouse from '@/lib/clickhouse'; + +export async function saveSessionData(data: { + websiteId: string; + sessionId: string; + sessionData: DynamicData; +}) { + return runQuery({ + [PRISMA]: () => relationalQuery(data), + [CLICKHOUSE]: () => clickhouseQuery(data), + }); +} + +export async function relationalQuery(data: { + websiteId: string; + sessionId: string; + sessionData: DynamicData; +}) { + const { client } = prisma; + const { websiteId, sessionId, sessionData } = data; + + const jsonKeys = flattenJSON(sessionData); + + const flattenedData = jsonKeys.map(a => ({ + id: uuid(), + websiteId, + sessionId, + dataKey: a.key, + stringValue: getStringValue(a.value, a.dataType), + numberValue: a.dataType === DATA_TYPE.number ? a.value : null, + dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null, + dataType: a.dataType, + })); + + const existing = await client.sessionData.findMany({ + where: { + sessionId, + }, + select: { + id: true, + sessionId: true, + dataKey: true, + }, + }); + + for (const data of flattenedData) { + const { sessionId, dataKey, ...props } = data; + const record = existing.find(e => e.sessionId === sessionId && e.dataKey === dataKey); + + if (record) { + await client.sessionData.update({ + where: { + id: record.id, + }, + data: { + ...props, + }, + }); + } else { + await client.sessionData.create({ + data, + }); + } + } + + return flattenedData; +} + +async function clickhouseQuery(data: { + websiteId: string; + sessionId: string; + sessionData: DynamicData; +}) { + const { websiteId, sessionId, sessionData } = data; + + const { insert, getUTCString } = clickhouse; + const { sendMessage } = kafka; + const createdAt = getUTCString(); + + const jsonKeys = flattenJSON(sessionData); + + const messages = jsonKeys.map(({ key, value, dataType }) => { + return { + website_id: websiteId, + session_id: sessionId, + data_key: key, + data_type: dataType, + string_value: getStringValue(value, dataType), + number_value: dataType === DATA_TYPE.number ? value : null, + date_value: dataType === DATA_TYPE.date ? getUTCString(value) : null, + created_at: createdAt, + }; + }); + + if (kafka.enabled) { + await sendMessage('session_data', messages); + } else { + await insert('session_data', messages); + } + + return data; +} diff --git a/src/store/app.ts b/src/store/app.ts index 4d547d4e8..0890b7e9c 100644 --- a/src/store/app.ts +++ b/src/store/app.ts @@ -7,9 +7,9 @@ import { LOCALE_CONFIG, THEME_CONFIG, TIMEZONE_CONFIG, -} from 'lib/constants'; -import { getItem } from 'next-basics'; -import { getTimezone } from 'lib/date'; +} from '@/lib/constants'; +import { getItem } from '@/lib/storage'; +import { getTimezone } from '@/lib/date'; function getDefaultTheme() { return typeof window !== 'undefined' diff --git a/src/store/dashboard.ts b/src/store/dashboard.ts index f66775428..a34ec384c 100644 --- a/src/store/dashboard.ts +++ b/src/store/dashboard.ts @@ -1,12 +1,14 @@ import { create } from 'zustand'; -import { DASHBOARD_CONFIG, DEFAULT_WEBSITE_LIMIT } from 'lib/constants'; -import { getItem, setItem } from 'next-basics'; +import { DASHBOARD_CONFIG, DEFAULT_WEBSITE_LIMIT } from '@/lib/constants'; +import { getItem, setItem } from '@/lib/storage'; export const initialState = { showCharts: true, limit: DEFAULT_WEBSITE_LIMIT, websiteOrder: [], + websiteActive: [], editing: false, + isEdited: false, }; const store = create(() => ({ ...initialState, ...getItem(DASHBOARD_CONFIG) })); diff --git a/src/store/version.ts b/src/store/version.ts index 3b5afaac5..9a889636f 100644 --- a/src/store/version.ts +++ b/src/store/version.ts @@ -1,8 +1,8 @@ import { create } from 'zustand'; import { produce } from 'immer'; import semver from 'semver'; -import { CURRENT_VERSION, VERSION_CHECK, UPDATES_URL } from 'lib/constants'; -import { getItem } from 'next-basics'; +import { CURRENT_VERSION, VERSION_CHECK, UPDATES_URL } from '@/lib/constants'; +import { getItem } from '@/lib/storage'; const initialState = { current: CURRENT_VERSION, diff --git a/src/store/websites.ts b/src/store/websites.ts index a9f6b44da..e9271abda 100644 --- a/src/store/websites.ts +++ b/src/store/websites.ts @@ -1,6 +1,6 @@ import { create } from 'zustand'; import { produce } from 'immer'; -import { DateRange } from 'lib/types'; +import { DateRange } from '@/lib/types'; const store = create(() => ({})); @@ -18,4 +18,18 @@ export function setWebsiteDateRange(websiteId: string, dateRange: DateRange) { ); } +export function setWebsiteDateCompare(websiteId: string, dateCompare: string) { + store.setState( + produce(state => { + if (!state[websiteId]) { + state[websiteId] = {}; + } + + state[websiteId].dateCompare = dateCompare; + + return state; + }), + ); +} + export default store; diff --git a/src/styles/locale.css b/src/styles/locale.css deleted file mode 100644 index dddf495b6..000000000 --- a/src/styles/locale.css +++ /dev/null @@ -1,29 +0,0 @@ -.zh-CN { - font-family: '方体', 'PingFang SC', '黑体', 'Heiti SC', 'Microsoft JhengHei UI', - 'Microsoft JhengHei', Roboto, Noto, 'Noto Sans CJK SC', sans-serif !important; -} - -.zh-TW { - font-family: '方體', 'PingFang TC', '黑體', 'Heiti TC', 'Microsoft JhengHei UI', - 'Microsoft JhengHei', Roboto, Noto, 'Noto Sans CJK TC', sans-serif !important; -} - -.ja-JP { - font-family: '游ゴシック体', YuGothic, 'ヒラギノ丸ゴ', 'Hiragino Sans', 'Yu Gothic UI', - 'Meiryo UI', 'MS Gothic', Roboto, Noto, 'Noto Sans CJK JP', sans-serif !important; -} - -.ko-KR { - font-family: 'Nanum Gothic', 'Apple SD Gothic Neo', 'Malgun Gothic', Roboto, Noto, - 'Noto Sans CJK KR', sans-serif !important; -} - -.ar-SA { - font-family: 'Geeza Pro', 'Arabic Typesetting', Roboto, Noto, 'Noto Naskh Arabic', - 'Times New Roman', serif !important; -} - -.he-IL { - font-family: 'New Peninim MT', 'Arial Hebrew', Gisha, 'Times New Roman', Roboto, Noto, - 'Noto Sans Hebrew', sans-serif !important; -} diff --git a/src/tracker/index.js b/src/tracker/index.js index 43a515013..dbd47b7c6 100644 --- a/src/tracker/index.js +++ b/src/tracker/index.js @@ -3,12 +3,13 @@ screen: { width, height }, navigator: { language }, location, - localStorage, document, history, + top, } = window; - const { hostname, pathname, search } = location; + const { hostname, href, origin } = location; const { currentScript, referrer } = document; + const localStorage = href.startsWith('data:') ? undefined : window.localStorage; if (!currentScript) return; @@ -21,6 +22,7 @@ const tag = attr(_data + 'tag'); const autoTrack = attr(_data + 'auto-track') !== _false; const excludeSearch = attr(_data + 'exclude-search') === _true; + const excludeHash = attr(_data + 'exclude-hash') === _true; const domain = attr(_data + 'domains') || ''; const domains = domain.split(',').map(n => n.trim()); const host = @@ -33,36 +35,14 @@ /* Helper functions */ - const encode = str => { - if (!str) { - return undefined; - } - - try { - const result = decodeURI(str); - - if (result !== str) { - return result; - } - } catch { - return str; - } - - return encodeURI(str); - }; - - const parseURL = url => { - return excludeSearch ? url.split('?')[0] : url; - }; - const getPayload = () => ({ website, - hostname, screen, language, - title: encode(title), - url: encode(currentUrl), - referrer: encode(currentRef), + title, + hostname, + url: currentUrl, + referrer: currentRef, tag: tag ? tag : undefined, }); @@ -72,7 +52,17 @@ if (!url) return; currentRef = currentUrl; - currentUrl = parseURL(url.toString()); + currentUrl = new URL(url, location.href); + + if (excludeSearch) { + currentUrl.search = ''; + } + + if (excludeHash) { + currentUrl.hash = ''; + } + + currentUrl = currentUrl.toString(); if (currentUrl !== currentRef) { setTimeout(track, delayDuration); @@ -169,7 +159,9 @@ e.preventDefault(); } return trackElement(parentElement).then(() => { - if (!external) location.href = href; + if (!external) { + (target === '_top' ? top.location : location).href = href; + } }); } } else if (parentElement.tagName === 'BUTTON') { @@ -187,31 +179,51 @@ /* Tracking functions */ const trackingDisabled = () => + disabled || + !website || (localStorage && localStorage.getItem('umami.disabled')) || (domain && !domains.includes(hostname)); const send = async (payload, type = 'event') => { if (trackingDisabled()) return; + const headers = { 'Content-Type': 'application/json', }; + if (typeof cache !== 'undefined') { headers['x-umami-cache'] = cache; } + try { const res = await fetch(endpoint, { method: 'POST', body: JSON.stringify({ type, payload }), headers, + credentials: 'omit', }); - const text = await res.text(); - return (cache = text); - } catch { + const data = await res.json(); + + if (data) { + disabled = !!data.disabled; + cache = data.cache; + } + } catch (e) { /* empty */ } }; + const init = () => { + if (!initialized) { + track(); + handlePathChanges(); + handleTitleChanges(); + handleClicks(); + initialized = true; + } + }; + const track = (obj, data) => { if (typeof obj === 'string') { return send({ @@ -238,26 +250,18 @@ }; } - let currentUrl = `${pathname}${search}`; - let currentRef = referrer !== hostname ? referrer : ''; + let currentUrl = href; + let currentRef = referrer.startsWith(origin) ? '' : referrer; let title = document.title; let cache; let initialized; + let disabled = false; if (autoTrack && !trackingDisabled()) { - handlePathChanges(); - handleTitleChanges(); - handleClicks(); - - const init = () => { - if (document.readyState === 'complete' && !initialized) { - track(); - initialized = true; - } - }; - - document.addEventListener('readystatechange', init, true); - - init(); + if (document.readyState === 'complete') { + init(); + } else { + document.addEventListener('readystatechange', init, true); + } } })(window);