watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-05 13:17:19 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Refactor filter handling for queries.

Browse source
This commit is contained in:
Mike Cao 2025-07-02 01:44:12 -07:00
parent 5b300f1ff5
commit ee6c68d27c
107 changed files with 731 additions and 835 deletions
Download patch file Download diff file Expand all files Collapse all files

55
src/lib/clickhouse.ts
View file

@ -2,11 +2,9 @@ import { ClickHouseClient, createClient } from '@clickhouse/client';
import { formatInTimeZone } from 'date-fns-tz';
import debug from 'debug';
import { CLICKHOUSE } from '@/lib/db';
import { getWebsite } from '@/queries';
import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
import { maxDate } from './date';
import { filtersToArray } from './params';
import { PageParams, QueryFilters, QueryOptions } from './types';
import { QueryFilters, QueryOptions } from './types';
export const CLICKHOUSE_DATE_FORMATS = {
utc: '%Y-%m-%dT%H:%i:%SZ',
@ -89,7 +87,7 @@ function mapFilter(column: string, operator: string, name: string, type: string
}
}
function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
function getFilterQuery(filters: Record<string, any>, options: QueryOptions = {}) {
const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
if (column) {
arr.push(`and ${mapFilter(column, operator, name)}`);
@ -105,7 +103,7 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
return query.join('\n');
}
function getDateQuery(filters: QueryFilters = {}) {
function getDateQuery(filters: Record<string, any>) {
const { startDate, endDate, timezone } = filters;
if (startDate) {
@ -125,36 +123,33 @@ function getDateQuery(filters: QueryFilters = {}) {
return '';
}
function getFilterParams(filters: QueryFilters = {}) {
return filtersToArray(filters).reduce((obj, { name, value }) => {
if (name && value !== undefined) {
obj[name] = value;
}
return obj;
}, {});
}
async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {
const website = await getWebsite(websiteId);
function getQueryParams(filters: Record<string, any>) {
return {
filterQuery: getFilterQuery(filters, options),
dateQuery: getDateQuery(filters),
filterParams: {
...getFilterParams(filters),
websiteId,
startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
},
...filters,
...filtersToArray(filters).reduce((obj, { name, value }) => {
if (name && value !== undefined) {
obj[name] = value;
}
return obj;
}, {}),
};
}
async function pagedQuery(
async function parseFilters(filters: Record<string, any>, options?: QueryOptions) {
return {
filterQuery: getFilterQuery(filters, options),
dateQuery: getDateQuery(filters),
queryParams: getQueryParams(filters),
};
}
async function pagedRawQuery(
query: string,
queryParams: { [key: string]: any },
pageParams: PageParams = {},
queryParams: Record<string, any>,
filters: QueryFilters,
) {
const { page = 1, pageSize, orderBy, sortDescending = false } = pageParams;
const { page = 1, pageSize, orderBy, sortDescending = false } = filters;
const size = +pageSize || DEFAULT_PAGE_SIZE;
const offset = +size * (+page - 1);
const direction = sortDescending ? 'desc' : 'asc';
@ -236,7 +231,7 @@ export default {
getFilterQuery,
getUTCString,
parseFilters,
pagedQuery,
pagedRawQuery,
findUnique,
findFirst,
rawQuery,

2
src/lib/constants.ts
View file

@ -19,7 +19,7 @@ export const DEFAULT_ANIMATION_DURATION = 300;
export const DEFAULT_DATE_RANGE_VALUE = '24hour';
export const DEFAULT_WEBSITE_LIMIT = 10;
export const DEFAULT_RESET_DATE = '2000-01-01';
export const DEFAULT_PAGE_SIZE = 10;
export const DEFAULT_PAGE_SIZE = 20;
export const DEFAULT_DATE_COMPARE = 'prev';
export const REALTIME_RANGE = 30;

2
src/lib/data.ts
View file

@ -2,7 +2,7 @@ import { DATA_TYPE, DATETIME_REGEX } from './constants';
import { DynamicDataType } from './types';
export function flattenJSON(
eventData: { [key: string]: any },
eventData: Record<string, any>,
keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],
parentKey = '',
): { key: string; value: any; dataType: DynamicDataType }[] {

8
src/lib/date.ts
View file

@ -292,9 +292,13 @@ export function getCompareDate(compare: string, startDate: Date, endDate: Date)
return { startDate: subYears(startDate, 1), endDate: subYears(endDate, 1) };
}
const diff = differenceInMinutes(endDate, startDate);
if (compare === 'prev') {
const diff = differenceInMinutes(endDate, startDate);
return { startDate: subMinutes(startDate, diff), endDate: subMinutes(endDate, diff) };
return { startDate: subMinutes(startDate, diff), endDate: subMinutes(endDate, diff) };
}
return {};
}
export function getDayOfWeekAsDate(dayOfWeek: number) {

2
src/lib/kafka.ts
View file

@ -64,7 +64,7 @@ async function getProducer(): Promise<Producer> {
async function sendMessage(
topic: string,
message: { [key: string]: string | number } | { [key: string]: string | number }[],
message: Record<string, string | number> | Record<string, string | number>[],
): Promise<RecordMetadata[]> {
try {
await connect();

2
src/lib/params.ts
View file

@ -18,7 +18,7 @@ export function isSearchOperator(operator: any) {
return [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator);
}
export function filtersToArray(filters: QueryFilters = {}, options: QueryOptions = {}) {
export function filtersToArray(filters: QueryFilters, options: QueryOptions = {}) {
return Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];

113
src/lib/prisma.ts
View file

@ -2,12 +2,9 @@ import debug from 'debug';
import { PrismaClient } from '@/generated/prisma/client';
import { PrismaPg } from '@prisma/adapter-pg';
import { readReplicas } from '@prisma/extension-read-replicas';
import { formatInTimeZone } from 'date-fns-tz';
import { MYSQL, POSTGRESQL, getDatabaseType } from '@/lib/db';
import { SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants';
import { fetchWebsite } from './load';
import { maxDate } from './date';
import { QueryFilters, QueryOptions, PageParams } from './types';
import { QueryOptions, QueryFilters } from './types';
import { filtersToArray } from './params';
const log = debug('umami:prisma');
@ -22,14 +19,6 @@ const PRISMA_LOG_OPTIONS = {
],
};
const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%dT%H:%i:00',
hour: '%Y-%m-%d %H:00:00',
day: '%Y-%m-%d 00:00:00',
month: '%Y-%m-01 00:00:00',
year: '%Y-01-01 00:00:00',
};
const POSTGRESQL_DATE_FORMATS = {
minute: 'YYYY-MM-DD HH24:MI:00',
hour: 'YYYY-MM-DD HH24:00:00',
@ -75,59 +64,23 @@ function getCastColumnQuery(field: string, type: string): string {
}
function getDateSQL(field: string, unit: string, timezone?: string): string {
const db = getDatabaseType();
if (db === POSTGRESQL) {
if (timezone) {
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
if (timezone) {
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
if (db === MYSQL) {
if (timezone) {
const tz = formatInTimeZone(new Date(), timezone, 'xxx');
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
}
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
}
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
}
function getDateWeeklySQL(field: string, timezone?: string) {
const db = getDatabaseType();
if (db === POSTGRESQL) {
return `concat(extract(dow from (${field} at time zone '${timezone}')), ':', to_char((${field} at time zone '${timezone}'), 'HH24'))`;
}
if (db === MYSQL) {
const tz = formatInTimeZone(new Date(), timezone, 'xxx');
return `date_format(convert_tz(${field},'+00:00','${tz}'), '%w:%H')`;
}
return `concat(extract(dow from (${field} at time zone '${timezone}')), ':', to_char((${field} at time zone '${timezone}'), 'HH24'))`;
}
export function getTimestampSQL(field: string) {
const db = getDatabaseType();
if (db === POSTGRESQL) {
return `floor(extract(epoch from ${field}))`;
}
if (db === MYSQL) {
return `UNIX_TIMESTAMP(${field})`;
}
return `floor(extract(epoch from ${field}))`;
}
function getTimestampDiffSQL(field1: string, field2: string): string {
const db = getDatabaseType();
if (db === POSTGRESQL) {
return `floor(extract(epoch from (${field2} - ${field1})))`;
}
if (db === MYSQL) {
return `timestampdiff(second, ${field1}, ${field2})`;
}
return `floor(extract(epoch from (${field2} - ${field1})))`;
}
function getSearchSQL(column: string, param: string = 'search'): string {
@ -156,7 +109,7 @@ function mapFilter(column: string, operator: string, name: string, type: string
}
}
function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {
function getFilterQuery(filters: Record<string, any>, options: QueryOptions = {}): string {
const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
if (column) {
arr.push(`and ${mapFilter(column, operator, name)}`);
@ -174,7 +127,7 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):
return query.join('\n');
}
function getDateQuery(filters: QueryFilters = {}) {
function getDateQuery(filters: Record<string, any>) {
const { startDate, endDate } = filters;
if (startDate) {
@ -188,38 +141,32 @@ function getDateQuery(filters: QueryFilters = {}) {
return '';
}
function getFilterParams(filters: QueryFilters = {}) {
return filtersToArray(filters).reduce((obj, { name, operator, value }) => {
obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator)
? `%${value}%`
: value;
function getQueryParams(filters: Record<string, any>) {
return {
...filters,
...filtersToArray(filters).reduce((obj, { name, operator, value }) => {
obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator)
? `%${value}%`
: value;
return obj;
}, {});
return obj;
}, {}),
};
}
async function parseFilters(
websiteId: string,
filters: QueryFilters = {},
options: QueryOptions = {},
) {
const website = await fetchWebsite(websiteId);
async function parseFilters(filters: Record<string, any>, options?: QueryOptions) {
const joinSession = Object.keys(filters).find(key =>
['referrer', ...SESSION_COLUMNS].includes(key),
);
return {
joinSession:
joinSessionQuery:
options?.joinSession || joinSession
? `inner join session on website_event.session_id = session.session_id`
: '',
filterQuery: getFilterQuery(filters, options),
dateQuery: getDateQuery(filters),
filterParams: {
...getFilterParams(filters),
websiteId,
startDate: maxDate(filters.startDate, website?.resetAt),
},
filterQuery: getFilterQuery(filters, options),
queryParams: getQueryParams(filters),
};
}
@ -251,8 +198,8 @@ async function rawQuery(sql: string, data: object): Promise<any> {
: client.$queryRawUnsafe(query, ...params);
}
async function pagedQuery<T>(model: string, criteria: T, pageParams: PageParams) {
const { page = 1, pageSize, orderBy, sortDescending = false } = pageParams || {};
async function pagedQuery<T>(model: string, criteria: T, filters?: QueryFilters) {
const { page = 1, pageSize, orderBy, sortDescending = false } = filters || {};
const size = +pageSize || DEFAULT_PAGE_SIZE;
const data = await client[model].findMany({
@ -276,10 +223,10 @@ async function pagedQuery<T>(model: string, criteria: T, pageParams: PageParams)
async function pagedRawQuery(
query: string,
queryParams: { [key: string]: any },
pageParams: PageParams = {},
filters: QueryFilters,
queryParams: Record<string, any>,
) {
const { page = 1, pageSize, orderBy, sortDescending = false } = pageParams;
const { page = 1, pageSize, orderBy, sortDescending = false } = filters;
const size = +pageSize || DEFAULT_PAGE_SIZE;
const offset = +size * (+page - 1);
const direction = sortDescending ? 'desc' : 'asc';
@ -310,11 +257,11 @@ function getQueryMode(): { mode?: 'default' | 'insensitive' } {
return {};
}
function getSearchParameters(query: string, filters: { [key: string]: any }[]) {
function getSearchParameters(query: string, filters: Record<string, any>[]) {
if (!query) return;
const mode = getQueryMode();
const parseFilter = (filter: { [key: string]: any }) => {
const parseFilter = (filter: Record<string, any>) => {
const [[key, value]] = Object.entries(filter);
return {

39
src/lib/request.ts
View file

@ -1,8 +1,10 @@
import { z } from 'zod/v4';
import { FILTER_COLUMNS } from '@/lib/constants';
import { FILTER_COLUMNS, DEFAULT_PAGE_SIZE } from '@/lib/constants';
import { badRequest, unauthorized } from '@/lib/response';
import { getAllowedUnits, getCompareDate, getMinimumUnit } from '@/lib/date';
import { getAllowedUnits, getCompareDate, getMinimumUnit, maxDate } from '@/lib/date';
import { checkAuth } from '@/lib/auth';
import { fetchWebsite } from '@/lib/load';
import { QueryFilters } from '@/lib/types';
export async function parseRequest(
request: Request,
@ -47,8 +49,8 @@ export async function getJsonBody(request: Request) {
}
}
export async function getRequestDateRange(query: Record<string, string>) {
const { startAt, endAt, unit, compare } = query;
export function getRequestDateRange(query: Record<string, string>) {
const { startAt, endAt, unit, compare, timezone } = query;
const startDate = new Date(+startAt);
const endDate = new Date(+endAt);
@ -62,8 +64,10 @@ export async function getRequestDateRange(query: Record<string, string>) {
return {
startDate,
endDate,
compare,
compareStartDate,
compareEndDate,
timezone,
unit: getAllowedUnits(startDate, endDate).includes(unit)
? unit
: getMinimumUnit(startDate, endDate),
@ -81,3 +85,30 @@ export function getRequestFilters(query: Record<string, any>) {
return obj;
}, {});
}
export async function getQueryFilters(params: Record<string, any>): Promise<QueryFilters> {
const dateRange = getRequestDateRange(params);
const filters = getRequestFilters(params);
const data = {
...dateRange,
...filters,
page: params?.page,
pageSize: params?.page ? params?.pageSize || DEFAULT_PAGE_SIZE : undefined,
orderBy: params?.orderBy,
sortDescending: params?.sortDescending,
search: params?.search,
websiteId: undefined,
};
const { websiteId } = params;
if (websiteId) {
const website = await fetchWebsite(websiteId);
data.websiteId = websiteId;
data.startDate = maxDate(data.startDate, new Date(website?.resetAt));
}
return data;
}

36
src/lib/schema.ts
View file

@ -2,6 +2,22 @@ import { z } from 'zod';
import { isValidTimezone } from '@/lib/date';
import { UNIT_TYPES } from './constants';
export const timezoneParam = z.string().refine(value => isValidTimezone(value), {
message: 'Invalid timezone',
});
export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value), {
message: 'Invalid unit',
});
export const dateRangeParams = {
startAt: z.coerce.number(),
endAt: z.coerce.number(),
timezone: timezoneParam.optional(),
unit: unitParam.optional(),
compare: z.string().optional(),
};
export const filterParams = {
path: z.string().optional(),
referrer: z.string().optional(),
@ -22,17 +38,13 @@ export const filterParams = {
export const pagingParams = {
page: z.coerce.number().int().positive().optional(),
pageSize: z.coerce.number().int().positive().optional(),
orderBy: z.string().optional(),
search: z.string().optional(),
};
export const timezoneParam = z.string().refine(value => isValidTimezone(value), {
message: 'Invalid timezone',
});
export const unitParam = z.string().refine(value => UNIT_TYPES.includes(value), {
message: 'Invalid unit',
});
export const sortingParams = {
orderBy: z.string().optional(),
sortDescending: z.string().optional(),
};
export const userRoleParam = z.enum(['admin', 'user', 'view-only']);
@ -86,11 +98,11 @@ export const reportParms = {
dateRange: z.object({
startDate: z.coerce.date(),
endDate: z.coerce.date(),
num: z.coerce.number().optional(),
offset: z.coerce.number().optional(),
timezone: timezoneParam.optional(),
unit: unitParam.optional(),
value: z.string().optional(),
compare: z.string().optional(),
compareStartDate: z.coerce.date().optional(),
compareEndDate: z.coerce.date().optional(),
}),
};
@ -191,7 +203,7 @@ export const reportSchema = z.intersection(reportBaseSchema, reportTypeSchema);
export const reportResultSchema = z.intersection(
z.object({
...reportParms,
...filterParams,
filters: z.object({ ...filterParams }),
}),
reportTypeSchema,
);

119
src/lib/types.ts
View file

@ -1,45 +1,16 @@
import { Dispatch, SetStateAction } from 'react';
import { UseQueryOptions } from '@tanstack/react-query';
import { DATA_TYPE, PERMISSIONS, ROLES } from './constants';
import { TIME_UNIT } from './date';
export type ObjectValues<T> = T[keyof T];
export type ReactQueryOptions<T> = Omit<UseQueryOptions<T, Error, T>, 'queryKey' | 'queryFn'>;
export type ReactQueryOptions<T = any> = Omit<UseQueryOptions<T, Error, T>, 'queryKey' | 'queryFn'>;
export type TimeUnit = ObjectValues<typeof TIME_UNIT>;
export type Permission = ObjectValues<typeof PERMISSIONS>;
export type Role = ObjectValues<typeof ROLES>;
export type DynamicDataType = ObjectValues<typeof DATA_TYPE>;
export interface PageParams {
search?: string;
page?: string | number;
pageSize?: string;
orderBy?: string;
sortDescending?: boolean;
}
export interface PageResult<T> {
data: T;
count: number;
page: number;
pageSize: number;
orderBy?: string;
sortDescending?: boolean;
}
export interface PagedQueryResult<T = any> {
result: PageResult<T>;
query: any;
params: PageParams;
setParams: Dispatch<SetStateAction<T | PageParams>>;
}
export interface DynamicData {
[key: string]: number | string | number[] | string[];
}
export interface Auth {
user?: {
id: string;
@ -54,20 +25,35 @@ export interface Auth {
}
export interface DateRange {
value: string;
startDate: Date;
endDate: Date;
value?: string;
unit?: TimeUnit;
num?: number;
offset?: number;
}
export interface DynamicData {
[key: string]: number | string | number[] | string[];
}
export interface QueryOptions {
joinSession?: boolean;
columns?: Record<string, string>;
limit?: number;
}
export interface QueryFilters {
websiteId?: string;
// Date range
startDate?: Date;
endDate?: Date;
timezone?: string;
compareStartDate?: Date;
compareEndDate?: Date;
compare?: string;
unit?: string;
eventType?: number;
timezone?: string;
// Filters
path?: string;
referrer?: string;
title?: string;
@ -83,54 +69,29 @@ export interface QueryFilters {
event?: string;
search?: string;
tag?: string;
eventType?: number;
// Paging
page?: number;
pageSize?: number;
// Sorting
orderBy?: string;
sortDescending?: boolean;
}
export interface QueryOptions {
joinSession?: boolean;
columns?: { [key: string]: string };
limit?: number;
export interface PageParams {
page: number;
pageSize: number;
orderBy?: string;
sortDescending?: boolean;
search?: string;
}
export interface RealtimeData {
countries: { [key: string]: number };
events: any[];
pageviews: any[];
referrers: { [key: string]: number };
timestamp: number;
series: {
views: any[];
visitors: any[];
};
totals: {
views: number;
visitors: number;
events: number;
countries: number;
};
urls: { [key: string]: number };
visitors: any[];
}
export interface SessionData {
id: string;
websiteId: string;
visitId: string;
hostname: string;
browser: string;
os: string;
device: string;
screen: string;
language: string;
country: string;
region: string;
city: string;
ip?: string;
userAgent?: string;
}
export interface InputItem {
id: string;
label: string;
icon: any;
seperator?: boolean;
export interface PageResult<T> {
data: T;
count: number;
page: number;
pageSize: number;
orderBy?: string;
sortDescending?: boolean;
search?: string;
}