mirror of
https://github.com/umami-software/umami.git
synced 2026-02-04 04:37:11 +01:00
Merge branch 'dev'
# Conflicts: # components/common/Calendar.js # components/common/EventDataButton.js # components/forms/EventDataForm.js # components/layout/Header.js # db/mysql/migrations/02_add_event_data/migration.sql # lang/be-BY.json # lang/de-CH.json # lang/es-MX.json # lang/fa-IR.json # lang/km-KH.json # lang/lt-LT.json # lang/th-TH.json # lib/auth.js # lib/detect.ts # lib/lang.js # lib/middleware.js # lib/prisma.js # package.json # pages/_app.js # pages/api/accounts/[id]/password.js # pages/api/collect.js # pages/api/realtime/init.js # pages/api/websites/[id]/index.js # pages/api/websites/[id]/reset.ts # pages/api/websites/index.js # public/intl/messages/fa-IR.json # public/intl/messages/lt-LT.json # public/intl/messages/pt-BR.json # public/intl/messages/th-TH.json # queries/analytics/event/getEventData.js # queries/analytics/event/getEventMetrics.js # queries/analytics/pageview/getPageviewMetrics.js # queries/analytics/pageview/getPageviewParams.js # queries/analytics/pageview/getPageviewStats.js # queries/analytics/session/getSessionMetrics.js # queries/analytics/stats/getActiveVisitors.js # queries/analytics/stats/getWebsiteStats.js # sql/schema.mysql.sql # styles/index.css # yarn.lock
This commit is contained in:
Mike Cao
commit
dfae0c150d
592 changed files with 44367 additions and 31628 deletions
18
lib/array.js
18
lib/array.js
|
|
@ -1,18 +0,0 @@
|
|||
export function chunk(arr, size) {
|
||||
const chunks = [];
|
||||
|
||||
let index = 0;
|
||||
while (index < arr.length) {
|
||||
chunks.push(arr.slice(index, size + index));
|
||||
index += size;
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
export function sortArrayByMap(arr, map = [], key) {
|
||||
if (!arr) return [];
|
||||
if (map.length === 0) return arr;
|
||||
|
||||
return map.map(id => arr.find(item => item[key] === id));
|
||||
}
|
||||
64
lib/auth.js
64
lib/auth.js
|
|
@ -1,64 +0,0 @@
|
|||
import { parseSecureToken, parseToken } from 'next-basics';
|
||||
import { getAccount, getWebsite } from 'queries';
|
||||
import { SHARE_TOKEN_HEADER, TYPE_ACCOUNT, TYPE_WEBSITE } from 'lib/constants';
|
||||
import { secret } from 'lib/crypto';
|
||||
|
||||
export function getAuthToken(req) {
|
||||
try {
|
||||
const token = req.headers.authorization;
|
||||
|
||||
return parseSecureToken(token.split(' ')[1], secret());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function getShareToken(req) {
|
||||
try {
|
||||
return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidToken(token, validation) {
|
||||
try {
|
||||
if (typeof validation === 'object') {
|
||||
return !Object.keys(validation).find(key => token[key] !== validation[key]);
|
||||
} else if (typeof validation === 'function') {
|
||||
return validation(token);
|
||||
}
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function allowQuery(req, type, allowShareToken = true) {
|
||||
const { id } = req.query;
|
||||
|
||||
const { userId, isAdmin, shareToken } = req.auth ?? {};
|
||||
|
||||
if (isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (allowShareToken && shareToken) {
|
||||
return isValidToken(shareToken, { id });
|
||||
}
|
||||
|
||||
if (userId) {
|
||||
if (type === TYPE_WEBSITE) {
|
||||
const website = await getWebsite({ websiteUuid: id });
|
||||
|
||||
return website && website.userId === userId;
|
||||
} else if (type === TYPE_ACCOUNT) {
|
||||
const account = await getAccount({ accountUuid: id });
|
||||
|
||||
return account && account.accountUuid === id;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
252
lib/auth.ts
Normal file
252
lib/auth.ts
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
import debug from 'debug';
|
||||
import redis from '@umami/redis-client';
|
||||
import cache from 'lib/cache';
|
||||
import { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER } from 'lib/constants';
|
||||
import { secret } from 'lib/crypto';
|
||||
import {
|
||||
createSecureToken,
|
||||
ensureArray,
|
||||
getRandomChars,
|
||||
parseSecureToken,
|
||||
parseToken,
|
||||
} from 'next-basics';
|
||||
import { getTeamUser, getTeamUserById } from 'queries';
|
||||
import { getTeamWebsite, getTeamWebsiteByTeamMemberId } from 'queries/admin/teamWebsite';
|
||||
import { validate } from 'uuid';
|
||||
import { Auth } from './types';
|
||||
import { loadWebsite } from './query';
|
||||
|
||||
const log = debug('umami:auth');
|
||||
|
||||
export async function setAuthKey(user, expire = 0) {
|
||||
const authKey = `auth:${getRandomChars(32)}`;
|
||||
|
||||
await redis.set(authKey, user);
|
||||
|
||||
if (expire) {
|
||||
await redis.expire(authKey, expire);
|
||||
}
|
||||
|
||||
return createSecureToken({ authKey }, secret());
|
||||
}
|
||||
|
||||
export function getAuthToken(req) {
|
||||
try {
|
||||
return req.headers.authorization.split(' ')[1];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseAuthToken(req) {
|
||||
try {
|
||||
return parseSecureToken(getAuthToken(req), secret());
|
||||
} catch (e) {
|
||||
log(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function parseShareToken(req) {
|
||||
try {
|
||||
return parseToken(req.headers[SHARE_TOKEN_HEADER], secret());
|
||||
} catch (e) {
|
||||
log(e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function isValidToken(token, validation) {
|
||||
try {
|
||||
if (typeof validation === 'object') {
|
||||
return !Object.keys(validation).find(key => token[key] !== validation[key]);
|
||||
} else if (typeof validation === 'function') {
|
||||
return validation(token);
|
||||
}
|
||||
} catch (e) {
|
||||
log(e);
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canViewWebsite({ user, shareToken }: Auth, websiteId: string) {
|
||||
if (user?.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (shareToken?.websiteId === websiteId) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const teamWebsite = await getTeamWebsiteByTeamMemberId(websiteId, user.id);
|
||||
|
||||
if (teamWebsite) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const website = await loadWebsite(websiteId);
|
||||
|
||||
if (website.userId) {
|
||||
return user.id === website.userId;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canCreateWebsite({ user }: Auth) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return hasPermission(user.role, PERMISSIONS.websiteCreate);
|
||||
}
|
||||
|
||||
export async function canUpdateWebsite({ user }: Auth, websiteId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!validate(websiteId)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const website = await loadWebsite(websiteId);
|
||||
|
||||
if (website.userId) {
|
||||
return user.id === website.userId;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canDeleteWebsite({ user }: Auth, websiteId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const website = await loadWebsite(websiteId);
|
||||
|
||||
if (website.userId) {
|
||||
return user.id === website.userId;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// To-do: Implement when payments are setup.
|
||||
export async function canCreateTeam({ user }: Auth) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return !!user;
|
||||
}
|
||||
|
||||
// To-do: Implement when payments are setup.
|
||||
export async function canViewTeam({ user }: Auth, teamId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return getTeamUser(teamId, user.id);
|
||||
}
|
||||
|
||||
export async function canUpdateTeam({ user }: Auth, teamId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (validate(teamId)) {
|
||||
const teamUser = await getTeamUser(teamId, user.id);
|
||||
|
||||
return hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canDeleteTeam({ user }: Auth, teamId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (validate(teamId)) {
|
||||
const teamUser = await getTeamUser(teamId, user.id);
|
||||
|
||||
return hasPermission(teamUser.role, PERMISSIONS.teamDelete);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canDeleteTeamUser({ user }: Auth, teamId: string, removeUserId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (validate(teamId) && validate(removeUserId)) {
|
||||
if (removeUserId === user.id) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const teamUser = await getTeamUser(teamId, user.id);
|
||||
|
||||
return hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canDeleteTeamWebsite({ user }: Auth, teamId: string, websiteId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (validate(teamId) && validate(websiteId)) {
|
||||
const teamWebsite = await getTeamWebsite(teamId, websiteId);
|
||||
|
||||
if (teamWebsite.website.userId === user.id) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const teamUser = await getTeamUser(teamWebsite.teamId, user.id);
|
||||
|
||||
return hasPermission(teamUser.role, PERMISSIONS.teamUpdate);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function canCreateUser({ user }: Auth) {
|
||||
return user.isAdmin;
|
||||
}
|
||||
|
||||
export async function canViewUser({ user }: Auth, viewedUserId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return user.id === viewedUserId;
|
||||
}
|
||||
|
||||
export async function canViewUsers({ user }: Auth) {
|
||||
return user.isAdmin;
|
||||
}
|
||||
|
||||
export async function canUpdateUser({ user }: Auth, viewedUserId: string) {
|
||||
if (user.isAdmin) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return user.id === viewedUserId;
|
||||
}
|
||||
|
||||
export async function canDeleteUser({ user }: Auth) {
|
||||
return user.isAdmin;
|
||||
}
|
||||
|
||||
export async function hasPermission(role: string, permission: string | string[]) {
|
||||
return ensureArray(permission).some(e => ROLE_PERMISSIONS[role]?.includes(e));
|
||||
}
|
||||
91
lib/cache.ts
Normal file
91
lib/cache.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import { User, Website } from '@prisma/client';
|
||||
import redis from '@umami/redis-client';
|
||||
import { getSession, getUser, getWebsite } from '../queries';
|
||||
|
||||
const DELETED = 'DELETED';
|
||||
|
||||
async function fetchObject(key, query) {
|
||||
const obj = await redis.get(key);
|
||||
|
||||
if (obj === DELETED) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!obj) {
|
||||
return query().then(async data => {
|
||||
if (data) {
|
||||
await redis.set(key, data);
|
||||
}
|
||||
|
||||
return data;
|
||||
});
|
||||
}
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
async function storeObject(key, data) {
|
||||
return redis.set(key, data);
|
||||
}
|
||||
|
||||
async function deleteObject(key, soft = false) {
|
||||
return soft ? redis.set(key, DELETED) : redis.del(key);
|
||||
}
|
||||
|
||||
async function fetchWebsite(id): Promise<Website> {
|
||||
return fetchObject(`website:${id}`, () => getWebsite({ id }));
|
||||
}
|
||||
|
||||
async function storeWebsite(data) {
|
||||
const { id } = data;
|
||||
const key = `website:${id}`;
|
||||
|
||||
return storeObject(key, data);
|
||||
}
|
||||
|
||||
async function deleteWebsite(id) {
|
||||
return deleteObject(`website:${id}`);
|
||||
}
|
||||
|
||||
async function fetchUser(id): Promise<User> {
|
||||
return fetchObject(`user:${id}`, () => getUser({ id }, { includePassword: true }));
|
||||
}
|
||||
|
||||
async function storeUser(data) {
|
||||
const { id } = data;
|
||||
const key = `user:${id}`;
|
||||
|
||||
return storeObject(key, data);
|
||||
}
|
||||
|
||||
async function deleteUser(id) {
|
||||
return deleteObject(`user:${id}`);
|
||||
}
|
||||
|
||||
async function fetchSession(id) {
|
||||
return fetchObject(`session:${id}`, () => getSession({ id }));
|
||||
}
|
||||
|
||||
async function storeSession(data) {
|
||||
const { id } = data;
|
||||
const key = `session:${id}`;
|
||||
|
||||
return storeObject(key, data);
|
||||
}
|
||||
|
||||
async function deleteSession(id) {
|
||||
return deleteObject(`session:${id}`);
|
||||
}
|
||||
|
||||
export default {
|
||||
fetchWebsite,
|
||||
storeWebsite,
|
||||
deleteWebsite,
|
||||
fetchUser,
|
||||
storeUser,
|
||||
deleteUser,
|
||||
fetchSession,
|
||||
storeSession,
|
||||
deleteSession,
|
||||
enabled: redis.enabled,
|
||||
};
|
||||
|
|
@ -1,233 +0,0 @@
|
|||
import { ClickHouse } from 'clickhouse';
|
||||
import dateFormat from 'dateformat';
|
||||
import debug from 'debug';
|
||||
import { FILTER_IGNORED } from 'lib/constants';
|
||||
import { CLICKHOUSE } from 'lib/db';
|
||||
|
||||
export const CLICKHOUSE_DATE_FORMATS = {
|
||||
minute: '%Y-%m-%d %H:%M:00',
|
||||
hour: '%Y-%m-%d %H:00:00',
|
||||
day: '%Y-%m-%d',
|
||||
month: '%Y-%m-01',
|
||||
year: '%Y-01-01',
|
||||
};
|
||||
|
||||
const log = debug('umami:clickhouse');
|
||||
|
||||
let clickhouse;
|
||||
const enabled = Boolean(process.env.CLICKHOUSE_URL);
|
||||
|
||||
function getClient() {
|
||||
const {
|
||||
hostname,
|
||||
port,
|
||||
pathname,
|
||||
username = 'default',
|
||||
password,
|
||||
} = new URL(process.env.CLICKHOUSE_URL);
|
||||
|
||||
const client = new ClickHouse({
|
||||
url: hostname,
|
||||
port: Number(port),
|
||||
format: 'json',
|
||||
config: {
|
||||
database: pathname.replace('/', ''),
|
||||
},
|
||||
basicAuth: password ? { username, password } : null,
|
||||
});
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global[CLICKHOUSE] = client;
|
||||
}
|
||||
|
||||
log('Clickhouse initialized');
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
function getDateStringQuery(data, unit) {
|
||||
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
function getDateQuery(field, unit, timezone) {
|
||||
if (timezone) {
|
||||
return `date_trunc('${unit}', ${field}, '${timezone}')`;
|
||||
}
|
||||
return `date_trunc('${unit}', ${field})`;
|
||||
}
|
||||
|
||||
function getDateFormat(date) {
|
||||
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
|
||||
}
|
||||
|
||||
function getCommaSeparatedStringFormat(data) {
|
||||
return data.map(a => `'${a}'`).join(',') || '';
|
||||
}
|
||||
|
||||
function getBetweenDates(field, start_at, end_at) {
|
||||
return `${field} between ${getDateFormat(start_at)} and ${getDateFormat(end_at)}`;
|
||||
}
|
||||
|
||||
function getJsonField(column, property) {
|
||||
return `${column}.${property}`;
|
||||
}
|
||||
|
||||
function getEventDataColumnsQuery(column, columns) {
|
||||
const query = Object.keys(columns).reduce((arr, key) => {
|
||||
const filter = columns[key];
|
||||
|
||||
if (filter === undefined) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
arr.push(`${filter}(${getJsonField(column, key)}) as "${filter}(${key})"`);
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join(',\n');
|
||||
}
|
||||
|
||||
function getEventDataFilterQuery(column, filters) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
arr.push(
|
||||
`${getJsonField(column, key)} = ${typeof filter === 'string' ? `'${filter}'` : filter}`,
|
||||
);
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\nand ');
|
||||
}
|
||||
|
||||
function getFilterQuery(column, filters = {}, params = []) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined || filter === FILTER_IGNORED) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (key) {
|
||||
case 'url':
|
||||
case 'os':
|
||||
case 'browser':
|
||||
case 'device':
|
||||
case 'country':
|
||||
case 'event_name':
|
||||
arr.push(`and ${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
break;
|
||||
|
||||
case 'referrer':
|
||||
arr.push(`and referrer like $${params.length + 1}`);
|
||||
params.push(`%${decodeURIComponent(filter)}%`);
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
arr.push(`and referrer not like $${params.length + 1}`);
|
||||
arr.push(`and referrer not like '/%'`);
|
||||
params.push(`%://${filter}/%`);
|
||||
break;
|
||||
|
||||
case 'query':
|
||||
arr.push(`and url like '%?%'`);
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function parseFilters(column, filters = {}, params = []) {
|
||||
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
|
||||
filters;
|
||||
|
||||
const pageviewFilters = { domain, url, referrer, query };
|
||||
const sessionFilters = { os, browser, device, country };
|
||||
const eventFilters = { url: event_url, event_name };
|
||||
|
||||
return {
|
||||
pageviewFilters,
|
||||
sessionFilters,
|
||||
eventFilters,
|
||||
event: { event_name },
|
||||
pageviewQuery: getFilterQuery(column, pageviewFilters, params),
|
||||
sessionQuery: getFilterQuery(column, sessionFilters, params),
|
||||
eventQuery: getFilterQuery(column, eventFilters, params),
|
||||
};
|
||||
}
|
||||
|
||||
function formatQuery(str, params = []) {
|
||||
let formattedString = str;
|
||||
|
||||
params.forEach((param, i) => {
|
||||
let replace = param;
|
||||
|
||||
if (typeof param === 'string' || param instanceof String) {
|
||||
replace = `'${replace}'`;
|
||||
}
|
||||
|
||||
formattedString = formattedString.replace(`$${i + 1}`, replace);
|
||||
});
|
||||
|
||||
return formattedString;
|
||||
}
|
||||
|
||||
async function rawQuery(query, params = []) {
|
||||
let formattedQuery = formatQuery(query, params);
|
||||
|
||||
if (process.env.LOG_QUERY) {
|
||||
log(formattedQuery);
|
||||
}
|
||||
|
||||
await connect();
|
||||
|
||||
return clickhouse.query(formattedQuery).toPromise();
|
||||
}
|
||||
|
||||
async function findUnique(data) {
|
||||
if (data.length > 1) {
|
||||
throw `${data.length} records found when expecting 1.`;
|
||||
}
|
||||
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
async function findFirst(data) {
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
if (!clickhouse) {
|
||||
clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient());
|
||||
}
|
||||
|
||||
return clickhouse;
|
||||
}
|
||||
|
||||
export default {
|
||||
enabled,
|
||||
client: clickhouse,
|
||||
log,
|
||||
connect,
|
||||
getDateStringQuery,
|
||||
getDateQuery,
|
||||
getDateFormat,
|
||||
getCommaSeparatedStringFormat,
|
||||
getBetweenDates,
|
||||
getEventDataColumnsQuery,
|
||||
getEventDataFilterQuery,
|
||||
getFilterQuery,
|
||||
parseFilters,
|
||||
findUnique,
|
||||
findFirst,
|
||||
rawQuery,
|
||||
};
|
||||
176
lib/clickhouse.ts
Normal file
176
lib/clickhouse.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { ClickHouse } from 'clickhouse';
|
||||
import dateFormat from 'dateformat';
|
||||
import debug from 'debug';
|
||||
import { CLICKHOUSE } from 'lib/db';
|
||||
import { getEventDataType } from './eventData';
|
||||
import { WebsiteMetricFilter } from './types';
|
||||
import { FILTER_COLUMNS } from './constants';
|
||||
|
||||
export const CLICKHOUSE_DATE_FORMATS = {
|
||||
minute: '%Y-%m-%d %H:%M:00',
|
||||
hour: '%Y-%m-%d %H:00:00',
|
||||
day: '%Y-%m-%d',
|
||||
month: '%Y-%m-01',
|
||||
year: '%Y-01-01',
|
||||
};
|
||||
|
||||
const log = debug('umami:clickhouse');
|
||||
|
||||
let clickhouse: ClickHouse;
|
||||
const enabled = Boolean(process.env.CLICKHOUSE_URL);
|
||||
|
||||
function getClient() {
|
||||
const {
|
||||
hostname,
|
||||
port,
|
||||
pathname,
|
||||
username = 'default',
|
||||
password,
|
||||
} = new URL(process.env.CLICKHOUSE_URL);
|
||||
|
||||
const client = new ClickHouse({
|
||||
url: hostname,
|
||||
port: Number(port),
|
||||
format: 'json',
|
||||
config: {
|
||||
database: pathname.replace('/', ''),
|
||||
},
|
||||
basicAuth: password ? { username, password } : null,
|
||||
});
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global[CLICKHOUSE] = client;
|
||||
}
|
||||
|
||||
log('Clickhouse initialized');
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
function getDateStringQuery(data, unit) {
|
||||
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
function getDateQuery(field, unit, timezone?) {
|
||||
if (timezone) {
|
||||
return `date_trunc('${unit}', ${field}, '${timezone}')`;
|
||||
}
|
||||
return `date_trunc('${unit}', ${field})`;
|
||||
}
|
||||
|
||||
function getDateFormat(date) {
|
||||
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
|
||||
}
|
||||
|
||||
function getBetweenDates(field, startAt, endAt) {
|
||||
return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`;
|
||||
}
|
||||
|
||||
function getEventDataFilterQuery(
|
||||
filters: {
|
||||
eventKey?: string;
|
||||
eventValue?: string | number | boolean | Date;
|
||||
}[] = [],
|
||||
params: any,
|
||||
) {
|
||||
const query = filters.reduce((ac, cv, i) => {
|
||||
const type = getEventDataType(cv.eventValue);
|
||||
|
||||
let value = cv.eventValue;
|
||||
|
||||
ac.push(`and (event_key = {eventKey${i}:String}`);
|
||||
|
||||
switch (type) {
|
||||
case 'number':
|
||||
ac.push(`and event_numeric_value = {eventValue${i}:UInt64})`);
|
||||
break;
|
||||
case 'string':
|
||||
ac.push(`and event_string_value = {eventValue${i}:String})`);
|
||||
break;
|
||||
case 'boolean':
|
||||
ac.push(`and event_string_value = {eventValue${i}:String})`);
|
||||
value = cv ? 'true' : 'false';
|
||||
break;
|
||||
case 'date':
|
||||
ac.push(`and event_date_value = {eventValue${i}:DateTime('UTC')})`);
|
||||
break;
|
||||
}
|
||||
|
||||
params[`eventKey${i}`] = cv.eventKey;
|
||||
params[`eventValue${i}`] = value;
|
||||
|
||||
return ac;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function getFilterQuery(filters = {}, params = {}) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter !== undefined) {
|
||||
const column = FILTER_COLUMNS[key] || key;
|
||||
arr.push(`and ${column} = {${key}:String}`);
|
||||
params[key] = decodeURIComponent(filter);
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
|
||||
return {
|
||||
filterQuery: getFilterQuery(filters, params),
|
||||
};
|
||||
}
|
||||
|
||||
async function rawQuery(query, params = {}) {
|
||||
if (process.env.LOG_QUERY) {
|
||||
log('QUERY:\n', query);
|
||||
log('PARAMETERS:\n', params);
|
||||
}
|
||||
|
||||
await connect();
|
||||
|
||||
return clickhouse.query(query, { params }).toPromise();
|
||||
}
|
||||
|
||||
async function findUnique(data) {
|
||||
if (data.length > 1) {
|
||||
throw `${data.length} records found when expecting 1.`;
|
||||
}
|
||||
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
async function findFirst(data) {
|
||||
return data[0] ?? null;
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
if (enabled && !clickhouse) {
|
||||
clickhouse = process.env.CLICKHOUSE_URL && (global[CLICKHOUSE] || getClient());
|
||||
}
|
||||
|
||||
return clickhouse;
|
||||
}
|
||||
|
||||
export default {
|
||||
enabled,
|
||||
client: clickhouse,
|
||||
log,
|
||||
connect,
|
||||
getDateStringQuery,
|
||||
getDateQuery,
|
||||
getDateFormat,
|
||||
getBetweenDates,
|
||||
getFilterQuery,
|
||||
getEventDataFilterQuery,
|
||||
parseFilters,
|
||||
findUnique,
|
||||
findFirst,
|
||||
rawQuery,
|
||||
};
|
||||
14
lib/client.ts
Normal file
14
lib/client.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { getItem, setItem, removeItem } from 'next-basics';
|
||||
import { AUTH_TOKEN } from './constants';
|
||||
|
||||
export function getClientAuthToken() {
|
||||
return getItem(AUTH_TOKEN);
|
||||
}
|
||||
|
||||
export function setClientAuthToken(token) {
|
||||
setItem(AUTH_TOKEN, token);
|
||||
}
|
||||
|
||||
export function removeClientAuthToken() {
|
||||
removeItem(AUTH_TOKEN);
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
/* eslint-disable no-unused-vars */
|
||||
export const CURRENT_VERSION = process.env.currentVersion;
|
||||
export const AUTH_TOKEN = 'umami.auth';
|
||||
export const LOCALE_CONFIG = 'umami.locale';
|
||||
|
|
@ -11,6 +12,7 @@ export const HOMEPAGE_URL = 'https://umami.is';
|
|||
export const REPO_URL = 'https://github.com/umami-software/umami';
|
||||
export const UPDATES_URL = 'https://api.umami.is/v1/updates';
|
||||
export const TELEMETRY_PIXEL = 'https://i.umami.is/a.png';
|
||||
export const TRACKER_SCRIPT_URL = '/script.js';
|
||||
|
||||
export const DEFAULT_LOCALE = 'en-US';
|
||||
export const DEFAULT_THEME = 'light';
|
||||
|
|
@ -19,10 +21,83 @@ export const DEFAULT_DATE_RANGE = '24hour';
|
|||
export const DEFAULT_WEBSITE_LIMIT = 10;
|
||||
|
||||
export const REALTIME_RANGE = 30;
|
||||
export const REALTIME_INTERVAL = 3000;
|
||||
export const REALTIME_INTERVAL = 5000;
|
||||
|
||||
export const TYPE_WEBSITE = 'website';
|
||||
export const TYPE_ACCOUNT = 'account';
|
||||
export const FILTER_COMBINED = 'filter-combined';
|
||||
export const FILTER_RAW = 'filter-raw';
|
||||
export const FILTER_DAY = 'filter-day';
|
||||
export const FILTER_RANGE = 'filter-range';
|
||||
export const FILTER_REFERRERS = 'filter-referrers';
|
||||
export const FILTER_PAGES = 'filter-pages';
|
||||
|
||||
export const EVENT_COLUMNS = ['url', 'referrer', 'title', 'query', 'event'];
|
||||
|
||||
export const SESSION_COLUMNS = [
|
||||
'browser',
|
||||
'os',
|
||||
'device',
|
||||
'screen',
|
||||
'language',
|
||||
'country',
|
||||
'region',
|
||||
'city',
|
||||
];
|
||||
|
||||
export const FILTER_COLUMNS = {
|
||||
url: 'url_path',
|
||||
referrer: 'referrer_domain',
|
||||
title: 'page_title',
|
||||
query: 'url_query',
|
||||
event: 'event_name',
|
||||
region: 'subdivision1',
|
||||
};
|
||||
|
||||
export const EVENT_TYPE = {
|
||||
pageView: 1,
|
||||
customEvent: 2,
|
||||
} as const;
|
||||
|
||||
export const EVENT_DATA_TYPE = {
|
||||
string: 1,
|
||||
number: 2,
|
||||
boolean: 3,
|
||||
date: 4,
|
||||
array: 5,
|
||||
} as const;
|
||||
|
||||
export const KAFKA_TOPIC = {
|
||||
event: 'event',
|
||||
eventData: 'event_data',
|
||||
} as const;
|
||||
|
||||
export const ROLES = {
|
||||
admin: 'admin',
|
||||
user: 'user',
|
||||
teamOwner: 'team-owner',
|
||||
teamMember: 'team-member',
|
||||
} as const;
|
||||
|
||||
export const PERMISSIONS = {
|
||||
all: 'all',
|
||||
websiteCreate: 'website:create',
|
||||
websiteUpdate: 'website:update',
|
||||
websiteDelete: 'website:delete',
|
||||
teamCreate: 'team:create',
|
||||
teamUpdate: 'team:update',
|
||||
teamDelete: 'team:delete',
|
||||
} as const;
|
||||
|
||||
export const ROLE_PERMISSIONS = {
|
||||
[ROLES.admin]: [PERMISSIONS.all],
|
||||
[ROLES.user]: [
|
||||
PERMISSIONS.websiteCreate,
|
||||
PERMISSIONS.websiteUpdate,
|
||||
PERMISSIONS.websiteDelete,
|
||||
PERMISSIONS.teamCreate,
|
||||
],
|
||||
[ROLES.teamOwner]: [PERMISSIONS.teamUpdate, PERMISSIONS.teamDelete],
|
||||
[ROLES.teamMember]: [],
|
||||
} as const;
|
||||
|
||||
export const THEME_COLORS = {
|
||||
light: {
|
||||
|
|
@ -70,8 +145,6 @@ export const EVENT_COLORS = [
|
|||
'#ffec16',
|
||||
];
|
||||
|
||||
export const FILTER_IGNORED = Symbol.for('filter-ignored');
|
||||
|
||||
export const DOMAIN_REGEX =
|
||||
/^(localhost(:[1-9]\d{0,4})?|((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63})$/;
|
||||
|
||||
|
|
@ -1,19 +1,24 @@
|
|||
import crypto from 'crypto';
|
||||
import { v4, v5 } from 'uuid';
|
||||
import { startOfMonth } from 'date-fns';
|
||||
import { hash } from 'next-basics';
|
||||
|
||||
export function secret() {
|
||||
return hash(process.env.HASH_SALT || process.env.DATABASE_URL);
|
||||
return hash(process.env.APP_SECRET || process.env.DATABASE_URL);
|
||||
}
|
||||
|
||||
export function salt() {
|
||||
const ROTATING_SALT = hash(startOfMonth(new Date()).toUTCString());
|
||||
|
||||
return hash([secret(), ROTATING_SALT]);
|
||||
return hash(secret(), ROTATING_SALT);
|
||||
}
|
||||
|
||||
export function uuid(...args) {
|
||||
if (!args.length) return v4();
|
||||
|
||||
return v5(hash([...args, salt()]), v5.DNS);
|
||||
return v5(hash(...args, salt()), v5.DNS);
|
||||
}
|
||||
|
||||
export function md5(...args) {
|
||||
return crypto.createHash('md5').update(args.join('')).digest('hex');
|
||||
}
|
||||
|
|
|
|||
22
lib/date.js
22
lib/date.js
|
|
@ -26,6 +26,7 @@ import {
|
|||
differenceInCalendarMonths,
|
||||
differenceInCalendarYears,
|
||||
format,
|
||||
parseISO,
|
||||
} from 'date-fns';
|
||||
import { getDateLocale } from 'lib/lang';
|
||||
|
||||
|
|
@ -37,7 +38,16 @@ export function getLocalTime(t) {
|
|||
return addMinutes(new Date(t), new Date().getTimezoneOffset());
|
||||
}
|
||||
|
||||
export function getDateRange(value, locale = 'en-US') {
|
||||
export function parseDateRange(value, locale = 'en-US') {
|
||||
if (typeof value === 'object') {
|
||||
const { startDate, endDate } = value;
|
||||
return {
|
||||
...value,
|
||||
startDate: typeof startDate === 'string' ? parseISO(startDate) : startDate,
|
||||
endDate: typeof endDate === 'string' ? parseISO(endDate) : endDate,
|
||||
};
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
const dateLocale = getDateLocale(locale);
|
||||
|
||||
|
|
@ -170,19 +180,19 @@ export function getDateArray(data, startDate, endDate, unit) {
|
|||
const [diff, add, normalize] = dateFuncs[unit];
|
||||
const n = diff(endDate, startDate) + 1;
|
||||
|
||||
function findData(t) {
|
||||
const x = data.find(e => {
|
||||
return normalize(getDateFromString(e.t)).getTime() === t.getTime();
|
||||
function findData(date) {
|
||||
const d = data.find(({ x }) => {
|
||||
return normalize(getDateFromString(x)).getTime() === date.getTime();
|
||||
});
|
||||
|
||||
return x?.y || 0;
|
||||
return d?.y || 0;
|
||||
}
|
||||
|
||||
for (let i = 0; i < n; i++) {
|
||||
const t = normalize(add(startDate, i));
|
||||
const y = findData(t);
|
||||
|
||||
arr.push({ ...data[i], t, y });
|
||||
arr.push({ x: t, y });
|
||||
}
|
||||
|
||||
return arr;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ export const MYSQL = 'mysql';
|
|||
export const CLICKHOUSE = 'clickhouse';
|
||||
export const KAFKA = 'kafka';
|
||||
export const KAFKA_PRODUCER = 'kafka-producer';
|
||||
export const REDIS = 'redis';
|
||||
|
||||
// Fixes issue with converting bigint values
|
||||
BigInt.prototype.toJSON = function () {
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import {
|
|||
LAPTOP_SCREEN_WIDTH,
|
||||
MOBILE_SCREEN_WIDTH,
|
||||
} from './constants';
|
||||
import { NextApiRequestCollect } from 'pages/api/send';
|
||||
|
||||
let lookup;
|
||||
|
||||
|
|
@ -27,7 +28,7 @@ export function getIpAddress(req) {
|
|||
return requestIp.getClientIp(req);
|
||||
}
|
||||
|
||||
export function getDevice(screen, browser, os) {
|
||||
export function getDevice(screen, os) {
|
||||
if (!screen) return;
|
||||
|
||||
const [width] = screen.split('x');
|
||||
|
|
@ -55,12 +56,7 @@ export function getDevice(screen, browser, os) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function getCountry(req, ip) {
|
||||
// Cloudflare
|
||||
if (req.headers['cf-ipcountry']) {
|
||||
return req.headers['cf-ipcountry'];
|
||||
}
|
||||
|
||||
export async function getLocation(ip) {
|
||||
// Ignore local ips
|
||||
if (await isLocalhost(ip)) {
|
||||
return;
|
||||
|
|
@ -68,27 +64,36 @@ export async function getCountry(req, ip) {
|
|||
|
||||
// Database lookup
|
||||
if (!lookup) {
|
||||
lookup = await maxmind.open(path.resolve('node_modules/.geo/GeoLite2-Country.mmdb'));
|
||||
const dir = path.join(process.cwd(), 'geo');
|
||||
|
||||
lookup = await maxmind.open(path.resolve(dir, 'GeoLite2-City.mmdb'));
|
||||
}
|
||||
|
||||
const result = lookup.get(ip);
|
||||
const country = result?.country?.iso_code ?? result?.registered_country?.iso_code;
|
||||
const subdivision1 = result?.subdivisions?.[0]?.iso_code;
|
||||
const subdivision2 = result?.subdivisions?.[1]?.names?.en;
|
||||
const city = result?.city?.names?.en;
|
||||
|
||||
// country can not be set, fallback to registerd_country in this case
|
||||
return result?.country?.iso_code ?? result?.registered_country?.iso_code;
|
||||
return { country, subdivision1, subdivision2, city };
|
||||
}
|
||||
|
||||
export async function getClientInfo(req, { screen }) {
|
||||
export async function getClientInfo(req: NextApiRequestCollect, { screen }) {
|
||||
const userAgent = req.headers['user-agent'];
|
||||
const ip = getIpAddress(req);
|
||||
const country = await getCountry(req, ip);
|
||||
const location = await getLocation(ip);
|
||||
const country = location?.country;
|
||||
const subdivision1 = location?.subdivision1;
|
||||
const subdivision2 = location?.subdivision2;
|
||||
const city = location?.city;
|
||||
const browser = browserName(userAgent);
|
||||
const os = detectOS(userAgent);
|
||||
const device = getDevice(screen, browser, os);
|
||||
const device = getDevice(screen, os);
|
||||
|
||||
return { userAgent, browser, os, ip, country, device };
|
||||
return { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device };
|
||||
}
|
||||
|
||||
export function getJsonBody(req) {
|
||||
export function getJsonBody<T>(req): T {
|
||||
if ((req.headers['content-type'] || '').indexOf('text/plain') !== -1) {
|
||||
return JSON.parse(req.body);
|
||||
}
|
||||
74
lib/eventData.ts
Normal file
74
lib/eventData.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
import { isValid, parseISO } from 'date-fns';
|
||||
import { EVENT_DATA_TYPE } from './constants';
|
||||
import { EventDataTypes } from './types';
|
||||
|
||||
export function flattenJSON(
|
||||
eventData: { [key: string]: any },
|
||||
keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],
|
||||
parentKey = '',
|
||||
): { key: string; value: any; eventDataType: EventDataTypes }[] {
|
||||
return Object.keys(eventData).reduce(
|
||||
(acc, key) => {
|
||||
const value = eventData[key];
|
||||
const type = typeof eventData[key];
|
||||
|
||||
// nested object
|
||||
if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) {
|
||||
flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));
|
||||
} else {
|
||||
createKey(getKeyName(key, parentKey), value, acc);
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
{ keyValues, parentKey },
|
||||
).keyValues;
|
||||
}
|
||||
|
||||
export function getEventDataType(value: any): string {
|
||||
let type: string = typeof value;
|
||||
|
||||
if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
|
||||
type = 'date';
|
||||
}
|
||||
|
||||
return type;
|
||||
}
|
||||
|
||||
function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
|
||||
const type = getEventDataType(value);
|
||||
|
||||
let eventDataType = null;
|
||||
|
||||
switch (type) {
|
||||
case 'number':
|
||||
eventDataType = EVENT_DATA_TYPE.number;
|
||||
break;
|
||||
case 'string':
|
||||
eventDataType = EVENT_DATA_TYPE.string;
|
||||
break;
|
||||
case 'boolean':
|
||||
eventDataType = EVENT_DATA_TYPE.boolean;
|
||||
break;
|
||||
case 'date':
|
||||
eventDataType = EVENT_DATA_TYPE.date;
|
||||
break;
|
||||
case 'object':
|
||||
eventDataType = EVENT_DATA_TYPE.array;
|
||||
value = JSON.stringify(value);
|
||||
break;
|
||||
default:
|
||||
eventDataType = EVENT_DATA_TYPE.string;
|
||||
break;
|
||||
}
|
||||
|
||||
acc.keyValues.push({ key, value, eventDataType });
|
||||
}
|
||||
|
||||
function getKeyName(key, parentKey) {
|
||||
if (!parentKey) {
|
||||
return key;
|
||||
}
|
||||
|
||||
return `${parentKey}.${key}`;
|
||||
}
|
||||
|
|
@ -1,34 +1,10 @@
|
|||
export const urlFilter = data => {
|
||||
const isValidUrl = url => {
|
||||
return url !== '' && url !== null && !url.startsWith('#');
|
||||
};
|
||||
|
||||
const cleanUrl = url => {
|
||||
try {
|
||||
const { pathname, search } = new URL(url, location.origin);
|
||||
|
||||
if (search.startsWith('?')) {
|
||||
return `${pathname}${search}`;
|
||||
}
|
||||
|
||||
return pathname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const map = data.reduce((obj, { x, y }) => {
|
||||
if (!isValidUrl(x)) {
|
||||
return obj;
|
||||
}
|
||||
|
||||
const url = cleanUrl(x);
|
||||
|
||||
if (url) {
|
||||
if (!obj[url]) {
|
||||
obj[url] = y;
|
||||
if (x) {
|
||||
if (!obj[x]) {
|
||||
obj[x] = y;
|
||||
} else {
|
||||
obj[url] += y;
|
||||
obj[x] += y;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -66,6 +42,10 @@ export const refFilter = data => {
|
|||
return Object.keys(map).map(key => ({ x: key, y: map[key], w: links[key] }));
|
||||
};
|
||||
|
||||
export const emptyFilter = data => {
|
||||
return data.map(item => (item.x ? item : null)).filter(n => n);
|
||||
};
|
||||
|
||||
export const percentFilter = data => {
|
||||
const total = data.reduce((n, { y }) => n + y, 0);
|
||||
return data.map(({ x, y, ...props }) => ({ x, y, z: total ? (y / total) * 100 : 0, ...props }));
|
||||
|
|
@ -74,7 +54,7 @@ export const percentFilter = data => {
|
|||
export const paramFilter = data => {
|
||||
const map = data.reduce((obj, { x, y }) => {
|
||||
try {
|
||||
const searchParams = new URLSearchParams(x.split('?')[1]);
|
||||
const searchParams = new URLSearchParams(x);
|
||||
|
||||
for (const [key, value] of searchParams) {
|
||||
if (!obj[key]) {
|
||||
|
|
|
|||
|
|
@ -1,19 +1,20 @@
|
|||
import { Kafka, logLevel } from 'kafkajs';
|
||||
import dateFormat from 'dateformat';
|
||||
import debug from 'debug';
|
||||
import { Kafka, Mechanism, Producer, RecordMetadata, SASLOptions, logLevel } from 'kafkajs';
|
||||
import { KAFKA, KAFKA_PRODUCER } from 'lib/db';
|
||||
import * as tls from 'tls';
|
||||
|
||||
const log = debug('umami:kafka');
|
||||
|
||||
let kafka;
|
||||
let producer;
|
||||
let kafka: Kafka;
|
||||
let producer: Producer;
|
||||
const enabled = Boolean(process.env.KAFKA_URL && process.env.KAFKA_BROKER);
|
||||
|
||||
function getClient() {
|
||||
const { username, password } = new URL(process.env.KAFKA_URL);
|
||||
const brokers = process.env.KAFKA_BROKER.split(',');
|
||||
|
||||
const ssl =
|
||||
const ssl: { ssl?: tls.ConnectionOptions | boolean; sasl?: SASLOptions | Mechanism } =
|
||||
username && password
|
||||
? {
|
||||
ssl: {
|
||||
|
|
@ -30,7 +31,7 @@ function getClient() {
|
|||
}
|
||||
: {};
|
||||
|
||||
const client = new Kafka({
|
||||
const client: Kafka = new Kafka({
|
||||
clientId: 'umami',
|
||||
brokers: brokers,
|
||||
connectionTimeout: 3000,
|
||||
|
|
@ -47,7 +48,7 @@ function getClient() {
|
|||
return client;
|
||||
}
|
||||
|
||||
async function getProducer() {
|
||||
async function getProducer(): Promise<Producer> {
|
||||
const producer = kafka.producer();
|
||||
await producer.connect();
|
||||
|
||||
|
|
@ -60,25 +61,40 @@ async function getProducer() {
|
|||
return producer;
|
||||
}
|
||||
|
||||
function getDateFormat(date) {
|
||||
function getDateFormat(date): string {
|
||||
return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
|
||||
}
|
||||
|
||||
async function sendMessage(params, topic) {
|
||||
async function sendMessage(
|
||||
message: { [key: string]: string | number },
|
||||
topic: string,
|
||||
): Promise<RecordMetadata[]> {
|
||||
await connect();
|
||||
|
||||
return producer.send({
|
||||
topic,
|
||||
messages: [
|
||||
{
|
||||
value: JSON.stringify(message),
|
||||
},
|
||||
],
|
||||
acks: -1,
|
||||
});
|
||||
}
|
||||
|
||||
async function sendMessages(messages: { [key: string]: string | number }[], topic: string) {
|
||||
await connect();
|
||||
|
||||
await producer.send({
|
||||
topic,
|
||||
messages: [
|
||||
{
|
||||
value: JSON.stringify(params),
|
||||
},
|
||||
],
|
||||
messages: messages.map(a => {
|
||||
return { value: JSON.stringify(a) };
|
||||
}),
|
||||
acks: 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
async function connect(): Promise<Kafka> {
|
||||
if (!kafka) {
|
||||
kafka = process.env.KAFKA_URL && process.env.KAFKA_BROKER && (global[KAFKA] || getClient());
|
||||
|
||||
|
|
@ -98,4 +114,5 @@ export default {
|
|||
connect,
|
||||
getDateFormat,
|
||||
sendMessage,
|
||||
sendMessages,
|
||||
};
|
||||
23
lib/lang.js
23
lib/lang.js
|
|
@ -1,7 +1,7 @@
|
|||
import {
|
||||
arSA,
|
||||
bn,
|
||||
be,
|
||||
bn,
|
||||
cs,
|
||||
sk,
|
||||
da,
|
||||
|
|
@ -15,10 +15,10 @@ import {
|
|||
faIR,
|
||||
he,
|
||||
hi,
|
||||
hr,
|
||||
id,
|
||||
it,
|
||||
ja,
|
||||
km,
|
||||
ko,
|
||||
lt,
|
||||
mn,
|
||||
|
|
@ -41,19 +41,17 @@ import {
|
|||
ca,
|
||||
hu,
|
||||
vi,
|
||||
si,
|
||||
} from 'date-fns/locale';
|
||||
|
||||
export const languages = {
|
||||
'ar-SA': { label: 'العربية', dateLocale: arSA, dir: 'rtl' },
|
||||
'bn-BD': { label: 'বাংলা', dateLocale: bn },
|
||||
'zh-CN': { label: '中文', dateLocale: zhCN },
|
||||
'zh-TW': { label: '中文(繁體)', dateLocale: zhTW },
|
||||
'be-BY': { label: 'Беларуская', dateLocale: be },
|
||||
'bn-BD': { label: 'বাংলা', dateLocale: bn },
|
||||
'ca-ES': { label: 'Català', dateLocale: ca },
|
||||
'cs-CZ': { label: 'Čeština', dateLocale: cs },
|
||||
'da-DK': { label: 'Dansk', dateLocale: da },
|
||||
'de-DE': { label: 'Deutsch', dateLocale: de },
|
||||
'el-GR': { label: 'Ελληνικά', dateLocale: el },
|
||||
'en-US': { label: 'English (US)', dateLocale: enUS },
|
||||
'en-GB': { label: 'English (UK)', dateLocale: enGB },
|
||||
'es-MX': { label: 'Español', dateLocale: es },
|
||||
|
|
@ -61,36 +59,37 @@ export const languages = {
|
|||
'fo-FO': { label: 'Føroyskt' },
|
||||
'fr-FR': { label: 'Français', dateLocale: fr },
|
||||
'ga-ES': { label: 'Galacian (Spain)', dateLocale: es },
|
||||
'el-GR': { label: 'Ελληνικά', dateLocale: el },
|
||||
'he-IL': { label: 'עברית', dateLocale: he },
|
||||
'hi-IN': { label: 'हिन्दी', dateLocale: hi },
|
||||
'hu-HU': { label: 'Hungarian', dateLocale: hu },
|
||||
'hr-HR': { label: 'Hrvatski', dateLocale: hr },
|
||||
'it-IT': { label: 'Italiano', dateLocale: it },
|
||||
'id-ID': { label: 'Bahasa Indonesia', dateLocale: id },
|
||||
'it-IT': { label: 'Italiano', dateLocale: it },
|
||||
'ja-JP': { label: '日本語', dateLocale: ja },
|
||||
'km-KH': { label: 'ភាសាខ្មែរ', dateLocale: km },
|
||||
'ko-KR': { label: '한국어', dateLocale: ko },
|
||||
'lt-LT': { label: 'Lietuvių', dateLocale: lt },
|
||||
'ms-MY': { label: 'Malay', dateLocale: ms },
|
||||
'mn-MN': { label: 'Монгол', dateLocale: mn },
|
||||
'ms-MY': { label: 'Malay', dateLocale: ms },
|
||||
'nl-NL': { label: 'Nederlands', dateLocale: nl },
|
||||
'nb-NO': { label: 'Norsk Bokmål', dateLocale: nb },
|
||||
'pl-PL': { label: 'Polski', dateLocale: pl },
|
||||
'pt-PT': { label: 'Português', dateLocale: pt },
|
||||
'pt-BR': { label: 'Português do Brasil', dateLocale: ptBR },
|
||||
'pt-PT': { label: 'Português', dateLocale: pt },
|
||||
'ru-RU': { label: 'Русский', dateLocale: ru },
|
||||
'ro-RO': { label: 'Română', dateLocale: ro },
|
||||
'sk-SK': { label: 'Slovenčina', dateLocale: sk },
|
||||
'sl-SI': { label: 'Slovenščina', dateLocale: sl },
|
||||
'fi-FI': { label: 'Suomi', dateLocale: fi },
|
||||
'sv-SE': { label: 'Svenska', dateLocale: sv },
|
||||
'de-CH': { label: 'Schwiizerdütsch', dateLocale: de },
|
||||
'ta-IN': { label: 'தமிழ்', dateLocale: ta },
|
||||
'si-LK': { label: 'සිංහල', dateLocale: si },
|
||||
'th-TH': { label: 'ภาษาไทย', dateLocale: th },
|
||||
'tr-TR': { label: 'Türkçe', dateLocale: tr },
|
||||
'uk-UA': { label: 'українська', dateLocale: uk },
|
||||
'ur-PK': { label: 'Urdu (Pakistan)', dateLocale: uk, dir: 'rtl' },
|
||||
'vi-VN': { label: 'Tiếng Việt', dateLocale: vi },
|
||||
'zh-CN': { label: '中文', dateLocale: zhCN },
|
||||
'zh-TW': { label: '中文(繁體)', dateLocale: zhTW },
|
||||
};
|
||||
|
||||
export function getDateLocale(locale) {
|
||||
|
|
|
|||
|
|
@ -1,43 +0,0 @@
|
|||
import { createMiddleware, unauthorized, badRequest, serverError } from 'next-basics';
|
||||
import cors from 'cors';
|
||||
import { getSession } from './session';
|
||||
import { getAuthToken, getShareToken } from './auth';
|
||||
|
||||
export const useCors = createMiddleware(
|
||||
cors({
|
||||
// Cache CORS preflight request 24 hours by default
|
||||
maxAge: process.env.CORS_MAX_AGE || 86400,
|
||||
}),
|
||||
);
|
||||
|
||||
export const useSession = createMiddleware(async (req, res, next) => {
|
||||
let session;
|
||||
|
||||
try {
|
||||
session = await getSession(req);
|
||||
} catch (e) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(e);
|
||||
|
||||
return serverError(res, e.message);
|
||||
}
|
||||
|
||||
if (!session) {
|
||||
return badRequest(res);
|
||||
}
|
||||
|
||||
req.session = session;
|
||||
next();
|
||||
});
|
||||
|
||||
export const useAuth = createMiddleware(async (req, res, next) => {
|
||||
const token = await getAuthToken(req);
|
||||
const shareToken = await getShareToken(req);
|
||||
|
||||
if (!token && !shareToken) {
|
||||
return unauthorized(res);
|
||||
}
|
||||
|
||||
req.auth = { ...token, shareToken };
|
||||
next();
|
||||
});
|
||||
63
lib/middleware.ts
Normal file
63
lib/middleware.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import { createMiddleware, unauthorized, badRequest, parseSecureToken } from 'next-basics';
|
||||
import debug from 'debug';
|
||||
import cors from 'cors';
|
||||
import { validate } from 'uuid';
|
||||
import redis from '@umami/redis-client';
|
||||
import { findSession } from 'lib/session';
|
||||
import { getAuthToken, parseShareToken } from 'lib/auth';
|
||||
import { secret } from 'lib/crypto';
|
||||
import { ROLES } from 'lib/constants';
|
||||
import { getUser } from '../queries';
|
||||
import { NextApiRequestCollect } from 'pages/api/send';
|
||||
|
||||
const log = debug('umami:middleware');
|
||||
|
||||
export const useCors = createMiddleware(
|
||||
cors({
|
||||
// Cache CORS preflight request 24 hours by default
|
||||
maxAge: process.env.CORS_MAX_AGE || 86400,
|
||||
}),
|
||||
);
|
||||
|
||||
export const useSession = createMiddleware(async (req, res, next) => {
|
||||
const session = await findSession(req as NextApiRequestCollect);
|
||||
|
||||
if (!session) {
|
||||
log('useSession: Session not found');
|
||||
return badRequest(res, 'Session not found.');
|
||||
}
|
||||
|
||||
(req as any).session = session;
|
||||
next();
|
||||
});
|
||||
|
||||
export const useAuth = createMiddleware(async (req, res, next) => {
|
||||
const token = getAuthToken(req);
|
||||
const payload = parseSecureToken(token, secret());
|
||||
const shareToken = await parseShareToken(req);
|
||||
|
||||
let user = null;
|
||||
const { userId, authKey } = payload || {};
|
||||
|
||||
if (validate(userId)) {
|
||||
user = await getUser({ id: userId });
|
||||
} else if (redis.enabled && authKey) {
|
||||
user = await redis.get(authKey);
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
log({ token, shareToken, payload, user });
|
||||
}
|
||||
|
||||
if (!user?.id && !shareToken) {
|
||||
log('useAuth: User not authorized');
|
||||
return unauthorized(res);
|
||||
}
|
||||
|
||||
if (user) {
|
||||
user.isAdmin = user.role === ROLES.admin;
|
||||
}
|
||||
|
||||
(req as any).auth = { user, token, shareToken, authKey };
|
||||
next();
|
||||
});
|
||||
291
lib/prisma.js
291
lib/prisma.js
|
|
@ -1,291 +0,0 @@
|
|||
import { PrismaClient } from '@prisma/client';
|
||||
import chalk from 'chalk';
|
||||
import moment from 'moment-timezone';
|
||||
import debug from 'debug';
|
||||
import { PRISMA, MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
|
||||
import { FILTER_IGNORED } from 'lib/constants';
|
||||
|
||||
const MYSQL_DATE_FORMATS = {
|
||||
minute: '%Y-%m-%d %H:%i:00',
|
||||
hour: '%Y-%m-%d %H:00:00',
|
||||
day: '%Y-%m-%d',
|
||||
month: '%Y-%m-01',
|
||||
year: '%Y-01-01',
|
||||
};
|
||||
|
||||
const POSTGRESQL_DATE_FORMATS = {
|
||||
minute: 'YYYY-MM-DD HH24:MI:00',
|
||||
hour: 'YYYY-MM-DD HH24:00:00',
|
||||
day: 'YYYY-MM-DD',
|
||||
month: 'YYYY-MM-01',
|
||||
year: 'YYYY-01-01',
|
||||
};
|
||||
|
||||
const log = debug('umami:prisma');
|
||||
|
||||
const PRISMA_OPTIONS = {
|
||||
log: [
|
||||
{
|
||||
emit: 'event',
|
||||
level: 'query',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function logQuery(e) {
|
||||
log(chalk.yellow(e.params), '->', e.query, chalk.greenBright(`${e.duration}ms`));
|
||||
}
|
||||
|
||||
function toUuid() {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
return '::uuid';
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function getClient(options) {
|
||||
const prisma = new PrismaClient(options);
|
||||
|
||||
if (process.env.LOG_QUERY) {
|
||||
prisma.$on('query', logQuery);
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global[PRISMA] = prisma;
|
||||
}
|
||||
|
||||
log('Prisma initialized');
|
||||
|
||||
return prisma;
|
||||
}
|
||||
|
||||
function getDateQuery(field, unit, timezone) {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
if (timezone) {
|
||||
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
if (timezone) {
|
||||
const tz = moment.tz(timezone).format('Z');
|
||||
|
||||
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
}
|
||||
|
||||
function getTimestampInterval(field) {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
return `floor(extract(epoch from max(${field}) - min(${field})))`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`;
|
||||
}
|
||||
}
|
||||
|
||||
function getSanitizedColumns(columns) {
|
||||
return Object.keys(columns).reduce((acc, keyName) => {
|
||||
const sanitizedProperty = keyName.replace(/[^\w\s_]/g, '');
|
||||
|
||||
acc[sanitizedProperty] = columns[keyName];
|
||||
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function getJsonField(column, property, isNumber, params) {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
params.push(property);
|
||||
|
||||
let accessor = `${column} ->> $${params.length}`;
|
||||
|
||||
if (isNumber) {
|
||||
accessor = `CAST(${accessor} AS DECIMAL)`;
|
||||
}
|
||||
|
||||
return accessor;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return `${column} ->> '$.${property}'`;
|
||||
}
|
||||
}
|
||||
|
||||
function getEventDataColumnsQuery(column, columns, params) {
|
||||
const query = Object.keys(columns).reduce((arr, key, i) => {
|
||||
const filter = columns[key];
|
||||
|
||||
if (filter === undefined) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (filter) {
|
||||
case 'sum':
|
||||
case 'avg':
|
||||
case 'min':
|
||||
case 'max':
|
||||
arr.push(`${filter}(${getJsonField(column, key, true, params)}) as "${i}"`);
|
||||
break;
|
||||
case 'count':
|
||||
arr.push(`${filter}(${getJsonField(column, key, false, params)}) as "${i}"`);
|
||||
break;
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join(',\n');
|
||||
}
|
||||
|
||||
function getEventDataFilterQuery(column, filters, params) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
const isNumber = filter && typeof filter === 'number';
|
||||
|
||||
arr.push(`${getJsonField(column, key, isNumber, params)} = $${params.length + 1}`);
|
||||
|
||||
params.push(filter);
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\nand ');
|
||||
}
|
||||
|
||||
function getFilterQuery(table, column, filters = {}, params = []) {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter === undefined || filter === FILTER_IGNORED) {
|
||||
return arr;
|
||||
}
|
||||
|
||||
switch (key) {
|
||||
case 'url':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'os':
|
||||
case 'browser':
|
||||
case 'device':
|
||||
case 'country':
|
||||
if (table === 'session') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'event_name':
|
||||
if (table === 'event') {
|
||||
arr.push(`and ${table}.${key}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
break;
|
||||
|
||||
case 'referrer':
|
||||
if (table === 'pageview' || table === 'event') {
|
||||
arr.push(`and ${table}.referrer like $${params.length + 1}`);
|
||||
params.push(`%${decodeURIComponent(filter)}%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.referrer not like $${params.length + 1}`);
|
||||
arr.push(`and ${table}.referrer not like '/%'`);
|
||||
params.push(`%://${filter}/%`);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'query':
|
||||
if (table === 'pageview') {
|
||||
arr.push(`and ${table}.url like '%?%'`);
|
||||
}
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
|
||||
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
|
||||
filters;
|
||||
|
||||
const pageviewFilters = { domain, url, referrer, query };
|
||||
const sessionFilters = { os, browser, device, country };
|
||||
const eventFilters = { url: event_url, event_name };
|
||||
|
||||
return {
|
||||
pageviewFilters,
|
||||
sessionFilters,
|
||||
eventFilters,
|
||||
event: { event_name },
|
||||
joinSession:
|
||||
os || browser || device || country
|
||||
? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
|
||||
: '',
|
||||
pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
|
||||
sessionQuery: getFilterQuery('session', column, sessionFilters, params),
|
||||
eventQuery: getFilterQuery('event', column, eventFilters, params),
|
||||
};
|
||||
}
|
||||
|
||||
async function rawQuery(query, params = []) {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db !== POSTGRESQL && db !== MYSQL) {
|
||||
return Promise.reject(new Error('Unknown database.'));
|
||||
}
|
||||
|
||||
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
|
||||
|
||||
return prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]);
|
||||
}
|
||||
|
||||
async function transaction(queries) {
|
||||
return prisma.$transaction(queries);
|
||||
}
|
||||
|
||||
// Initialization
|
||||
const prisma = global[PRISMA] || getClient(PRISMA_OPTIONS);
|
||||
|
||||
export default {
|
||||
client: prisma,
|
||||
log,
|
||||
toUuid,
|
||||
getDateQuery,
|
||||
getTimestampInterval,
|
||||
getFilterQuery,
|
||||
getEventDataColumnsQuery,
|
||||
getEventDataFilterQuery,
|
||||
getSanitizedColumns,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
transaction,
|
||||
};
|
||||
162
lib/prisma.ts
Normal file
162
lib/prisma.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import prisma from '@umami/prisma-client';
|
||||
import moment from 'moment-timezone';
|
||||
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
|
||||
import { getEventDataType } from './eventData';
|
||||
import { FILTER_COLUMNS } from './constants';
|
||||
|
||||
const MYSQL_DATE_FORMATS = {
|
||||
minute: '%Y-%m-%d %H:%i:00',
|
||||
hour: '%Y-%m-%d %H:00:00',
|
||||
day: '%Y-%m-%d',
|
||||
month: '%Y-%m-01',
|
||||
year: '%Y-01-01',
|
||||
};
|
||||
|
||||
const POSTGRESQL_DATE_FORMATS = {
|
||||
minute: 'YYYY-MM-DD HH24:MI:00',
|
||||
hour: 'YYYY-MM-DD HH24:00:00',
|
||||
day: 'YYYY-MM-DD',
|
||||
month: 'YYYY-MM-01',
|
||||
year: 'YYYY-01-01',
|
||||
};
|
||||
|
||||
function toUuid(): string {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
return '::uuid';
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function getDateQuery(field: string, unit: string, timezone?: string): string {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
if (timezone) {
|
||||
return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
if (timezone) {
|
||||
const tz = moment.tz(timezone).format('Z');
|
||||
|
||||
return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
|
||||
return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
|
||||
}
|
||||
}
|
||||
|
||||
function getTimestampInterval(field: string): string {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db === POSTGRESQL) {
|
||||
return `floor(extract(epoch from max(${field}) - min(${field})))`;
|
||||
}
|
||||
|
||||
if (db === MYSQL) {
|
||||
return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`;
|
||||
}
|
||||
}
|
||||
|
||||
function getEventDataFilterQuery(
|
||||
filters: {
|
||||
eventKey?: string;
|
||||
eventValue?: string | number | boolean | Date;
|
||||
}[],
|
||||
params: any[],
|
||||
) {
|
||||
const query = filters.reduce((ac, cv) => {
|
||||
const type = getEventDataType(cv.eventValue);
|
||||
|
||||
let value = cv.eventValue;
|
||||
|
||||
ac.push(`and (event_key = $${params.length + 1}`);
|
||||
params.push(cv.eventKey);
|
||||
|
||||
switch (type) {
|
||||
case 'number':
|
||||
ac.push(`and event_numeric_value = $${params.length + 1})`);
|
||||
params.push(value);
|
||||
break;
|
||||
case 'string':
|
||||
ac.push(`and event_string_value = $${params.length + 1})`);
|
||||
params.push(decodeURIComponent(cv.eventValue as string));
|
||||
break;
|
||||
case 'boolean':
|
||||
ac.push(`and event_string_value = $${params.length + 1})`);
|
||||
params.push(decodeURIComponent(cv.eventValue as string));
|
||||
value = cv ? 'true' : 'false';
|
||||
break;
|
||||
case 'date':
|
||||
ac.push(`and event_date_value = $${params.length + 1})`);
|
||||
params.push(cv.eventValue);
|
||||
break;
|
||||
}
|
||||
|
||||
return ac;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function getFilterQuery(filters = {}, params = []): string {
|
||||
const query = Object.keys(filters).reduce((arr, key) => {
|
||||
const filter = filters[key];
|
||||
|
||||
if (filter !== undefined) {
|
||||
const column = FILTER_COLUMNS[key] || key;
|
||||
arr.push(`and ${column}=$${params.length + 1}`);
|
||||
params.push(decodeURIComponent(filter));
|
||||
}
|
||||
|
||||
return arr;
|
||||
}, []);
|
||||
|
||||
return query.join('\n');
|
||||
}
|
||||
|
||||
function parseFilters(
|
||||
filters: { [key: string]: any } = {},
|
||||
params = [],
|
||||
sessionKey = 'session_id',
|
||||
) {
|
||||
const { os, browser, device, country, region, city } = filters;
|
||||
|
||||
return {
|
||||
joinSession:
|
||||
os || browser || device || country || region || city
|
||||
? `inner join session on website_event.${sessionKey} = session.${sessionKey}`
|
||||
: '',
|
||||
filterQuery: getFilterQuery(filters, params),
|
||||
};
|
||||
}
|
||||
|
||||
async function rawQuery(query: string, params: never[] = []): Promise<any> {
|
||||
const db = getDatabaseType(process.env.DATABASE_URL);
|
||||
|
||||
if (db !== POSTGRESQL && db !== MYSQL) {
|
||||
return Promise.reject(new Error('Unknown database.'));
|
||||
}
|
||||
|
||||
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
|
||||
|
||||
return prisma.rawQuery(sql, params);
|
||||
}
|
||||
|
||||
export default {
|
||||
...prisma,
|
||||
getDateQuery,
|
||||
getTimestampInterval,
|
||||
getFilterQuery,
|
||||
getEventDataFilterQuery,
|
||||
toUuid,
|
||||
parseFilters,
|
||||
rawQuery,
|
||||
};
|
||||
51
lib/query.ts
Normal file
51
lib/query.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
import cache from 'lib/cache';
|
||||
import { getWebsite, getSession, getUser } from 'queries';
|
||||
import { User, Website, Session } from '@prisma/client';
|
||||
|
||||
export async function loadWebsite(websiteId: string): Promise<Website> {
|
||||
let website;
|
||||
|
||||
if (cache.enabled) {
|
||||
website = await cache.fetchWebsite(websiteId);
|
||||
} else {
|
||||
website = await getWebsite({ id: websiteId });
|
||||
}
|
||||
|
||||
if (!website || website.deletedAt) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return website;
|
||||
}
|
||||
|
||||
export async function loadSession(sessionId: string): Promise<Session> {
|
||||
let session;
|
||||
|
||||
if (cache.enabled) {
|
||||
session = await cache.fetchSession(sessionId);
|
||||
} else {
|
||||
session = await getSession({ id: sessionId });
|
||||
}
|
||||
|
||||
if (!session) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return session;
|
||||
}
|
||||
|
||||
export async function loadUser(userId: string): Promise<User> {
|
||||
let user;
|
||||
|
||||
if (cache.enabled) {
|
||||
user = await cache.fetchUser(userId);
|
||||
} else {
|
||||
user = await getUser({ id: userId });
|
||||
}
|
||||
|
||||
if (!user || user.deletedAt) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return user;
|
||||
}
|
||||
79
lib/redis.js
79
lib/redis.js
|
|
@ -1,79 +0,0 @@
|
|||
import Redis from 'ioredis';
|
||||
import { startOfMonth } from 'date-fns';
|
||||
import debug from 'debug';
|
||||
import { getSessions, getAllWebsites } from 'queries';
|
||||
import { REDIS } from 'lib/db';
|
||||
|
||||
const log = debug('umami:redis');
|
||||
const INITIALIZED = 'redis:initialized';
|
||||
export const DELETED = 'deleted';
|
||||
|
||||
let redis;
|
||||
const enabled = Boolean(process.env.REDIS_URL);
|
||||
|
||||
function getClient() {
|
||||
if (!process.env.REDIS_URL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const redis = new Redis(process.env.REDIS_URL, {
|
||||
retryStrategy(times) {
|
||||
log(`Redis reconnecting attempt: ${times}`);
|
||||
return 5000;
|
||||
},
|
||||
});
|
||||
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
global[REDIS] = redis;
|
||||
}
|
||||
|
||||
log('Redis initialized');
|
||||
|
||||
return redis;
|
||||
}
|
||||
|
||||
async function stageData() {
|
||||
const sessions = await getSessions([], startOfMonth(new Date()));
|
||||
const websites = await getAllWebsites();
|
||||
|
||||
const sessionUuids = sessions.map(a => {
|
||||
return { key: `session:${a.sessionUuid}`, value: 1 };
|
||||
});
|
||||
const websiteIds = websites.map(a => {
|
||||
return { key: `website:${a.websiteUuid}`, value: Number(a.websiteId) };
|
||||
});
|
||||
|
||||
await addSet(sessionUuids);
|
||||
await addSet(websiteIds);
|
||||
|
||||
await redis.set(INITIALIZED, 1);
|
||||
}
|
||||
|
||||
async function addSet(ids) {
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
const { key, value } = ids[i];
|
||||
await redis.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
async function get(key) {
|
||||
await connect();
|
||||
|
||||
return redis.get(key);
|
||||
}
|
||||
|
||||
async function set(key, value) {
|
||||
await connect();
|
||||
|
||||
return redis.set(key, value);
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
if (!redis) {
|
||||
redis = process.env.REDIS_URL && (global[REDIS] || getClient());
|
||||
}
|
||||
|
||||
return redis;
|
||||
}
|
||||
|
||||
export default { enabled, client: redis, log, connect, get, set, stageData };
|
||||
106
lib/session.js
106
lib/session.js
|
|
@ -1,106 +0,0 @@
|
|||
import { parseToken } from 'next-basics';
|
||||
import { validate } from 'uuid';
|
||||
import { secret, uuid } from 'lib/crypto';
|
||||
import redis, { DELETED } from 'lib/redis';
|
||||
import clickhouse from 'lib/clickhouse';
|
||||
import { getClientInfo, getJsonBody } from 'lib/request';
|
||||
import { createSession, getSessionByUuid, getWebsite } from 'queries';
|
||||
|
||||
export async function getSession(req) {
|
||||
const { payload } = getJsonBody(req);
|
||||
|
||||
if (!payload) {
|
||||
throw new Error('Invalid request');
|
||||
}
|
||||
|
||||
const cache = req.headers['x-umami-cache'];
|
||||
|
||||
if (cache) {
|
||||
const result = await parseToken(cache, secret());
|
||||
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
const { website: websiteUuid, hostname, screen, language } = payload;
|
||||
|
||||
if (!validate(websiteUuid)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let websiteId = null;
|
||||
|
||||
// Check if website exists
|
||||
if (redis.enabled) {
|
||||
websiteId = Number(await redis.get(`website:${websiteUuid}`));
|
||||
}
|
||||
|
||||
// Check database if does not exists in Redis
|
||||
if (!websiteId) {
|
||||
const website = await getWebsite({ websiteUuid });
|
||||
websiteId = website ? website.id : null;
|
||||
}
|
||||
|
||||
if (!websiteId || websiteId === DELETED) {
|
||||
throw new Error(`Website not found: ${websiteUuid}`);
|
||||
}
|
||||
|
||||
const { userAgent, browser, os, ip, country, device } = await getClientInfo(req, payload);
|
||||
const sessionUuid = uuid(websiteUuid, hostname, ip, userAgent);
|
||||
|
||||
let sessionId = null;
|
||||
let session = null;
|
||||
|
||||
if (!clickhouse.enabled) {
|
||||
// Check if session exists
|
||||
if (redis.enabled) {
|
||||
sessionId = Number(await redis.get(`session:${sessionUuid}`));
|
||||
}
|
||||
|
||||
// Check database if does not exists in Redis
|
||||
if (!sessionId) {
|
||||
session = await getSessionByUuid(sessionUuid);
|
||||
sessionId = session ? session.id : null;
|
||||
}
|
||||
|
||||
if (!sessionId) {
|
||||
try {
|
||||
session = await createSession(websiteId, {
|
||||
sessionUuid,
|
||||
hostname,
|
||||
browser,
|
||||
os,
|
||||
screen,
|
||||
language,
|
||||
country,
|
||||
device,
|
||||
});
|
||||
} catch (e) {
|
||||
if (!e.message.toLowerCase().includes('unique constraint')) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
session = {
|
||||
sessionId,
|
||||
sessionUuid,
|
||||
hostname,
|
||||
browser,
|
||||
os,
|
||||
screen,
|
||||
language,
|
||||
country,
|
||||
device,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
website: {
|
||||
websiteId,
|
||||
websiteUuid,
|
||||
},
|
||||
session,
|
||||
};
|
||||
}
|
||||
92
lib/session.ts
Normal file
92
lib/session.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import clickhouse from 'lib/clickhouse';
|
||||
import { secret, uuid } from 'lib/crypto';
|
||||
import { getClientInfo, getJsonBody } from 'lib/detect';
|
||||
import { parseToken } from 'next-basics';
|
||||
import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
|
||||
import { createSession } from 'queries';
|
||||
import { validate } from 'uuid';
|
||||
import { loadSession, loadWebsite } from './query';
|
||||
|
||||
export async function findSession(req: NextApiRequestCollect) {
|
||||
const { payload } = getJsonBody<CollectRequestBody>(req);
|
||||
|
||||
if (!payload) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check if cache token is passed
|
||||
const cacheToken = req.headers['x-umami-cache'];
|
||||
|
||||
if (cacheToken) {
|
||||
const result = await parseToken(cacheToken, secret());
|
||||
|
||||
if (result) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify payload
|
||||
const { website: websiteId, hostname, screen, language } = payload;
|
||||
|
||||
if (!validate(websiteId)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Find website
|
||||
const website = await loadWebsite(websiteId);
|
||||
|
||||
if (!website) {
|
||||
throw new Error(`Website not found: ${websiteId}`);
|
||||
}
|
||||
|
||||
const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
|
||||
await getClientInfo(req, payload);
|
||||
const sessionId = uuid(websiteId, hostname, ip, userAgent);
|
||||
|
||||
// Clickhouse does not require session lookup
|
||||
if (clickhouse.enabled) {
|
||||
return {
|
||||
id: sessionId,
|
||||
websiteId,
|
||||
hostname,
|
||||
browser,
|
||||
os,
|
||||
device,
|
||||
screen,
|
||||
language,
|
||||
country,
|
||||
subdivision1,
|
||||
subdivision2,
|
||||
city,
|
||||
};
|
||||
}
|
||||
|
||||
// Find session
|
||||
let session = await loadSession(sessionId);
|
||||
|
||||
// Create a session if not found
|
||||
if (!session) {
|
||||
try {
|
||||
session = await createSession({
|
||||
id: sessionId,
|
||||
websiteId,
|
||||
hostname,
|
||||
browser,
|
||||
os,
|
||||
device,
|
||||
screen,
|
||||
language,
|
||||
country,
|
||||
subdivision1,
|
||||
subdivision2,
|
||||
city,
|
||||
});
|
||||
} catch (e: any) {
|
||||
if (!e.message.toLowerCase().includes('unique constraint')) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return session;
|
||||
}
|
||||
131
lib/types.ts
Normal file
131
lib/types.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
import { NextApiRequest } from 'next';
|
||||
import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
|
||||
|
||||
type ObjectValues<T> = T[keyof T];
|
||||
|
||||
export type Roles = ObjectValues<typeof ROLES>;
|
||||
|
||||
export type EventTypes = ObjectValues<typeof EVENT_TYPE>;
|
||||
|
||||
export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;
|
||||
|
||||
export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;
|
||||
|
||||
export interface EventData {
|
||||
[key: string]: number | string | EventData | number[] | string[] | EventData[];
|
||||
}
|
||||
|
||||
export interface Auth {
|
||||
user?: {
|
||||
id: string;
|
||||
username: string;
|
||||
role: string;
|
||||
isAdmin: boolean;
|
||||
};
|
||||
shareToken?: {
|
||||
websiteId: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface NextApiRequestQueryBody<TQuery = any, TBody = any> extends NextApiRequest {
|
||||
auth?: Auth;
|
||||
query: TQuery & { [key: string]: string | string[] };
|
||||
body: TBody;
|
||||
headers: any;
|
||||
}
|
||||
|
||||
export interface NextApiRequestAuth extends NextApiRequest {
|
||||
auth?: Auth;
|
||||
headers: any;
|
||||
}
|
||||
|
||||
export interface User {
|
||||
id: string;
|
||||
username: string;
|
||||
password?: string;
|
||||
role: string;
|
||||
createdAt?: Date;
|
||||
}
|
||||
|
||||
export interface Website {
|
||||
id: string;
|
||||
userId: string;
|
||||
resetAt: Date;
|
||||
name: string;
|
||||
domain: string;
|
||||
shareId: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Share {
|
||||
id: string;
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface WebsiteActive {
|
||||
x: number;
|
||||
}
|
||||
|
||||
export interface WebsiteMetric {
|
||||
x: string;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface WebsiteMetricFilter {
|
||||
domain?: string;
|
||||
url?: string;
|
||||
referrer?: string;
|
||||
title?: string;
|
||||
query?: string;
|
||||
event?: string;
|
||||
os?: string;
|
||||
browser?: string;
|
||||
device?: string;
|
||||
country?: string;
|
||||
region?: string;
|
||||
city?: string;
|
||||
}
|
||||
|
||||
export interface WebsiteEventMetric {
|
||||
x: string;
|
||||
t: string;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface WebsiteEventDataMetric {
|
||||
x: string;
|
||||
t: string;
|
||||
eventName?: string;
|
||||
urlPath?: string;
|
||||
}
|
||||
|
||||
export interface WebsitePageviews {
|
||||
pageviews: {
|
||||
t: string;
|
||||
y: number;
|
||||
};
|
||||
sessions: {
|
||||
t: string;
|
||||
y: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface WebsiteStats {
|
||||
pageviews: { value: number; change: number };
|
||||
uniques: { value: number; change: number };
|
||||
bounces: { value: number; change: number };
|
||||
totalTime: { value: number; change: number };
|
||||
}
|
||||
|
||||
export interface RealtimeInit {
|
||||
websites: Website[];
|
||||
token: string;
|
||||
data: RealtimeUpdate;
|
||||
}
|
||||
|
||||
export interface RealtimeUpdate {
|
||||
pageviews: any[];
|
||||
sessions: any[];
|
||||
events: any[];
|
||||
timestamp: number;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue