Merge branch 'dev' into fix-1966-issue-IDN

lib/auth.ts

@@ -1,6 +1,6 @@
-import debug from 'debug';
+import { Report } from '@prisma/client';
 import redis from '@umami/redis-client';
-import cache from 'lib/cache';
+import debug from 'debug';
 import { PERMISSIONS, ROLE_PERMISSIONS, SHARE_TOKEN_HEADER } from 'lib/constants';
 import { secret } from 'lib/crypto';
 import {
@@ -10,11 +10,11 @@ import {
   parseSecureToken,
   parseToken,
 } from 'next-basics';
-import { getTeamUser, getTeamUserById } from 'queries';
+import { getTeamUser } from 'queries';
 import { getTeamWebsite, getTeamWebsiteByTeamMemberId } from 'queries/admin/teamWebsite';
 import { validate } from 'uuid';
-import { Auth } from './types';
 import { loadWebsite } from './query';
+import { Auth } from './types';
 
 const log = debug('umami:auth');
 
@@ -135,7 +135,34 @@ export async function canDeleteWebsite({ user }: Auth, websiteId: string) {
   return false;
 }
 
-// To-do: Implement when payments are setup.
+export async function canViewReport(auth: Auth, report: Report) {
+  if (auth.user.isAdmin) {
+    return true;
+  }
+
+  if ((auth.user.id = report.userId)) {
+    return true;
+  }
+
+  if (await canViewWebsite(auth, report.websiteId)) {
+    return true;
+  }
+
+  return false;
+}
+
+export async function canUpdateReport(auth: Auth, report: Report) {
+  if (auth.user.isAdmin) {
+    return true;
+  }
+
+  if ((auth.user.id = report.userId)) {
+    return true;
+  }
+
+  return false;
+}
+
 export async function canCreateTeam({ user }: Auth) {
   if (user.isAdmin) {
     return true;
@@ -144,7 +171,6 @@ export async function canCreateTeam({ user }: Auth) {
   return !!user;
 }
 
-// To-do: Implement when payments are setup.
 export async function canViewTeam({ user }: Auth, teamId: string) {
   if (user.isAdmin) {
     return true;

lib/cache.ts

@@ -2,35 +2,7 @@ import { User, Website } from '@prisma/client';
 import redis from '@umami/redis-client';
 import { getSession, getUser, getWebsite } from '../queries';
 
-const DELETED = 'DELETED';
-
-async function fetchObject(key, query) {
-  const obj = await redis.get(key);
-
-  if (obj === DELETED) {
-    return null;
-  }
-
-  if (!obj) {
-    return query().then(async data => {
-      if (data) {
-        await redis.set(key, data);
-      }
-
-      return data;
-    });
-  }
-
-  return obj;
-}
-
-async function storeObject(key, data) {
-  return redis.set(key, data);
-}
-
-async function deleteObject(key, soft = false) {
-  return soft ? redis.set(key, DELETED) : redis.del(key);
-}
+const { fetchObject, storeObject, deleteObject } = redis;
 
 async function fetchWebsite(id): Promise<Website> {
   return fetchObject(`website:${id}`, () => getWebsite({ id }));
@@ -77,6 +49,16 @@ async function deleteSession(id) {
   return deleteObject(`session:${id}`);
 }
 
+async function fetchUserBlock(userId: string) {
+  const key = `user:block:${userId}`;
+  return redis.get(key);
+}
+
+async function incrementUserBlock(userId: string) {
+  const key = `user:block:${userId}`;
+  return redis.incr(key);
+}
+
 export default {
   fetchWebsite,
   storeWebsite,
@@ -87,5 +69,7 @@ export default {
   fetchSession,
   storeSession,
   deleteSession,
+  fetchUserBlock,
+  incrementUserBlock,
   enabled: redis.enabled,
 };

lib/charts.js

@@ -0,0 +1,62 @@
+import { StatusLight } from 'react-basics';
+import { dateFormat } from 'lib/date';
+import { formatLongNumber } from 'lib/format';
+
+export function renderNumberLabels(label) {
+  return +label > 1000 ? formatLongNumber(label) : label;
+}
+
+export function renderDateLabels(unit, locale) {
+  return (label, index, values) => {
+    const d = new Date(values[index].value);
+
+    switch (unit) {
+      case 'minute':
+        return dateFormat(d, 'h:mm', locale);
+      case 'hour':
+        return dateFormat(d, 'p', locale);
+      case 'day':
+        return dateFormat(d, 'MMM d', locale);
+      case 'month':
+        return dateFormat(d, 'MMM', locale);
+      case 'year':
+        return dateFormat(d, 'YYY', locale);
+      default:
+        return label;
+    }
+  };
+}
+
+export function renderStatusTooltipPopup(unit, locale) {
+  return (setTooltipPopup, model) => {
+    const { opacity, labelColors, dataPoints } = model.tooltip;
+
+    if (!dataPoints?.length || !opacity) {
+      setTooltipPopup(null);
+      return;
+    }
+
+    const formats = {
+      millisecond: 'T',
+      second: 'pp',
+      minute: 'p',
+      hour: 'h:mm aaa - PP',
+      day: 'PPPP',
+      week: 'PPPP',
+      month: 'LLLL yyyy',
+      quarter: 'qqq',
+      year: 'yyyy',
+    };
+
+    setTooltipPopup(
+      <>
+        <div>{dateFormat(new Date(dataPoints[0].raw.x), formats[unit], locale)}</div>
+        <div>
+          <StatusLight color={labelColors?.[0]?.backgroundColor}>
+            {formatLongNumber(dataPoints[0].raw.y)} {dataPoints[0].dataset.label}
+          </StatusLight>
+        </div>
+      </>,
+    );
+  };
+}

lib/clickhouse.ts

@@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { WebsiteMetricFilter } from './types';
 import { FILTER_COLUMNS } from './constants';
 
@@ -74,7 +74,7 @@ function getEventDataFilterQuery(
   params: any,
 ) {
   const query = filters.reduce((ac, cv, i) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 
@@ -121,13 +121,36 @@ function getFilterQuery(filters = {}, params = {}) {
   return query.join('\n');
 }
 
+function getFunnelQuery(urls: string[]): {
+  columnsQuery: string;
+  conditionQuery: string;
+  urlParams: { [key: string]: string };
+} {
+  return urls.reduce(
+    (pv, cv, i) => {
+      pv.columnsQuery += `\n,url_path = {url${i}:String}${
+        i > 0 && urls[i - 1] ? ` AND referrer_path = {url${i - 1}:String}` : ''
+      }`;
+      pv.conditionQuery += `${i > 0 ? ',' : ''} {url${i}:String}`;
+      pv.urlParams[`url${i}`] = cv;
+
+      return pv;
+    },
+    {
+      columnsQuery: '',
+      conditionQuery: '',
+      urlParams: {},
+    },
+  );
+}
+
 function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
   return {
     filterQuery: getFilterQuery(filters, params),
   };
 }
 
-async function rawQuery(query, params = {}) {
+async function rawQuery<T>(query, params = {}): Promise<T> {
   if (process.env.LOG_QUERY) {
     log('QUERY:\n', query);
     log('PARAMETERS:\n', params);
@@ -135,7 +158,7 @@ async function rawQuery(query, params = {}) {
 
   await connect();
 
-  return clickhouse.query(query, { params }).toPromise();
+  return clickhouse.query(query, { params }).toPromise() as Promise<T>;
 }
 
 async function findUnique(data) {
@@ -168,6 +191,7 @@ export default {
   getDateFormat,
   getBetweenDates,
   getFilterQuery,
+  getFunnelQuery,
   getEventDataFilterQuery,
   parseFilters,
   findUnique,

lib/constants.ts

@@ -18,6 +18,7 @@ export const DEFAULT_THEME = 'light';
 export const DEFAULT_ANIMATION_DURATION = 300;
 export const DEFAULT_DATE_RANGE = '24hour';
 export const DEFAULT_WEBSITE_LIMIT = 10;
+export const DEFAULT_CREATED_AT = '2000-01-01';
 
 export const REALTIME_RANGE = 30;
 export const REALTIME_INTERVAL = 5000;
@@ -42,6 +43,11 @@ export const SESSION_COLUMNS = [
   'city',
 ];
 
+export const COLLECTION_TYPE = {
+  event: 'event',
+  identify: 'identify',
+};
+
 export const FILTER_COLUMNS = {
   url: 'url_path',
   referrer: 'referrer_domain',
@@ -56,7 +62,7 @@ export const EVENT_TYPE = {
   customEvent: 2,
 } as const;
 
-export const EVENT_DATA_TYPE = {
+export const DATA_TYPE = {
   string: 1,
   number: 2,
   boolean: 3,
@@ -64,6 +70,14 @@ export const EVENT_DATA_TYPE = {
   array: 5,
 } as const;
 
+export const DATA_TYPES = {
+  [DATA_TYPE.string]: 'string',
+  [DATA_TYPE.number]: 'number',
+  [DATA_TYPE.boolean]: 'boolean',
+  [DATA_TYPE.date]: 'date',
+  [DATA_TYPE.array]: 'array',
+};
+
 export const KAFKA_TOPIC = {
   event: 'event',
   eventData: 'event_data',
@@ -148,6 +162,8 @@ export const DOMAIN_REGEX =
   /^(localhost(:[1-9]\d{0,4})?|((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9-]+(-[a-z0-9-]+)*\.)+(xn--)?[a-z0-9-]{2,63})$/;
 
 
+export const SHARE_ID_REGEX = /^[a-zA-Z0-9]{16}$/;
+
 export const DESKTOP_SCREEN_WIDTH = 1920;
 export const LAPTOP_SCREEN_WIDTH = 1024;
 export const MOBILE_SCREEN_WIDTH = 479;

lib/date.js

@@ -40,20 +40,27 @@ export function getLocalTime(t) {
 
 export function parseDateRange(value, locale = 'en-US') {
   if (typeof value === 'object') {
-    const { startDate, endDate } = value;
+    return value;
+  }
+
+  if (value?.startsWith?.('range')) {
+    const [, startAt, endAt] = value.split(':');
+
+    const startDate = new Date(+startAt);
+    const endDate = new Date(+endAt);
+
     return {
-      ...value,
-      startDate: typeof startDate === 'string' ? parseISO(startDate) : startDate,
-      endDate: typeof endDate === 'string' ? parseISO(endDate) : endDate,
+      ...getDateRangeValues(startDate, endDate),
+      value,
     };
   }
 
   const now = new Date();
   const dateLocale = getDateLocale(locale);
 
-  const match = value.match(/^(?<num>[0-9-]+)(?<unit>hour|day|week|month|year)$/);
+  const match = value?.match?.(/^(?<num>[0-9-]+)(?<unit>hour|day|week|month|year)$/);
 
-  if (!match) return {};
+  if (!match) return null;
 
   const { num, unit } = match.groups;
 

lib/dynamicData.ts

@@ -1,12 +1,12 @@
 import { isValid, parseISO } from 'date-fns';
-import { EVENT_DATA_TYPE } from './constants';
-import { EventDataTypes } from './types';
+import { DATA_TYPE } from './constants';
+import { DynamicDataType } from './types';
 
 export function flattenJSON(
   eventData: { [key: string]: any },
-  keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],
+  keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
   parentKey = '',
-): { key: string; value: any; eventDataType: EventDataTypes }[] {
+): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
   return Object.keys(eventData).reduce(
     (acc, key) => {
       const value = eventData[key];
@@ -25,7 +25,7 @@ export function flattenJSON(
   ).keyValues;
 }
 
-export function getEventDataType(value: any): string {
+export function getDynamicDataType(value: any): string {
   let type: string = typeof value;
 
   if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {
@@ -36,33 +36,34 @@ export function getEventDataType(value: any): string {
 }
 
 function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
-  const type = getEventDataType(value);
+  const type = getDynamicDataType(value);
 
-  let eventDataType = null;
+  let dynamicDataType = null;
 
   switch (type) {
     case 'number':
-      eventDataType = EVENT_DATA_TYPE.number;
+      dynamicDataType = DATA_TYPE.number;
       break;
     case 'string':
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DATA_TYPE.string;
       break;
     case 'boolean':
-      eventDataType = EVENT_DATA_TYPE.boolean;
+      dynamicDataType = DATA_TYPE.boolean;
+      value = value ? 'true' : 'false';
       break;
     case 'date':
-      eventDataType = EVENT_DATA_TYPE.date;
+      dynamicDataType = DATA_TYPE.date;
       break;
     case 'object':
-      eventDataType = EVENT_DATA_TYPE.array;
+      dynamicDataType = DATA_TYPE.array;
       value = JSON.stringify(value);
       break;
     default:
-      eventDataType = EVENT_DATA_TYPE.string;
+      dynamicDataType = DATA_TYPE.string;
       break;
   }
 
-  acc.keyValues.push({ key, value, eventDataType });
+  acc.keyValues.push({ key, value, dynamicDataType });
 }
 
 function getKeyName(key, parentKey) {

lib/middleware.ts

@@ -1,4 +1,10 @@
-import { createMiddleware, unauthorized, badRequest, parseSecureToken } from 'next-basics';
+import {
+  createMiddleware,
+  unauthorized,
+  badRequest,
+  parseSecureToken,
+  tooManyRequest,
+} from 'next-basics';
 import debug from 'debug';
 import cors from 'cors';
 import { validate } from 'uuid';
@@ -30,6 +36,9 @@ export const useSession = createMiddleware(async (req, res, next) => {
 
     (req as any).session = session;
   } catch (e: any) {
+    if (e.message === 'Usage Limit.') {
+      return tooManyRequest(res, e.message);
+    }
     return badRequest(res, e.message);
   }
 

lib/prisma.ts

@@ -1,7 +1,7 @@
 import prisma from '@umami/prisma-client';
 import moment from 'moment-timezone';
 import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
-import { getEventDataType } from './eventData';
+import { getDynamicDataType } from './dynamicData';
 import { FILTER_COLUMNS } from './constants';
 
 const MYSQL_DATE_FORMATS = {
@@ -32,6 +32,18 @@ function toUuid(): string {
   }
 }
 
+function getAddMinutesQuery(field: string, minutes: number) {
+  const db = getDatabaseType(process.env.DATABASE_URL);
+
+  if (db === POSTGRESQL) {
+    return `${field} + interval '${minutes} minute'`;
+  }
+
+  if (db === MYSQL) {
+    return `DATE_ADD(${field}, interval ${minutes} minute)`;
+  }
+}
+
 function getDateQuery(field: string, unit: string, timezone?: string): string {
   const db = getDatabaseType(process.env.DATABASE_URL);
 
@@ -73,7 +85,7 @@ function getEventDataFilterQuery(
   params: any[],
 ) {
   const query = filters.reduce((ac, cv) => {
-    const type = getEventDataType(cv.eventValue);
+    const type = getDynamicDataType(cv.eventValue);
 
     let value = cv.eventValue;
 
@@ -122,6 +134,53 @@ function getFilterQuery(filters = {}, params = []): string {
   return query.join('\n');
 }
 
+function getFunnelQuery(
+  urls: string[],
+  windowMinutes: number,
+): {
+  levelQuery: string;
+  sumQuery: string;
+  urlFilterQuery: string;
+} {
+  const initParamLength = 3;
+
+  return urls.reduce(
+    (pv, cv, i) => {
+      const levelNumber = i + 1;
+      const start = i > 0 ? ',' : '';
+
+      if (levelNumber >= 2) {
+        pv.levelQuery += `\n
+        , level${levelNumber} AS (
+          select cl.*,
+            l0.created_at level_${levelNumber}_created_at,
+            l0.url_path as level_${levelNumber}_url
+          from level${i} cl
+              left join website_event l0
+                  on cl.session_id = l0.session_id
+                  and l0.created_at between cl.level_${i}_created_at 
+                    and ${getAddMinutesQuery(`cl.level_${i}_created_at`, windowMinutes)}
+                  and l0.referrer_path = $${i + initParamLength}
+                  and l0.url_path = $${levelNumber + initParamLength}
+                  and created_at between $2 and $3
+                  and website_id = $1${toUuid()}
+        )`;
+      }
+
+      pv.sumQuery += `\n${start}SUM(CASE WHEN level_${levelNumber}_url is not null THEN 1 ELSE 0 END) AS level${levelNumber}`;
+
+      pv.urlFilterQuery += `\n${start}$${levelNumber + initParamLength} `;
+
+      return pv;
+    },
+    {
+      levelQuery: '',
+      sumQuery: '',
+      urlFilterQuery: '',
+    },
+  );
+}
+
 function parseFilters(
   filters: { [key: string]: any } = {},
   params = [],
@@ -152,9 +211,11 @@ async function rawQuery(query: string, params: never[] = []): Promise<any> {
 
 export default {
   ...prisma,
+  getAddMinutesQuery,
   getDateQuery,
   getTimestampInterval,
   getFilterQuery,
+  getFunnelQuery,
   getEventDataFilterQuery,
   toUuid,
   parseFilters,

lib/session.ts

@@ -1,10 +1,10 @@
-import clickhouse from 'lib/clickhouse';
 import { secret, uuid } from 'lib/crypto';
 import { getClientInfo, getJsonBody } from 'lib/detect';
 import { parseToken } from 'next-basics';
 import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
 import { createSession } from 'queries';
 import { validate } from 'uuid';
+import cache from './cache';
 import { loadSession, loadWebsite } from './query';
 
 export async function findSession(req: NextApiRequestCollect) {
@@ -21,6 +21,8 @@ export async function findSession(req: NextApiRequestCollect) {
     const result = await parseToken(cacheToken, secret());
 
     if (result) {
+      await checkUserBlock(result?.ownerId);
+
       return result;
     }
   }
@@ -39,27 +41,12 @@ export async function findSession(req: NextApiRequestCollect) {
     throw new Error(`Website not found: ${websiteId}.`);
   }
 
+  await checkUserBlock(website.userId);
+
   const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =
     await getClientInfo(req, payload);
-  const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
-  // Clickhouse does not require session lookup
-  if (clickhouse.enabled) {
-    return {
-      id: sessionId,
-      websiteId,
-      hostname,
-      browser,
-      os,
-      device,
-      screen,
-      language,
-      country,
-      subdivision1,
-      subdivision2,
-      city,
-    };
-  }
+  const sessionId = uuid(websiteId, hostname, ip, userAgent);
 
   // Find session
   let session = await loadSession(sessionId);
@@ -88,5 +75,13 @@ export async function findSession(req: NextApiRequestCollect) {
     }
   }
 
-  return session;
+  return { ...session, ownerId: website.userId };
+}
+
+async function checkUserBlock(userId: string) {
+  if (process.env.ENABLE_BLOCKER && (await cache.fetchUserBlock(userId))) {
+    await cache.incrementUserBlock(userId);
+
+    throw new Error('Usage Limit.');
+  }
 }

lib/types.ts

@@ -1,18 +1,20 @@
 import { NextApiRequest } from 'next';
-import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
+import { COLLECTION_TYPE, DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';
 
 type ObjectValues<T> = T[keyof T];
 
-export type Roles = ObjectValues<typeof ROLES>;
+export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;
 
-export type EventTypes = ObjectValues<typeof EVENT_TYPE>;
+export type Role = ObjectValues<typeof ROLES>;
 
-export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;
+export type EventType = ObjectValues<typeof EVENT_TYPE>;
 
-export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;
+export type DynamicDataType = ObjectValues<typeof DATA_TYPE>;
 
-export interface EventData {
-  [key: string]: number | string | EventData | number[] | string[] | EventData[];
+export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;
+
+export interface DynamicData {
+  [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];
 }
 
 export interface Auth {