Merge branch 'dev' into bug/um-362-relational-funnels-query

lib/auth.ts

@@ -13,7 +13,7 @@ import {
 import { getTeamUser } from 'queries';
 import { getTeamWebsite, getTeamWebsiteByTeamMemberId } from 'queries/admin/teamWebsite';
 import { validate } from 'uuid';
-import { loadWebsite } from './query';
+import { loadWebsite } from './load';
 import { Auth } from './types';
 
 const log = debug('umami:auth');

lib/clickhouse.ts

@@ -2,7 +2,6 @@ import { ClickHouse } from 'clickhouse';
 import dateFormat from 'dateformat';
 import debug from 'debug';
 import { CLICKHOUSE } from 'lib/db';
-import { getDynamicDataType } from './dynamicData';
 import { WebsiteMetricFilter } from './types';
 import { FILTER_COLUMNS } from './constants';
 
@@ -62,49 +61,6 @@ function getDateFormat(date) {
   return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
 }
 
-function getBetweenDates(field, startAt, endAt) {
-  return `${field} between ${getDateFormat(startAt)} and ${getDateFormat(endAt)}`;
-}
-
-function getEventDataFilterQuery(
-  filters: {
-    eventKey?: string;
-    eventValue?: string | number | boolean | Date;
-  }[] = [],
-  params: any,
-) {
-  const query = filters.reduce((ac, cv, i) => {
-    const type = getDynamicDataType(cv.eventValue);
-
-    let value = cv.eventValue;
-
-    ac.push(`and (event_key = {eventKey${i}:String}`);
-
-    switch (type) {
-      case 'number':
-        ac.push(`and number_value = {eventValue${i}:UInt64})`);
-        break;
-      case 'string':
-        ac.push(`and string_value = {eventValue${i}:String})`);
-        break;
-      case 'boolean':
-        ac.push(`and string_value = {eventValue${i}:String})`);
-        value = cv ? 'true' : 'false';
-        break;
-      case 'date':
-        ac.push(`and date_value = {eventValue${i}:DateTime('UTC')})`);
-        break;
-    }
-
-    params[`eventKey${i}`] = cv.eventKey;
-    params[`eventValue${i}`] = value;
-
-    return ac;
-  }, []);
-
-  return query.join('\n');
-}
-
 function getFilterQuery(filters = {}, params = {}) {
   const query = Object.keys(filters).reduce((arr, key) => {
     const filter = filters[key];
@@ -173,7 +129,7 @@ function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
   };
 }
 
-async function rawQuery<T>(query, params = {}): Promise<T> {
+async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
   if (process.env.LOG_QUERY) {
     log('QUERY:\n', query);
     log('PARAMETERS:\n', params);
@@ -189,7 +145,7 @@ async function findUnique(data) {
     throw `${data.length} records found when expecting 1.`;
   }
 
-  return data[0] ?? null;
+  return findFirst(data);
 }
 
 async function findFirst(data) {
@@ -212,10 +168,8 @@ export default {
   getDateStringQuery,
   getDateQuery,
   getDateFormat,
-  getBetweenDates,
   getFilterQuery,
   getFunnelQuery,
-  getEventDataFilterQuery,
   parseFilters,
   findUnique,
   findFirst,

lib/constants.ts

@@ -18,7 +18,7 @@ export const DEFAULT_THEME = 'light';
 export const DEFAULT_ANIMATION_DURATION = 300;
 export const DEFAULT_DATE_RANGE = '24hour';
 export const DEFAULT_WEBSITE_LIMIT = 10;
-export const DEFAULT_CREATED_AT = '2000-01-01';
+export const DEFAULT_RESET_DATE = '2000-01-01';
 
 export const REALTIME_RANGE = 30;
 export const REALTIME_INTERVAL = 5000;

lib/crypto.js

@@ -1,5 +1,3 @@
-import crypto from 'crypto';
-import { v4, v5 } from 'uuid';
 import { startOfMonth } from 'date-fns';
 import { hash } from 'next-basics';
 
@@ -12,13 +10,3 @@ export function salt() {
 
   return hash(secret(), ROTATING_SALT);
 }
-
-export function uuid(...args) {
-  if (!args.length) return v4();
-
-  return v5(hash(...args, salt()), v5.DNS);
-}
-
-export function md5(...args) {
-  return crypto.createHash('md5').update(args.join('')).digest('hex');
-}

lib/date.js

@@ -26,10 +26,20 @@ import {
   differenceInCalendarMonths,
   differenceInCalendarYears,
   format,
-  parseISO,
+  max,
+  min,
+  isDate,
 } from 'date-fns';
 import { getDateLocale } from 'lib/lang';
 
+const dateFuncs = {
+  minute: [differenceInMinutes, addMinutes, startOfMinute],
+  hour: [differenceInHours, addHours, startOfHour],
+  day: [differenceInCalendarDays, addDays, startOfDay],
+  month: [differenceInCalendarMonths, addMonths, startOfMonth],
+  year: [differenceInCalendarYears, addYears, startOfYear],
+};
+
 export function getTimezone() {
   return moment.tz.guess();
 }
@@ -43,11 +53,19 @@ export function parseDateRange(value, locale = 'en-US') {
     return value;
   }
 
-  if (value?.startsWith?.('range')) {
-    const [, startAt, endAt] = value.split(':');
+  if (value === 'all') {
+    return {
+      startDate: new Date(0),
+      endDate: new Date(1),
+      value,
+    };
+  }
 
-    const startDate = new Date(+startAt);
-    const endDate = new Date(+endAt);
+  if (value?.startsWith?.('range')) {
+    const [, startTime, endTime] = value.split(':');
+
+    const startDate = new Date(+startTime);
+    const endDate = new Date(+endTime);
 
     return {
       ...getDateRangeValues(startDate, endDate),
@@ -148,17 +166,34 @@ export function parseDateRange(value, locale = 'en-US') {
   }
 }
 
-export function getDateRangeValues(startDate, endDate) {
-  let unit = 'year';
-  if (differenceInHours(endDate, startDate) <= 48) {
-    unit = 'hour';
+export function getAllowedUnits(startDate, endDate) {
+  const units = ['minute', 'hour', 'day', 'month', 'year'];
+  const minUnit = getMinimumUnit(startDate, endDate);
+  const index = units.indexOf(minUnit);
+
+  return index >= 0 ? units.splice(index) : [];
+}
+
+export function getMinimumUnit(startDate, endDate) {
+  if (differenceInMinutes(endDate, startDate) <= 60) {
+    return 'minute';
+  } else if (differenceInHours(endDate, startDate) <= 48) {
+    return 'hour';
   } else if (differenceInCalendarDays(endDate, startDate) <= 90) {
-    unit = 'day';
+    return 'day';
   } else if (differenceInCalendarMonths(endDate, startDate) <= 24) {
-    unit = 'month';
+    return 'month';
   }
 
-  return { startDate: startOfDay(startDate), endDate: endOfDay(endDate), unit };
+  return 'year';
+}
+
+export function getDateRangeValues(startDate, endDate) {
+  return {
+    startDate: startOfDay(startDate),
+    endDate: endOfDay(endDate),
+    unit: getMinimumUnit(startDate, endDate),
+  };
 }
 
 export function getDateFromString(str) {
@@ -174,14 +209,6 @@ export function getDateFromString(str) {
   return new Date(year, month - 1, day);
 }
 
-const dateFuncs = {
-  minute: [differenceInMinutes, addMinutes, startOfMinute],
-  hour: [differenceInHours, addHours, startOfHour],
-  day: [differenceInCalendarDays, addDays, startOfDay],
-  month: [differenceInCalendarMonths, addMonths, startOfMonth],
-  year: [differenceInCalendarYears, addYears, startOfYear],
-};
-
 export function getDateArray(data, startDate, endDate, unit) {
   const arr = [];
   const [diff, add, normalize] = dateFuncs[unit];
@@ -227,3 +254,11 @@ export function dateFormat(date, str, locale = 'en-US') {
     locale: getDateLocale(locale),
   });
 }
+
+export function maxDate(...args) {
+  return max(args.filter(n => isDate(n)));
+}
+
+export function minDate(...args) {
+  return min(args.filter(n => isDate(n)));
+}

lib/db.js

@@ -35,3 +35,7 @@ export async function runQuery(queries) {
     return queries[CLICKHOUSE]();
   }
 }
+
+export function notImplemented() {
+  throw new Error('Not implemented.');
+}

lib/detect.ts

@@ -1,5 +1,5 @@
 import path from 'path';
-import requestIp from 'request-ip';
+import { getClientIp } from 'request-ip';
 import { browserName, detectOS } from 'detect-browser';
 import isLocalhost from 'is-localhost-ip';
 import maxmind from 'maxmind';
@@ -25,7 +25,7 @@ export function getIpAddress(req) {
     return req.headers['cf-connecting-ip'];
   }
 
-  return requestIp.getClientIp(req);
+  return getClientIp(req);
 }
 
 export function getDevice(screen, os) {

lib/kafka.ts

@@ -61,7 +61,7 @@ async function getProducer(): Promise<Producer> {
   return producer;
 }
 
-function getDateFormat(date, format?): string {
+function getDateFormat(date: Date, format?: string): string {
   return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');
 }
 

lib/load.ts

@@ -0,0 +1,51 @@
+import cache from 'lib/cache';
+import { getWebsite, getSession, getUser } from 'queries';
+import { User, Website, Session } from '@prisma/client';
+
+export async function loadWebsite(websiteId: string): Promise<Website> {
+  let website;
+
+  if (cache.enabled) {
+    website = await cache.fetchWebsite(websiteId);
+  } else {
+    website = await getWebsite({ id: websiteId });
+  }
+
+  if (!website || website.deletedAt) {
+    return null;
+  }
+
+  return website;
+}
+
+export async function loadSession(sessionId: string): Promise<Session> {
+  let session;
+
+  if (cache.enabled) {
+    session = await cache.fetchSession(sessionId);
+  } else {
+    session = await getSession({ id: sessionId });
+  }
+
+  if (!session) {
+    return null;
+  }
+
+  return session;
+}
+
+export async function loadUser(userId: string): Promise<User> {
+  let user;
+
+  if (cache.enabled) {
+    user = await cache.fetchUser(userId);
+  } else {
+    user = await getUser({ id: userId });
+  }
+
+  if (!user || user.deletedAt) {
+    return null;
+  }
+
+  return user;
+}

lib/middleware.ts

@@ -73,5 +73,6 @@ export const useAuth = createMiddleware(async (req, res, next) => {
   }
 
   (req as any).auth = { user, token, shareToken, authKey };
+
   next();
 });

lib/prisma.ts

@@ -1,7 +1,6 @@
 import prisma from '@umami/prisma-client';
 import moment from 'moment-timezone';
 import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
-import { getDynamicDataType } from './dynamicData';
 import { FILTER_COLUMNS } from './constants';
 
 const MYSQL_DATE_FORMATS = {
@@ -45,7 +44,7 @@ function getAddMinutesQuery(field: string, minutes: number): string {
 }
 
 function getDateQuery(field: string, unit: string, timezone?: string): string {
-  const db = getDatabaseType(process.env.DATABASE_URL);
+  const db = getDatabaseType();
 
   if (db === POSTGRESQL) {
     if (timezone) {
@@ -65,8 +64,8 @@ function getDateQuery(field: string, unit: string, timezone?: string): string {
   }
 }
 
-function getTimestampInterval(field: string): string {
-  const db = getDatabaseType(process.env.DATABASE_URL);
+function getTimestampIntervalQuery(field: string): string {
+  const db = getDatabaseType();
 
   if (db === POSTGRESQL) {
     return `floor(extract(epoch from max(${field}) - min(${field})))`;
@@ -77,47 +76,6 @@ function getTimestampInterval(field: string): string {
   }
 }
 
-function getEventDataFilterQuery(
-  filters: {
-    eventKey?: string;
-    eventValue?: string | number | boolean | Date;
-  }[],
-  params: any[],
-) {
-  const query = filters.reduce((ac, cv) => {
-    const type = getDynamicDataType(cv.eventValue);
-
-    let value = cv.eventValue;
-
-    ac.push(`and (event_key = $${params.length + 1}`);
-    params.push(cv.eventKey);
-
-    switch (type) {
-      case 'number':
-        ac.push(`and number_value = $${params.length + 1})`);
-        params.push(value);
-        break;
-      case 'string':
-        ac.push(`and string_value = $${params.length + 1})`);
-        params.push(decodeURIComponent(cv.eventValue as string));
-        break;
-      case 'boolean':
-        ac.push(`and string_value = $${params.length + 1})`);
-        params.push(decodeURIComponent(cv.eventValue as string));
-        value = cv ? 'true' : 'false';
-        break;
-      case 'date':
-        ac.push(`and date_value = $${params.length + 1})`);
-        params.push(cv.eventValue);
-        break;
-    }
-
-    return ac;
-  }, []);
-
-  return query.join('\n');
-}
-
 function getFilterQuery(filters = {}, params = []): string {
   const query = Object.keys(filters).reduce((arr, key) => {
     const filter = filters[key];
@@ -196,8 +154,8 @@ function getFunnelQuery(
               and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}
               and we.referrer_path = $${i + initParamLength}
               and we.url_path = $${levelNumber + initParamLength}
-              and we.created_at <= $3
-              and we.website_id = $1${toUuid()}
+              and we.created_at <= {{endDate}}
+              and we.website_id = {{websiteId}}${toUuid()}
         )`;
       }
 
@@ -214,25 +172,31 @@ function getFunnelQuery(
   );
 }
 
-async function rawQuery(query: string, params: never[] = []): Promise<any> {
-  const db = getDatabaseType(process.env.DATABASE_URL);
+async function rawQuery(sql: string, data: object): Promise<any> {
+  const db = getDatabaseType();
+  const params = [];
 
   if (db !== POSTGRESQL && db !== MYSQL) {
     return Promise.reject(new Error('Unknown database.'));
   }
 
-  const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
+  const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => {
+    const [, name, type] = args;
 
-  return prisma.rawQuery(sql, params);
+    params.push(data[name]);
+
+    return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`;
+  });
+
+  return prisma.rawQuery(query, params);
 }
 
 export default {
   ...prisma,
   getAddMinutesQuery,
   getDateQuery,
-  getTimestampInterval,
+  getTimestampIntervalQuery,
   getFilterQuery,
-  getEventDataFilterQuery,
   toUuid,
   parseFilters,
   getFunnelParams,

lib/query.ts

@@ -1,51 +1,31 @@
-import cache from 'lib/cache';
-import { getWebsite, getSession, getUser } from 'queries';
-import { User, Website, Session } from '@prisma/client';
+import { NextApiRequest } from 'next';
+import { getAllowedUnits, getMinimumUnit } from './date';
+import { getWebsiteDateRange } from '../queries';
 
-export async function loadWebsite(websiteId: string): Promise<Website> {
-  let website;
+export async function parseDateRangeQuery(req: NextApiRequest) {
+  const { id: websiteId, startAt, endAt, unit } = req.query;
 
-  if (cache.enabled) {
-    website = await cache.fetchWebsite(websiteId);
-  } else {
-    website = await getWebsite({ id: websiteId });
+  // All-time
+  if (+startAt === 0 && +endAt === 1) {
+    const result = await getWebsiteDateRange(websiteId as string);
+    const { min, max } = result[0];
+    const startDate = new Date(min);
+    const endDate = new Date(max);
+
+    return {
+      startDate,
+      endDate,
+      unit: getMinimumUnit(startDate, endDate),
+    };
   }
 
-  if (!website || website.deletedAt) {
-    return null;
-  }
+  const startDate = new Date(+startAt);
+  const endDate = new Date(+endAt);
+  const minUnit = getMinimumUnit(startDate, endDate);
 
-  return website;
-}
-
-export async function loadSession(sessionId: string): Promise<Session> {
-  let session;
-
-  if (cache.enabled) {
-    session = await cache.fetchSession(sessionId);
-  } else {
-    session = await getSession({ id: sessionId });
-  }
-
-  if (!session) {
-    return null;
-  }
-
-  return session;
-}
-
-export async function loadUser(userId: string): Promise<User> {
-  let user;
-
-  if (cache.enabled) {
-    user = await cache.fetchUser(userId);
-  } else {
-    user = await getUser({ id: userId });
-  }
-
-  if (!user || user.deletedAt) {
-    return null;
-  }
-
-  return user;
+  return {
+    startDate,
+    endDate,
+    unit: (getAllowedUnits(startDate, endDate).includes(unit as string) ? unit : minUnit) as string,
+  };
 }

lib/session.ts

@@ -1,11 +1,11 @@
-import { secret, uuid } from 'lib/crypto';
+import { secret } from 'lib/crypto';
 import { getClientInfo, getJsonBody } from 'lib/detect';
-import { parseToken } from 'next-basics';
+import { parseToken, uuid } from 'next-basics';
 import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';
 import { createSession } from 'queries';
 import { validate } from 'uuid';
 import cache from './cache';
-import { loadSession, loadWebsite } from './query';
+import { loadSession, loadWebsite } from './load';
 
 export async function findSession(req: NextApiRequestCollect) {
   const { payload } = getJsonBody<CollectRequestBody>(req);
@@ -30,7 +30,6 @@ export async function findSession(req: NextApiRequestCollect) {
   // Verify payload
   const { website: websiteId, hostname, screen, language } = payload;
 
-
   // Check the hostname value for legality to eliminate dirty data
   const validHostnameRegex = /^[\w-.]+$/;
   if (!validHostnameRegex.test(hostname)) {

lib/types.ts

@@ -137,3 +137,10 @@ export interface RealtimeUpdate {
   events: any[];
   timestamp: number;
 }
+
+export interface DateRange {
+  startDate: Date;
+  endDate: Date;
+  unit: string;
+  value: string;
+}