Merge branch 'dev' into brian/um-16-clickhouse-support

lib/constants.js

@@ -7,7 +7,8 @@ export const DASHBOARD_CONFIG = 'umami.dashboard';
 export const VERSION_CHECK = 'umami.version-check';
 export const SHARE_TOKEN_HEADER = 'x-umami-share-token';
 export const HOMEPAGE_URL = 'https://umami.is';
-export const VERSION_URL = 'https://github.com/mikecao/umami/releases';
+export const REPO_URL = 'https://github.com/umami-software/umami';
+export const UPDATES_URL = 'https://api.umami.is/v1/updates';
 
 export const DEFAULT_LOCALE = 'en-US';
 export const DEFAULT_THEME = 'light';

lib/db.js

@@ -30,17 +30,7 @@ function logQuery(e) {
 }
 
 function initializePrisma(options) {
-  let prisma;
-
-  if (process.env.NODE_ENV === 'production') {
-    prisma = new PrismaClient(options);
-  } else {
-    if (!global.prisma) {
-      global.prisma = new PrismaClient(options);
-    }
-
-    prisma = global.prisma;
-  }
+  const prisma = new PrismaClient(options);
 
   if (process.env.LOG_QUERY) {
     prisma.$on('query', logQuery);
@@ -55,7 +45,6 @@ function initializeClickhouse() {
   }
 
   const url = new URL(process.env.ANALYTICS_URL);
-
   const database = url.pathname.replace('/', '');
 
   return new ClickHouse({
@@ -74,10 +63,13 @@ function initializeClickhouse() {
   });
 }
 
-const prisma = initializePrisma(options);
-const clickhouse = initializeClickhouse();
+const prisma = global.prisma || initializePrisma(options);
+const clickhouse = global.clickhouse || initializeClickhouse();
 
-console.log('clickhouse1: ', clickhouse);
+if (process.env.NODE_ENV !== 'production') {
+  global.prisma = prisma;
+  global.clickhouse = clickhouse;
+}
 
 export { prisma, clickhouse };
 
@@ -109,18 +101,6 @@ export function getAnalyticsDatabase() {
   return type;
 }
 
-export function getDateStringQuery(data, unit) {
-  const db = getDatabase();
-
-  if (db === POSTGRESQL) {
-    return `to_char(${data}, '${POSTGRESQL_DATE_FORMATS[unit]}')`;
-  }
-
-  if (db === MYSQL) {
-    return `DATE_FORMAT(${data}, '${MYSQL_DATE_FORMATS[unit]}')`;
-  }
-}
-
 export function getDateStringQueryClickhouse(data, unit) {
   return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
 }
@@ -130,19 +110,19 @@ export function getDateQuery(field, unit, timezone) {
 
   if (db === POSTGRESQL) {
     if (timezone) {
-      return `date_trunc('${unit}', ${field} at time zone '${timezone}')`;
+      return `to_char(date_trunc('${unit}', ${field} at time zone '${timezone}'), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
     }
-    return `date_trunc('${unit}', ${field})`;
+    return `to_char(date_trunc('${unit}', ${field}), '${POSTGRESQL_DATE_FORMATS[unit]}')`;
   }
 
   if (db === MYSQL) {
     if (timezone) {
       const tz = moment.tz(timezone).format('Z');
 
-      return `convert_tz(${field},'+00:00','${tz}')`;
+      return `date_format(convert_tz(${field},'+00:00','${tz}'), '${MYSQL_DATE_FORMATS[unit]}')`;
     }
 
-    return `${field}`;
+    return `date_format(${field}, '${MYSQL_DATE_FORMATS[unit]}')`;
   }
 }
 
@@ -174,7 +154,7 @@ export function getTimestampInterval(maxColumn, minColumn) {
   }
 }
 
-export function getFilterQuery(table, filters = {}, params = []) {
+export function getFilterQuery(table, column, filters = {}, params = []) {
   const query = Object.keys(filters).reduce((arr, key) => {
     const value = filters[key];
 
@@ -230,7 +210,7 @@ export function getFilterQuery(table, filters = {}, params = []) {
   return query.join('\n');
 }
 
-export function parseFilters(table, filters = {}, params = [], sessionKey = 'session_id') {
+export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
   const { domain, url, event_url, referrer, os, browser, device, country, event_type } = filters;
 
   const pageviewFilters = { domain, url, referrer };
@@ -246,9 +226,9 @@ export function parseFilters(table, filters = {}, params = [], sessionKey = 'ses
       os || browser || device || country
         ? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`
         : '',
-    pageviewQuery: getFilterQuery('pageview', pageviewFilters, params),
-    sessionQuery: getFilterQuery('session', sessionFilters, params),
-    eventQuery: getFilterQuery('event', eventFilters, params),
+    pageviewQuery: getFilterQuery('pageview', column, pageviewFilters, params),
+    sessionQuery: getFilterQuery('session', column, sessionFilters, params),
+    eventQuery: getFilterQuery('event', column, eventFilters, params),
   };
 }
 

lib/filters.js

@@ -1,14 +1,10 @@
-import { removeTrailingSlash, removeWWW, getDomainName } from './url';
+import { removeWWW } from './url';
 
-export const urlFilter = (data, { raw }) => {
+export const urlFilter = data => {
   const isValidUrl = url => {
     return url !== '' && url !== null && !url.startsWith('#');
   };
 
-  if (raw) {
-    return data.filter(({ x }) => isValidUrl(x));
-  }
-
   const cleanUrl = url => {
     try {
       const { pathname, search } = new URL(url, location.origin);
@@ -44,66 +40,26 @@ export const urlFilter = (data, { raw }) => {
   return Object.keys(map).map(key => ({ x: key, y: map[key] }));
 };
 
-export const refFilter = (data, { domain, domainOnly, raw }) => {
-  const domainName = getDomainName(domain);
-  const regex = new RegExp(`http[s]?://([a-z0-9-]+\\.)*${domainName}`);
+export const refFilter = data => {
   const links = {};
 
-  const isValidRef = referrer => {
-    return (
-      referrer !== '' && referrer !== null && !referrer.startsWith('/') && !referrer.startsWith('#')
-    );
-  };
-
-  const cleanUrl = url => {
-    try {
-      const { hostname, origin, pathname, searchParams, protocol } = new URL(url);
-
-      if (regex.test(url)) {
-        return null;
-      }
-
-      if (domainOnly && hostname) {
-        return removeWWW(hostname);
-      }
-
-      if (!origin || origin === 'null') {
-        return `${protocol}${removeTrailingSlash(pathname)}`;
-      }
-
-      if (protocol.startsWith('http')) {
-        const path = removeTrailingSlash(pathname);
-        const referrer = searchParams.get('referrer');
-        const query = referrer ? `?referrer=${referrer}` : '';
-
-        return removeTrailingSlash(`${removeWWW(hostname)}${path}`) + query;
-      }
-
-      return null;
-    } catch {
-      return null;
-    }
-  };
-
-  if (raw) {
-    return data.filter(({ x }) => isValidRef(x) && !regex.test(x));
-  }
-
   const map = data.reduce((obj, { x, y }) => {
-    if (!isValidRef(x)) {
-      return obj;
+    let id;
+
+    try {
+      const url = new URL(x);
+
+      id = removeWWW(url.hostname) || url.href;
+    } catch {
+      id = '';
     }
 
-    const url = cleanUrl(x);
+    links[id] = x;
 
-    links[url] = x;
-
-    if (url) {
-      if (!obj[url]) {
-        obj[url] = y;
-      } else {
-        obj[url] += y;
-      }
+    if (!obj[id]) {
+      obj[id] = y;
+    } else {
+      obj[id] += y;
     }
 
     return obj;

lib/url.js

@@ -6,14 +6,6 @@ export function removeWWW(url) {
   return url && url.length > 1 && url.startsWith('www.') ? url.slice(4) : url;
 }
 
-export function getDomainName(str) {
-  try {
-    return new URL(str).hostname;
-  } catch (e) {
-    return str;
-  }
-}
-
 export function getQueryString(params = {}) {
   const map = Object.keys(params).reduce((arr, key) => {
     if (params[key] !== undefined) {