convert analytics queries

lib/constants.js

@@ -64,6 +64,7 @@ export const EVENT_COLORS = [
   '#ffec16',
 ];
 
+export const RELATIONAL = 'relational';
 export const POSTGRESQL = 'postgresql';
 export const MYSQL = 'mysql';
 export const CLICKHOUSE = 'clickhouse';

lib/db.js

@@ -7,8 +7,10 @@ import {
   POSTGRESQL,
   POSTGRESQL_DATE_FORMATS,
   CLICKHOUSE,
+  RELATIONAL,
 } from 'lib/constants';
 import moment from 'moment-timezone';
+import { CLICKHOUSE_DATE_FORMATS } from './constants';
 
 BigInt.prototype.toJSON = function () {
   return Number(this);
@@ -48,19 +50,48 @@ function initializePrisma(options) {
 }
 
 function initializeClickhouse() {
-  if (process.env.ANALYTICS_URL) {
+  if (!process.env.ANALYTICS_URL) {
     return null;
   }
 
+  const url = new URL(process.env.ANALYTICS_URL);
+
+  const database = url.pathname.replace('/', '');
+
   return new ClickHouse({
-    url: process.env.ANALYTICS_URL,
+    url: url.hostname,
+    port: Number(url.port),
+    basicAuth: url.password
+      ? {
+          username: url.username || 'default',
+          password: url.password,
+        }
+      : null,
     format: 'json',
+    config: {
+      database,
+    },
   });
+
+  // return new ClickHouse({
+  //   url: 'http://164.92.95.2',
+  //   port: 8123,
+  //   basicAuth: {
+  //     username: 'default',
+  //     password: 'shhhthisissupersecret!',
+  //   },
+  //   format: 'json',
+  //   config: {
+  //     database: 'umami_dev',
+  //   },
+  // });
 }
 
 const prisma = initializePrisma(options);
 const clickhouse = initializeClickhouse();
 
+console.log('clickhouse1: ', clickhouse);
+
 export { prisma, clickhouse };
 
 export function getDatabase() {
@@ -103,6 +134,10 @@ export function getDateStringQuery(data, unit) {
   }
 }
 
+export function getDateStringQueryClickhouse(data, unit) {
+  return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
+}
+
 export function getDateQuery(field, unit, timezone) {
   const db = getDatabase();
 
@@ -124,15 +159,31 @@ export function getDateQuery(field, unit, timezone) {
   }
 }
 
-export function getTimestampInterval(field) {
+export function getDateQueryClickhouse(field, unit, timezone) {
+  if (timezone) {
+    return `date_trunc('${unit}', ${field},'${timezone}')`;
+  }
+  return `date_trunc('${unit}', ${field})`;
+}
+
+export function getDateFormatClickhouse(date) {
+  return `parseDateTimeBestEffort('${date.toUTCString()}')`;
+}
+
+export function getBetweenDatesClickhouse(field, start_at, end_at) {
+  return `${field} between ${getDateFormatClickhouse(start_at)} 
+    and ${getDateFormatClickhouse(end_at)}`;
+}
+
+export function getTimestampInterval(maxColumn, minColumn) {
   const db = getDatabase();
 
   if (db === POSTGRESQL) {
-    return `floor(extract(epoch from max(${field}) - min(${field})))`;
+    return `floor(extract(epoch from max(${maxColumn}) - min(${minColumn})))`;
   }
 
   if (db === MYSQL) {
-    return `floor(unix_timestamp(max(${field})) - unix_timestamp(min(${field})))`;
+    return `floor(unix_timestamp(max(${maxColumn})) - unix_timestamp(min(${minColumn})))`;
   }
 }
 
@@ -149,6 +200,7 @@ export function getFilterQuery(table, filters = {}, params = []) {
         if (table === 'pageview' || table === 'event') {
           arr.push(`and ${table}.${key}=$${params.length + 1}`);
           params.push(decodeURIComponent(value));
+          console.log(params);
         }
         break;
 
@@ -213,6 +265,22 @@ export function parseFilters(table, filters = {}, params = []) {
   };
 }
 
+export function replaceQueryClickhouse(string, params = []) {
+  let formattedString = string;
+
+  params.forEach((a, i) => {
+    let replace = a;
+
+    if (typeof a === 'string' || a instanceof String) {
+      replace = `'${replace}'`;
+    }
+
+    formattedString = formattedString.replace(`$${i + 1}`, replace);
+  });
+
+  return formattedString;
+}
+
 export async function runQuery(query) {
   return query.catch(e => {
     throw e;
@@ -231,14 +299,24 @@ export async function rawQuery(query, params = []) {
   return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
 }
 
-export function runAnalyticsQuery(relational, clickhouse) {
+export async function rawQueryClickhouse(query, params = [], debug = false) {
+  let formattedQuery = replaceQueryClickhouse(query, params);
+
+  if (debug || process.env.LOG_QUERY) {
+    console.log(formattedQuery);
+  }
+
+  return clickhouse.query(formattedQuery).toPromise();
+}
+
+export async function runAnalyticsQuery(queries) {
   const db = getAnalyticsDatabase();
 
   if (db === POSTGRESQL || db === MYSQL) {
-    return relational();
+    return queries[`${RELATIONAL}`]();
   }
 
   if (db === CLICKHOUSE) {
-    return runQuery(clickhouse());
+    return queries[`${CLICKHOUSE}`]();
   }
 }