Merge branch 'dev' of https://github.com/umami-software/umami into feat/um-376-retention-report

queries/analytics/events/getEventMetrics.ts

@@ -47,11 +47,12 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite
     order by 2
     `,
     {
+      ...filters,
       websiteId,
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.customEvent,
-      ...filters,
+      domain: website.domain,
     },
   );
 }
@@ -82,6 +83,7 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.customEvent,
+      domain: website.domain,
     },
   );
 }

queries/analytics/pageviews/getPageviewMetrics.ts

@@ -34,22 +34,6 @@ async function relationalQuery(
   const { startDate, endDate, filters = {}, column } = criteria;
   const { rawQuery, parseFilters } = prisma;
   const website = await loadWebsite(websiteId);
-  const params: any = {
-    websiteId,
-    startDate: maxDate(startDate, website.resetAt),
-    endDate,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
-    ...filters,
-  };
-
-  let excludeDomain = '';
-
-  if (column === 'referrer_domain') {
-    excludeDomain =
-      'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)';
-
-    params.domain = website.domain;
-  }
 
   const { filterQuery, joinSession } = parseFilters(filters);
 
@@ -61,13 +45,19 @@ async function relationalQuery(
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
       and event_type = {{eventType}}
-      ${excludeDomain}
       ${filterQuery}
     group by 1
     order by 2 desc
     limit 100
     `,
-    params,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
   );
 }
 
@@ -83,22 +73,8 @@ async function clickhouseQuery(
   const { startDate, endDate, filters = {}, column } = criteria;
   const { rawQuery, parseFilters } = clickhouse;
   const website = await loadWebsite(websiteId);
-  const params = {
-    websiteId,
-    startDate: maxDate(startDate, website.resetAt),
-    endDate,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
-    domain: undefined,
-  };
 
-  let excludeDomain = '';
-
-  if (column === 'referrer_domain') {
-    excludeDomain = 'and referrer_domain != {domain:String}';
-    params.domain = website.domain;
-  }
-
-  const { filterQuery } = parseFilters(filters, params);
+  const { filterQuery } = parseFilters(filters);
 
   return rawQuery(
     `
@@ -107,12 +83,18 @@ async function clickhouseQuery(
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime} and {endDate:DateTime}
       and event_type = {eventType:UInt32}
-      ${excludeDomain}
       ${filterQuery}
     group by x
     order by y desc
     limit 100
     `,
-    params,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
   );
 }

queries/analytics/pageviews/getPageviewStats.ts

@@ -10,9 +10,19 @@ export interface PageviewStatsCriteria {
   endDate: Date;
   timezone?: string;
   unit?: string;
-  count?: string;
-  filters: object;
-  sessionKey?: string;
+  filters: {
+    url?: string;
+    referrer?: string;
+    title?: string;
+    browser?: string;
+    os?: string;
+    device?: string;
+    screen?: string;
+    language?: string;
+    country?: string;
+    region?: string;
+    city?: string;
+  };
 }
 
 export async function getPageviewStats(
@@ -25,15 +35,7 @@ export async function getPageviewStats(
 }
 
 async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) {
-  const {
-    startDate,
-    endDate,
-    timezone = 'utc',
-    unit = 'day',
-    count = '*',
-    filters = {},
-    sessionKey = 'session_id',
-  } = criteria;
+  const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
   const { getDateQuery, parseFilters, rawQuery } = prisma;
   const website = await loadWebsite(websiteId);
   const { filterQuery, joinSession } = parseFilters(filters);
@@ -42,7 +44,7 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
     `
     select
       ${getDateQuery('website_event.created_at', unit, timezone)} x,
-      count(${count !== '*' ? `${count}${sessionKey}` : count}) y
+      count(*) y
     from website_event
       ${joinSession}
     where website_event.website_id = {{websiteId::uuid}}
@@ -52,24 +54,18 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
     group by 1
     `,
     {
+      ...filters,
       websiteId,
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
-      ...filters,
+      domain: website.domain,
     },
   );
 }
 
 async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) {
-  const {
-    startDate,
-    endDate,
-    timezone = 'UTC',
-    unit = 'day',
-    count = '*',
-    filters = {},
-  } = criteria;
+  const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
   const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
   const website = await loadWebsite(websiteId);
   const { filterQuery } = parseFilters(filters);
@@ -82,7 +78,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
     from (
       select 
         ${getDateQuery('created_at', unit, timezone)} as t,
-        count(${count !== '*' ? 'distinct session_id' : count}) as y
+        count(*) as y
       from website_event
       where website_id = {websiteId:UUID}
         and created_at between {startDate:DateTime} and {endDate:DateTime}
@@ -98,6 +94,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }

queries/analytics/reports/getInsights.ts

@@ -1,11 +1,14 @@
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
+import { maxDate } from 'lib/date';
+import { EVENT_TYPE } from 'lib/constants';
+import { loadWebsite } from 'lib/load';
 
 export interface GetInsightsCriteria {
   startDate: Date;
   endDate: Date;
-  fields: string[];
+  fields: { name: string; type: string; value: string }[];
   filters: string[];
   groups: string[];
 }
@@ -26,7 +29,33 @@ async function relationalQuery(
     y: number;
   }[]
 > {
-  return null;
+  const { startDate, endDate, filters = [] } = criteria;
+  const { parseFilters, rawQuery } = prisma;
+  const website = await loadWebsite(websiteId);
+  const params = {};
+  const { filterQuery, joinSession } = parseFilters(params);
+
+  return rawQuery(
+    `
+    select
+      url_path,
+      count(*) y
+    from website_event
+      ${joinSession}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and website_event.event_type = {{eventType}}
+      ${filterQuery}
+    group by 1
+    `,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: EVENT_TYPE.pageView,
+    },
+  );
 }
 
 async function clickhouseQuery(
@@ -38,5 +67,54 @@ async function clickhouseQuery(
     y: number;
   }[]
 > {
-  return null;
+  const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria;
+  const { parseFilters, rawQuery } = clickhouse;
+  const website = await loadWebsite(websiteId);
+  const params = {};
+  const { filterQuery } = parseFilters(params);
+
+  const fieldsQuery = parseFields(fields);
+
+  return rawQuery(
+    `
+    select 
+      ${fieldsQuery}
+    from website_event
+    where website_id = {websiteId:UUID}
+      and created_at between {startDate:DateTime} and {endDate:DateTime}
+      and event_type = {eventType:UInt32}
+      ${filterQuery}
+    group by ${fields.map(({ name }) => name).join(',')}
+    order by total desc
+    limit 500
+    `,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: EVENT_TYPE.pageView,
+    },
+  );
+}
+
+function parseFields(fields) {
+  let count = false;
+  let distinct = false;
+
+  const query = fields.reduce((arr, field) => {
+    const { name, value } = field;
+
+    if (!count && value === 'total') {
+      count = true;
+      arr = arr.concat(`count(*) as views`);
+    } else if (!distinct && value === 'unique') {
+      distinct = true;
+      //arr = arr.concat(`count(distinct ${name})`);
+    }
+
+    return arr.concat(name);
+  }, []);
+
+  return query.join(',\n');
 }

queries/analytics/sessions/getSessionMetrics.ts

@@ -1,7 +1,7 @@
 import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
-import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants';
+import { EVENT_TYPE } from 'lib/constants';
 import { loadWebsite } from 'lib/load';
 import { maxDate } from 'lib/date';
 
@@ -28,17 +28,12 @@ async function relationalQuery(
 
   return rawQuery(
     `select ${column} x, count(*) y
-    from session as x
-    where x.session_id in (
-      select website_event.session_id
       from website_event
-        join website 
-          on website_event.website_id = website.website_id
-        ${joinSession}
-      where website.website_id = {{websiteId::uuid}}
+      ${joinSession}
+      where website_event.website_id = {{websiteId::uuid}}
         and website_event.created_at between {{startDate}} and {{endDate}}
       ${filterQuery}
-    )
+    ) as t
     group by 1
     order by 2 desc
     limit 100`,
@@ -64,7 +59,7 @@ async function clickhouseQuery(
     `
     select
       ${column} x, count(distinct session_id) y
-    from website_event as x
+    from website_event
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime} and {endDate:DateTime}
       and event_type = {eventType:UInt32}

queries/analytics/sessions/getSessionStats.ts

@@ -0,0 +1,100 @@
+import clickhouse from 'lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+import { EVENT_TYPE } from 'lib/constants';
+import { loadWebsite } from 'lib/load';
+import { maxDate } from 'lib/date';
+
+export interface SessionStatsCriteria {
+  startDate: Date;
+  endDate: Date;
+  timezone?: string;
+  unit?: string;
+  filters: {
+    url?: string;
+    referrer?: string;
+    title?: string;
+    browser?: string;
+    os?: string;
+    device?: string;
+    screen?: string;
+    language?: string;
+    country?: string;
+    region?: string;
+    city?: string;
+  };
+}
+
+export async function getSessionStats(
+  ...args: [websiteId: string, criteria: SessionStatsCriteria]
+) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) {
+  const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
+  const { getDateQuery, parseFilters, rawQuery } = prisma;
+  const website = await loadWebsite(websiteId);
+  const { filterQuery, joinSession } = parseFilters(filters);
+
+  return rawQuery(
+    `
+    select
+      ${getDateQuery('website_event.created_at', unit, timezone)} x,
+      count(distinct website_event.session_id) y
+    from website_event
+      ${joinSession}
+    where website_event.website_id = {{websiteId::uuid}}
+      and website_event.created_at between {{startDate}} and {{endDate}}
+      and event_type = {{eventType}}
+      ${filterQuery}
+    group by 1
+    `,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
+  );
+}
+
+async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) {
+  const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
+  const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
+  const website = await loadWebsite(websiteId);
+  const { filterQuery } = parseFilters(filters);
+
+  return rawQuery(
+    `
+    select
+      ${getDateStringQuery('g.t', unit)} as x, 
+      g.y as y
+    from (
+      select 
+        ${getDateQuery('created_at', unit, timezone)} as t,
+        count(distinct session_id) as y
+      from website_event
+      where website_id = {websiteId:UUID}
+        and created_at between {startDate:DateTime} and {endDate:DateTime}
+        and event_type = {eventType:UInt32}
+        ${filterQuery}
+      group by t
+    ) as g
+    order by t
+    `,
+    {
+      ...filters,
+      websiteId,
+      startDate: maxDate(startDate, website.resetAt),
+      endDate,
+      eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
+    },
+  );
+}

queries/analytics/stats/getWebsiteStats.ts

@@ -51,11 +51,12 @@ async function relationalQuery(
     ) as t
     `,
     {
+      ...filters,
       websiteId,
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
-      ...filters,
+      domain: website.domain,
     },
   );
 }
@@ -97,6 +98,7 @@ async function clickhouseQuery(
       startDate: maxDate(startDate, website.resetAt),
       endDate,
       eventType: EVENT_TYPE.pageView,
+      domain: website.domain,
     },
   );
 }