Updated pixel/link endpoints. Added name to rawQuery.

src/app/(collect)/p/[slug]/route.ts

@@ -1,6 +1,10 @@
+export const dynamic = 'force-dynamic';
+
 import { NextResponse } from 'next/server';
 import { notFound } from '@/lib/response';
+import redis from '@/lib/redis';
 import { findPixel } from '@/queries/prisma';
+import { Pixel } from '@/generated/prisma/client';
 import { POST } from '@/app/api/send/route';
 
 const image = Buffer.from('R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw', 'base64');
@@ -8,14 +12,34 @@ const image = Buffer.from('R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICR
 export async function GET(request: Request, { params }: { params: Promise<{ slug: string }> }) {
   const { slug } = await params;
 
-  const pixel = await findPixel({
-    where: {
-      slug,
-    },
-  });
+  let pixel: Pixel;
 
-  if (!pixel) {
-    return notFound();
+  if (redis.enabled) {
+    pixel = await redis.client.fetch(
+      `pixel:${slug}`,
+      async () => {
+        return findPixel({
+          where: {
+            slug,
+          },
+        });
+      },
+      86400,
+    );
+
+    if (!pixel) {
+      return notFound();
+    }
+  } else {
+    pixel = await findPixel({
+      where: {
+        slug,
+      },
+    });
+
+    if (!pixel) {
+      return notFound();
+    }
   }
 
   const payload = {
@@ -23,7 +47,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ slug
     payload: {
       pixel: pixel.id,
       url: request.url,
-      referrer: request.referrer,
+      referrer: request.headers.get('referer'),
     },
   };
 
@@ -33,13 +57,12 @@ export async function GET(request: Request, { params }: { params: Promise<{ slug
     body: JSON.stringify(payload),
   });
 
-  const res = await POST(req);
+  await POST(req);
 
   return new NextResponse(image, {
     headers: {
       'Content-Type': 'image/gif',
       'Content-Length': image.length.toString(),
-      'x-umami-collect': JSON.stringify(res),
     },
   });
 }

src/app/(collect)/q/[slug]/route.ts

@@ -1,19 +1,43 @@
+export const dynamic = 'force-dynamic';
+
 import { NextResponse } from 'next/server';
 import { notFound } from '@/lib/response';
 import { findLink } from '@/queries/prisma';
 import { POST } from '@/app/api/send/route';
+import { Link } from '@/generated/prisma/client';
+import redis from '@/lib/redis';
 
 export async function GET(request: Request, { params }: { params: Promise<{ slug: string }> }) {
   const { slug } = await params;
 
-  const link = await findLink({
-    where: {
-      slug,
-    },
-  });
+  let link: Link;
 
-  if (!link) {
-    return notFound();
+  if (redis.enabled) {
+    link = await redis.client.fetch(
+      `link:${slug}`,
+      async () => {
+        return findLink({
+          where: {
+            slug,
+          },
+        });
+      },
+      86400,
+    );
+
+    if (!link) {
+      return notFound();
+    }
+  } else {
+    link = await findLink({
+      where: {
+        slug,
+      },
+    });
+
+    if (!link) {
+      return notFound();
+    }
   }
 
   const payload = {
@@ -21,7 +45,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ slug
     payload: {
       link: link.id,
       url: request.url,
-      referrer: request.referrer,
+      referrer: request.headers.get('referer'),
     },
   };
 

src/app/api/websites/[websiteId]/metrics/expanded/route.ts

@@ -1,5 +1,5 @@
 import { canViewWebsite } from '@/permissions';
-import { EVENT_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
 import { getQueryFilters, parseRequest } from '@/lib/request';
 import { badRequest, json, unauthorized } from '@/lib/response';
 import { dateRangeParams, filterParams, searchParams } from '@/lib/schema';
@@ -50,21 +50,21 @@ export async function GET(
   }
 
   if (EVENT_COLUMNS.includes(type)) {
-    let data;
+    const column = FILTER_COLUMNS[type] || type;
 
-    if (type === 'event') {
-      data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters);
-    } else {
-      data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters);
+    if (column === 'event_name') {
+      filters.eventType = EVENT_TYPE.customEvent;
     }
 
-    return json(data);
+    if (type === 'event') {
+      return json(await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters));
+    } else {
+      return json(await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters));
+    }
   }
 
   if (type === 'channel') {
-    const data = await getChannelExpandedMetrics(websiteId, filters);
-
-    return json(data);
+    return json(await getChannelExpandedMetrics(websiteId, filters));
   }
 
   return badRequest();

src/app/api/websites/[websiteId]/metrics/route.ts

@@ -1,5 +1,5 @@
 import { canViewWebsite } from '@/permissions';
-import { EVENT_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
 import { getQueryFilters, parseRequest } from '@/lib/request';
 import { badRequest, json, unauthorized } from '@/lib/response';
 import {
@@ -50,21 +50,21 @@ export async function GET(
   }
 
   if (EVENT_COLUMNS.includes(type)) {
-    let data;
+    const column = FILTER_COLUMNS[type] || type;
 
-    if (type === 'event') {
-      data = await getEventMetrics(websiteId, { type, limit, offset }, filters);
-    } else {
-      data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters);
+    if (column === 'event_name') {
+      filters.eventType = EVENT_TYPE.customEvent;
     }
 
-    return json(data);
+    if (type === 'event') {
+      return json(await getEventMetrics(websiteId, { type, limit, offset }, filters));
+    } else {
+      return json(await getPageviewMetrics(websiteId, { type, limit, offset }, filters));
+    }
   }
 
   if (type === 'channel') {
-    const data = await getChannelMetrics(websiteId, filters);
-
-    return json(data);
+    return json(await getChannelMetrics(websiteId, filters));
   }
 
   return badRequest();

src/lib/clickhouse.ts

@@ -206,9 +206,10 @@ async function pagedRawQuery(
 async function rawQuery<T = unknown>(
   query: string,
   params: Record<string, unknown> = {},
+  name?: string,
 ): Promise<T> {
   if (process.env.LOG_QUERY) {
-    log({ query, params });
+    log({ query, params, name });
   }
 
   await connect();

src/lib/prisma.ts

@@ -164,10 +164,11 @@ function parseFilters(filters: Record<string, any>, options?: QueryOptions) {
   };
 }
 
-async function rawQuery(sql: string, data: object): Promise<any> {
+async function rawQuery(sql: string, data: Record<string, any>, name?: string): Promise<any> {
   if (process.env.LOG_QUERY) {
     log('QUERY:\n', sql);
     log('PARAMETERS:\n', data);
+    log('NAME:\n', name);
   }
   const params = [];
   const schema = getSchema();

src/queries/sql/events/getEventMetrics.ts

@@ -4,6 +4,8 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getEventMetrics';
+
 export interface EventMetricParameters {
   type: string;
   limit?: string;
@@ -58,6 +60,7 @@ async function relationalQuery(
     offset ${offset}
     `,
     { ...queryParams, ...parameters },
+    FUNCTION_NAME,
   );
 }
 
@@ -89,5 +92,6 @@ async function clickhouseQuery(
      offset ${offset}
     `,
     { ...queryParams, ...parameters },
+    FUNCTION_NAME,
   );
 }

src/queries/sql/events/saveEvent.ts

@@ -1,5 +1,5 @@
 import { uuid } from '@/lib/crypto';
-import { EVENT_NAME_LENGTH, URL_LENGTH, EVENT_TYPE, PAGE_TITLE_LENGTH } from '@/lib/constants';
+import { EVENT_NAME_LENGTH, URL_LENGTH, PAGE_TITLE_LENGTH } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import clickhouse from '@/lib/clickhouse';
 import kafka from '@/lib/kafka';
@@ -66,10 +66,9 @@ async function relationalQuery({
   websiteId,
   sessionId,
   visitId,
-  createdAt,
   eventType,
+  createdAt,
   pageTitle,
-  tag,
   hostname,
   urlPath,
   urlQuery,
@@ -78,6 +77,7 @@ async function relationalQuery({
   referrerDomain,
   eventName,
   eventData,
+  tag,
   utmSource,
   utmMedium,
   utmCampaign,
@@ -154,9 +154,16 @@ async function clickhouseQuery({
   websiteId,
   sessionId,
   visitId,
-  distinctId,
+  eventType,
   createdAt,
   pageTitle,
+  hostname,
+  urlPath,
+  urlQuery,
+  referrerPath,
+  referrerQuery,
+  referrerDomain,
+  distinctId,
   browser,
   os,
   device,
@@ -165,15 +172,9 @@ async function clickhouseQuery({
   country,
   region,
   city,
-  tag,
-  hostname,
-  urlPath,
-  urlQuery,
-  referrerPath,
-  referrerQuery,
-  referrerDomain,
   eventName,
   eventData,
+  tag,
   utmSource,
   utmMedium,
   utmCampaign,
@@ -215,7 +216,7 @@ async function clickhouseQuery({
     ttclid: ttclid,
     li_fat_id: lifatid,
     twclid: twclid,
-    event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
+    event_type: eventType,
     event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,
     tag: tag,
     distinct_id: distinctId,

src/queries/sql/getActiveVisitors.ts

@@ -3,6 +3,8 @@ import prisma from '@/lib/prisma';
 import clickhouse from '@/lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
 
+const FUNCTION_NAME = 'getActiveVisitors';
+
 export async function getActiveVisitors(...args: [websiteId: string]) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -22,6 +24,7 @@ async function relationalQuery(websiteId: string) {
     and created_at >= {{startDate}}
     `,
     { websiteId, startDate },
+    FUNCTION_NAME,
   );
 
   return result?.[0] ?? null;
@@ -40,6 +43,7 @@ async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {
       and created_at >= {startDate:DateTime64}
     `,
     { websiteId, startDate },
+    FUNCTION_NAME,
   );
 
   return result[0] ?? null;

src/queries/sql/getChannelExpandedMetrics.ts

@@ -12,6 +12,8 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getChannelExpandedMetrics';
+
 export interface ChannelExpandedMetricsParameters {
   limit?: number | string;
   offset?: number | string;
@@ -79,6 +81,7 @@ async function relationalQuery(
     order by y desc;
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }
 
@@ -145,6 +148,7 @@ async function clickhouseQuery(
     order by visitors desc, visits desc;
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }
 

src/queries/sql/getChannelMetrics.ts

@@ -12,6 +12,8 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getChannelMetrics';
+
 export async function getChannelMetrics(...args: [websiteId: string, filters?: QueryFilters]) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -60,6 +62,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     order by y desc;
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }
 
@@ -117,7 +120,7 @@ async function clickhouseQuery(
     order by y desc;
   `;
 
-  return rawQuery(sql, queryParams);
+  return rawQuery(sql, queryParams, FUNCTION_NAME);
 }
 
 function toClickHouseStringArray(arr: string[]): string {

src/queries/sql/getRealtimeActivity.ts

@@ -3,6 +3,8 @@ import clickhouse from '@/lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getRealtimeActivity';
+
 export async function getRealtimeActivity(...args: [websiteId: string, filters: QueryFilters]) {
   return runQuery({
     [PRISMA]: () => relationalQuery(...args),
@@ -40,6 +42,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
     limit 100
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }
 
@@ -71,5 +74,6 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters): Promis
         limit 100
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }

src/queries/sql/getValues.ts

@@ -3,6 +3,8 @@ import clickhouse from '@/lib/clickhouse';
 import { runQuery, CLICKHOUSE, PRISMA } from '@/lib/db';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getValues';
+
 export async function getValues(
   ...args: [websiteId: string, column: string, filters: QueryFilters]
 ) {
@@ -64,6 +66,7 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
       search: `%${search}%`,
       ...params,
     },
+    FUNCTION_NAME,
   );
 }
 
@@ -120,5 +123,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
       search,
       ...params,
     },
+    FUNCTION_NAME,
   );
 }

src/queries/sql/pageviews/getPageviewExpandedMetrics.ts

@@ -1,9 +1,11 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
+import { FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getPageviewExpandedMetrics';
+
 export interface PageviewExpandedMetricsParameters {
   type: string;
   limit?: number | string;
@@ -40,7 +42,6 @@ async function relationalQuery(
     {
       ...filters,
       websiteId,
-      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
     },
     { joinSession: SESSION_COLUMNS.includes(type) },
   );
@@ -89,6 +90,7 @@ async function relationalQuery(
     offset ${offset}
     `,
     queryParams,
+    FUNCTION_NAME,
   );
 }
 
@@ -103,7 +105,6 @@ async function clickhouseQuery(
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
     websiteId,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
   });
 
   let excludeDomain = '';
@@ -164,6 +165,7 @@ async function clickhouseQuery(
     offset ${offset}
     `,
     { ...queryParams, ...parameters },
+    FUNCTION_NAME,
   );
 }
 

src/queries/sql/pageviews/getPageviewMetrics.ts

@@ -1,9 +1,11 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getPageviewMetrics';
+
 export interface PageviewMetricsParameters {
   type: string;
   limit?: number | string;
@@ -36,7 +38,6 @@ async function relationalQuery(
     {
       ...filters,
       websiteId,
-      eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
     },
     { joinSession: SESSION_COLUMNS.includes(type) },
   );
@@ -86,6 +87,7 @@ async function relationalQuery(
     offset ${offset}
     `,
     { ...queryParams, ...parameters },
+    FUNCTION_NAME,
   );
 }
 
@@ -100,7 +102,6 @@ async function clickhouseQuery(
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
     websiteId,
-    eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
   });
 
   let sql = '';
@@ -183,5 +184,5 @@ async function clickhouseQuery(
     `;
   }
 
-  return rawQuery(sql, { ...queryParams, ...parameters });
+  return rawQuery(sql, { ...queryParams, ...parameters }, FUNCTION_NAME);
 }

src/queries/sql/sessions/getSessionMetrics.ts

@@ -1,9 +1,11 @@
 import clickhouse from '@/lib/clickhouse';
-import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
 import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
 import prisma from '@/lib/prisma';
 import { QueryFilters } from '@/lib/types';
 
+const FUNCTION_NAME = 'getSessionMetrics';
+
 export interface SessionMetricsParameters {
   type: string;
   limit?: number | string;
@@ -31,7 +33,6 @@ async function relationalQuery(
     {
       ...filters,
       websiteId,
-      eventType: EVENT_TYPE.pageView,
     },
     {
       joinSession: SESSION_COLUMNS.includes(type),
@@ -54,6 +55,7 @@ async function relationalQuery(
     ${joinSessionQuery}
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
+      and event_type != 2
     ${filterQuery}
     group by 1 
     ${includeCountry ? ', 3' : ''}
@@ -62,6 +64,7 @@ async function relationalQuery(
     offset ${offset}
     `,
     { ...queryParams, ...parameters },
+    FUNCTION_NAME,
   );
 }
 
@@ -76,7 +79,6 @@ async function clickhouseQuery(
   const { filterQuery, cohortQuery, queryParams } = parseFilters({
     ...filters,
     websiteId,
-    eventType: EVENT_TYPE.pageView,
   });
   const includeCountry = column === 'city' || column === 'region';
 
@@ -96,6 +98,7 @@ async function clickhouseQuery(
     ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_type != 2
       ${filterQuery}
     group by x 
     ${includeCountry ? ', country' : ''}
@@ -113,6 +116,7 @@ async function clickhouseQuery(
     ${cohortQuery}
     where website_id = {websiteId:UUID}
       and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+      and event_type != 2
       ${filterQuery}
     group by x 
     ${includeCountry ? ', country' : ''}
@@ -122,5 +126,5 @@ async function clickhouseQuery(
     `;
   }
 
-  return rawQuery(sql, { ...queryParams, ...parameters });
+  return rawQuery(sql, { ...queryParams, ...parameters }, FUNCTION_NAME);
 }