diff --git a/db/mysql/migrations/11_add_segment/migration.sql b/db/mysql/migrations/11_add_segment/migration.sql new file mode 100644 index 00000000..c79e916d --- /dev/null +++ b/db/mysql/migrations/11_add_segment/migration.sql @@ -0,0 +1,14 @@ +-- CreateTable +CREATE TABLE `segment` ( + `segment_id` VARCHAR(36) NOT NULL, + `website_id` VARCHAR(36) NOT NULL, + `type` VARCHAR(200) NOT NULL, + `name` VARCHAR(200) NOT NULL, + `parameters` JSON NOT NULL, + `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), + `updated_at` TIMESTAMP(0) NULL, + + UNIQUE INDEX `segment_segment_id_key`(`segment_id`), + INDEX `segment_website_id_idx`(`website_id`), + PRIMARY KEY (`segment_id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/db/mysql/migrations/12_update_report_parameter/migration.sql b/db/mysql/migrations/12_update_report_parameter/migration.sql new file mode 100644 index 00000000..f6a99c3f --- /dev/null +++ b/db/mysql/migrations/12_update_report_parameter/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE `report` MODIFY `parameters` JSON NOT NULL; diff --git a/db/mysql/migrations/13_add_revenue/migration.sql b/db/mysql/migrations/13_add_revenue/migration.sql new file mode 100644 index 00000000..96115a33 --- /dev/null +++ b/db/mysql/migrations/13_add_revenue/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE `revenue` ( + `revenue_id` VARCHAR(36) NOT NULL, + `website_id` VARCHAR(36) NOT NULL, + `session_id` VARCHAR(36) NOT NULL, + `event_id` VARCHAR(36) NOT NULL, + `event_name` VARCHAR(50) NOT NULL, + `currency` VARCHAR(100) NOT NULL, + `revenue` DECIMAL(19, 4) NULL, + `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), + + UNIQUE INDEX `revenue_revenue_id_key`(`revenue_id`), + INDEX `revenue_website_id_idx`(`website_id`), + INDEX `revenue_session_id_idx`(`session_id`), + INDEX `revenue_website_id_created_at_idx`(`website_id`, `created_at`), + INDEX `revenue_website_id_session_id_created_at_idx`(`website_id`, `session_id`, `created_at`), + PRIMARY KEY (`revenue_id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/package.json b/package.json index 3c0e3810..f58b58c0 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "is-localhost-ip": "^1.4.0", "isbot": "^5.1.16", "jsonwebtoken": "^9.0.2", + "jszip": "^3.10.1", "kafkajs": "^2.1.0", "lucide-react": "^0.517.0", "maxmind": "^4.3.27", @@ -114,6 +115,8 @@ "next": "15.4.3", "node-fetch": "^3.2.8", "npm-run-all": "^4.1.5", + "papaparse": "^5.5.3", + "prisma": "6.7.0", "pg": "^8.16.3", "prisma": "6.12.0", "pure-rand": "^6.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8adb2d24..b865e7a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -119,6 +119,9 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: 9.0.2 + jszip: + specifier: ^3.10.1 + version: 3.10.1 kafkajs: specifier: ^2.1.0 version: 2.2.4 @@ -140,6 +143,9 @@ importers: npm-run-all: specifier: ^4.1.5 version: 4.1.5 + papaparse: + specifier: ^5.5.3 + version: 5.5.3 pg: specifier: ^8.16.3 version: 8.16.3 @@ -4219,6 +4225,9 @@ packages: resolution: {integrity: sha512-gJzzk+PQNznz8ysRrC0aOkBNVRBDtE1n53IqyqEf3PXrYwomFs5q4pGMizBMJF+ykh03insJ27hB8gSrD2Hn8A==} engines: {node: '>= 4'} + immediate@3.0.6: + resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + immer@9.0.21: resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} @@ -4476,6 +4485,9 @@ packages: resolution: {integrity: sha512-mfcwb6IzQyOKTs84CQMrOwW4gQcaTOAWJ0zzJCl2WSPDrWk/OzDaImWFH3djXhb24g4eudZfLRozAvPGw4d9hQ==} engines: {node: '>= 0.4'} + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + isarray@2.0.5: resolution: {integrity: sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==} @@ -4746,6 +4758,9 @@ packages: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} + jszip@3.10.1: + resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@1.4.2: resolution: {integrity: sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw==} @@ -4792,6 +4807,9 @@ packages: resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} engines: {node: '>= 0.8.0'} + lie@3.3.0: + resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} + lilconfig@2.1.0: resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} engines: {node: '>=10'} @@ -5270,6 +5288,12 @@ packages: package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + pako@1.0.11: + resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + + papaparse@5.5.3: + resolution: {integrity: sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -5867,6 +5891,9 @@ packages: typescript: optional: true + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} @@ -6048,6 +6075,9 @@ packages: resolution: {integrity: sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==} engines: {node: '>=12'} + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + redent@3.0.0: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} @@ -6186,6 +6216,9 @@ packages: resolution: {integrity: sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q==} engines: {node: '>=0.4'} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -6242,6 +6275,9 @@ packages: resolution: {integrity: sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw==} engines: {node: '>= 0.4'} + setimmediate@1.0.5: + resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + sharp@0.34.3: resolution: {integrity: sha512-eX2IQ6nFohW4DbvHIOLRB3MHFpYqaqvXd3Tp5e/T/dSH83fxaNJQRvDMhASmkNTsNTVF2/OOopzRCt7xokgPfg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} @@ -6416,6 +6452,9 @@ packages: resolution: {integrity: sha512-UXSH262CSZY1tfu3G3Secr6uGLCFVPMhIqHjlgCUtCCcgihYc/xKs9djMTMUOb2j1mVSeU8EU6NWc/iQKU6Gfg==} engines: {node: '>= 0.4'} + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -11714,6 +11753,8 @@ snapshots: ignore@7.0.4: {} + immediate@3.0.6: {} + immer@9.0.21: {} import-cwd@3.0.0: @@ -11947,6 +11988,8 @@ snapshots: call-bound: 1.0.4 get-intrinsic: 1.3.0 + isarray@1.0.0: {} + isarray@2.0.5: {} isbot@5.1.28: {} @@ -12423,6 +12466,13 @@ snapshots: object.assign: 4.1.7 object.values: 1.2.1 + jszip@3.10.1: + dependencies: + lie: 3.3.0 + pako: 1.0.11 + readable-stream: 2.3.8 + setimmediate: 1.0.5 + jwa@1.4.2: dependencies: buffer-equal-constant-time: 1.0.1 @@ -12464,6 +12514,10 @@ snapshots: prelude-ls: 1.2.1 type-check: 0.4.0 + lie@3.3.0: + dependencies: + immediate: 3.0.6 + lilconfig@2.1.0: {} lines-and-columns@1.2.4: {} @@ -12963,6 +13017,10 @@ snapshots: package-json-from-dist@1.0.1: {} + pako@1.0.11: {} + + papaparse@5.5.3: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -13521,6 +13579,8 @@ snapshots: optionalDependencies: typescript: 5.8.3 + process-nextick-args@2.0.1: {} + process@0.11.10: {} promise.series@0.2.0: {} @@ -13800,6 +13860,16 @@ snapshots: parse-json: 5.2.0 type-fest: 1.4.0 + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.2 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + redent@3.0.0: dependencies: indent-string: 4.0.0 @@ -13973,6 +14043,8 @@ snapshots: has-symbols: 1.1.0 isarray: 2.0.5 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} safe-identifier@0.4.2: {} @@ -14034,6 +14106,8 @@ snapshots: es-errors: 1.3.0 es-object-atoms: 1.1.1 + setimmediate@1.0.5: {} + sharp@0.34.3: dependencies: color: 4.2.3 @@ -14277,6 +14351,10 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 diff --git a/scripts/data-migrations/populate-revenue-table.sql b/scripts/data-migrations/populate-revenue-table.sql new file mode 100644 index 00000000..9df75189 --- /dev/null +++ b/scripts/data-migrations/populate-revenue-table.sql @@ -0,0 +1,41 @@ +----------------------------------------------------- +-- PostgreSQL +----------------------------------------------------- +INSERT INTO "revenue" +SELECT gen_random_uuid() revenue_id, + ed.website_id, + we.session_id, + we.event_id, + we.event_name, + currency.string_value currency, + coalesce(ed.number_value, cast(ed.string_value as numeric(19,4))) revenue, + ed.created_at +FROM event_data ed +JOIN website_event we +ON we.event_id = ed.website_event_id +JOIN (SELECT website_event_id, string_value + FROM event_data + WHERE data_key ilike '%currency%') currency +ON currency.website_event_id = ed.website_event_id +WHERE ed.data_key ilike '%revenue%'; + +----------------------------------------------------- +-- MySQL +----------------------------------------------------- +INSERT INTO `revenue` +SELECT UUID() revenue_id, + ed.website_id, + we.session_id, + we.event_id, + we.event_name, + currency.string_value currency, + coalesce(ed.number_value, cast(ed.string_value as decimal(19,4))) revenue, + ed.created_at +FROM event_data ed +JOIN website_event we +ON we.event_id = ed.website_event_id +JOIN (SELECT website_event_id, string_value + FROM event_data + WHERE data_key like '%currency%') currency +ON currency.website_event_id = ed.website_event_id +WHERE ed.data_key like '%revenue%'; \ No newline at end of file diff --git a/src/app/(main)/websites/[websiteId]/events/EventsPage.tsx b/src/app/(main)/websites/[websiteId]/events/EventsPage.tsx index 7e4ef98a..6af76a91 100644 --- a/src/app/(main)/websites/[websiteId]/events/EventsPage.tsx +++ b/src/app/(main)/websites/[websiteId]/events/EventsPage.tsx @@ -1,28 +1,36 @@ 'use client'; import { TabList, Tab, Tabs, TabPanel, Column } from '@umami/react-zen'; import { EventsTable } from '@/components/metrics/EventsTable'; -import { useState } from 'react'; +import { useState, Key } from 'react'; import { EventsDataTable } from './EventsDataTable'; import { Panel } from '@/components/common/Panel'; import { EventsChart } from '@/components/metrics/EventsChart'; import { useMessages } from '@/components/hooks'; import { EventProperties } from './EventProperties'; import { WebsiteControls } from '@/app/(main)/websites/[websiteId]/WebsiteControls'; +import { getItem, setItem } from '@/lib/storage'; + +const KEY_NAME = 'umami.events.tab'; export function EventsPage({ websiteId }) { const [label, setLabel] = useState(null); - const [tab, setTab] = useState('activity'); + const [tab, setTab] = useState(getItem(KEY_NAME) || 'activity'); const { formatMessage, labels } = useMessages(); const handleLabelClick = (value: string) => { setLabel(value !== label ? value : ''); }; + const handleSelect = (value: Key) => { + setItem(KEY_NAME, value); + setTab(value); + }; + return ( - setTab(value)}> + handleSelect(key)}> {formatMessage(labels.activity)} {formatMessage(labels.chart)} diff --git a/src/app/api/reports/[reportId]/route.ts b/src/app/api/reports/[reportId]/route.ts index 7107bc28..c35c1eb0 100644 --- a/src/app/api/reports/[reportId]/route.ts +++ b/src/app/api/reports/[reportId]/route.ts @@ -51,8 +51,8 @@ export async function POST( type, name, description, - parameters, - }); + parameters: parameters, + } as any); return json(result); } diff --git a/src/app/api/websites/[websiteId]/events/series/route.ts b/src/app/api/websites/[websiteId]/events/series/route.ts index 9c33bd17..f85f24a5 100644 --- a/src/app/api/websites/[websiteId]/events/series/route.ts +++ b/src/app/api/websites/[websiteId]/events/series/route.ts @@ -3,7 +3,7 @@ import { parseRequest, getQueryFilters } from '@/lib/request'; import { unauthorized, json } from '@/lib/response'; import { canViewWebsite } from '@/lib/auth'; import { filterParams, timezoneParam, unitParam } from '@/lib/schema'; -import { getEventMetrics } from '@/queries'; +import { getEventStats } from '@/queries'; export async function GET( request: Request, @@ -31,7 +31,7 @@ export async function GET( const filters = await getQueryFilters(query, websiteId); - const data = await getEventMetrics(websiteId, filters); + const data = await getEventStats(websiteId, filters); return json(data); } diff --git a/src/app/api/websites/[websiteId]/export/route.ts b/src/app/api/websites/[websiteId]/export/route.ts new file mode 100644 index 00000000..ab9fdd12 --- /dev/null +++ b/src/app/api/websites/[websiteId]/export/route.ts @@ -0,0 +1,64 @@ +import { z } from 'zod'; +import JSZip from 'jszip'; +import Papa from 'papaparse'; +import { getQueryFilters, parseRequest } from '@/lib/request'; +import { unauthorized, json } from '@/lib/response'; +import { canViewWebsite } from '@/lib/auth'; +import { pagingParams, dateRangeParams } from '@/lib/schema'; +import { getEventMetrics, getPageviewMetrics, getSessionMetrics } from '@/queries'; + +export async function GET( + request: Request, + { params }: { params: Promise<{ websiteId: string }> }, +) { + const schema = z.object({ + ...dateRangeParams, + ...pagingParams, + }); + + const { auth, query, error } = await parseRequest(request, schema); + + if (error) { + return error(); + } + + const { websiteId } = await params; + + if (!(await canViewWebsite(auth, websiteId))) { + return unauthorized(); + } + + const filters = await getQueryFilters(query, websiteId); + + const [events, pages, referrers, browsers, os, devices, countries] = await Promise.all([ + getEventMetrics(websiteId, { type: 'event' }, filters), + getPageviewMetrics(websiteId, { type: 'path' }, filters), + getPageviewMetrics(websiteId, { type: 'referrer' }, filters), + getSessionMetrics(websiteId, { type: 'browser' }, filters), + getSessionMetrics(websiteId, { type: 'os' }, filters), + getSessionMetrics(websiteId, { type: 'device' }, filters), + getSessionMetrics(websiteId, { type: 'country' }, filters), + ]); + + const zip = new JSZip(); + + const parse = (data: any) => { + return Papa.unparse(data, { + header: true, + skipEmptyLines: true, + }); + }; + + zip.file('events.csv', parse(events)); + zip.file('pages.csv', parse(pages)); + zip.file('referrers.csv', parse(referrers)); + zip.file('browsers.csv', parse(browsers)); + zip.file('os.csv', parse(os)); + zip.file('devices.csv', parse(devices)); + zip.file('countries.csv', parse(countries)); + + const content = await zip.generateAsync({ type: 'nodebuffer' }); + const base64 = content.toString('base64'); + + return json({ zip: base64 }); +} diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index c576ff26..317b38da 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -13,7 +13,12 @@ import { } from '@/lib/constants'; import { parseRequest, getQueryFilters } from '@/lib/request'; import { json, unauthorized, badRequest } from '@/lib/response'; -import { getPageviewMetrics, getSessionMetrics, getChannelMetrics } from '@/queries'; +import { + getEventMetrics, + getPageviewMetrics, + getSessionMetrics, + getChannelMetrics, +} from '@/queries'; import { dateRangeParams, filterParams, searchParams } from '@/lib/schema'; export async function GET( @@ -71,7 +76,13 @@ export async function GET( } if (EVENT_COLUMNS.includes(type)) { - const data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters); + let data; + + if (type === 'event') { + data = await getEventMetrics(websiteId, { type, limit, offset }, filters); + } else { + data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters); + } return json(data); } diff --git a/src/assets/download.svg b/src/assets/download.svg new file mode 100644 index 00000000..b2482c9b --- /dev/null +++ b/src/assets/download.svg @@ -0,0 +1 @@ + diff --git a/src/assets/export.svg b/src/assets/export.svg new file mode 100644 index 00000000..d7585b15 --- /dev/null +++ b/src/assets/export.svg @@ -0,0 +1 @@ + diff --git a/src/components/declarations.d.ts b/src/components/declarations.d.ts deleted file mode 100644 index 4bce9eec..00000000 --- a/src/components/declarations.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -declare module '*.css'; -declare module '*.svg'; -declare module '*.json'; -declare module 'react-simple-maps'; -declare module 'uuid'; diff --git a/src/components/input/DownloadButton.tsx b/src/components/input/DownloadButton.tsx new file mode 100644 index 00000000..795db8dc --- /dev/null +++ b/src/components/input/DownloadButton.tsx @@ -0,0 +1,42 @@ +import Papa from 'papaparse'; +import { Button, Icon, TooltipTrigger, Tooltip } from '@umami/react-zen'; +import { Download } from '@/components/icons'; +import { useMessages } from '@/components/hooks'; + +export function DownloadButton({ + filename = 'data', + data, +}: { + filename?: string; + data?: any; + onClick?: () => void; +}) { + const { formatMessage, labels } = useMessages(); + + const handleClick = async () => { + downloadCsv(`${filename}.csv`, Papa.unparse(data)); + }; + + return ( + + + {formatMessage(labels.download)} + + ); +} + +function downloadCsv(filename: string, data: any) { + const blob = new Blob([data], { type: 'text/csv' }); + const url = URL.createObjectURL(blob); + + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.click(); + + URL.revokeObjectURL(url); +} diff --git a/src/components/input/ExportButton.tsx b/src/components/input/ExportButton.tsx new file mode 100644 index 00000000..76f63e2c --- /dev/null +++ b/src/components/input/ExportButton.tsx @@ -0,0 +1,64 @@ +import { useState } from 'react'; +import { Icon, Tooltip, TooltipTrigger, LoadingButton } from '@umami/react-zen'; +import { Download } from '@/components/icons'; +import { useMessages, useApi } from '@/components/hooks'; +import { useSearchParams } from 'next/navigation'; +import { useDateParameters } from '@/components/hooks/useDateParameters'; +import { useFilterParameters } from '@/components/hooks/useFilterParameters'; + +export function ExportButton({ websiteId }: { websiteId: string }) { + const { formatMessage, labels } = useMessages(); + const [isLoading, setIsLoading] = useState(false); + const date = useDateParameters(websiteId); + const filters = useFilterParameters(); + const searchParams = useSearchParams(); + const { get } = useApi(); + + const handleClick = async () => { + setIsLoading(true); + + const { zip } = await get(`/websites/${websiteId}/export`, { + ...date, + ...filters, + ...searchParams, + format: 'json', + }); + + await loadZip(zip); + + setIsLoading(false); + }; + + return ( + + + + + + + {formatMessage(labels.download)} + + ); +} + +async function loadZip(zip: string) { + const binary = atob(zip); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + + const blob = new Blob([bytes], { type: 'application/zip' }); + const url = URL.createObjectURL(blob); + + const a = document.createElement('a'); + a.href = url; + a.download = 'download.zip'; + a.click(); + URL.revokeObjectURL(url); +} diff --git a/src/components/input/WebsiteDateFilter.tsx b/src/components/input/WebsiteDateFilter.tsx index e78bcd8d..4737517a 100644 --- a/src/components/input/WebsiteDateFilter.tsx +++ b/src/components/input/WebsiteDateFilter.tsx @@ -12,6 +12,7 @@ import { isAfter } from 'date-fns'; import { Chevron, Close, Compare } from '@/components/icons'; import { useDateRange, useMessages, useNavigation } from '@/components/hooks'; import { DateFilter } from './DateFilter'; +import { ExportButton } from '@/components/input/ExportButton'; export function WebsiteDateFilter({ websiteId, @@ -99,6 +100,7 @@ export function WebsiteDateFilter({ {compare ? : } {formatMessage(compare ? labels.cancel : labels.compareDates)} + )} diff --git a/src/components/input/WebsiteSelect.tsx b/src/components/input/WebsiteSelect.tsx index 3b5bd669..18a1e4a7 100644 --- a/src/components/input/WebsiteSelect.tsx +++ b/src/components/input/WebsiteSelect.tsx @@ -1,5 +1,5 @@ import { useState } from 'react'; -import { Select, SelectProps, ListItem } from '@umami/react-zen'; +import { Select, SelectProps, ListItem, Text } from '@umami/react-zen'; import { useUserWebsitesQuery, useWebsiteQuery, useNavigation } from '@/components/hooks'; import { ButtonProps } from 'react-basics'; @@ -38,7 +38,11 @@ export function WebsiteSelect({ searchValue={search} onSearch={handleSearch} onChange={handleSelect} - renderValue={() => website?.name} + renderValue={() => ( + + {website?.name} + + )} > {({ id, name }: any) => {name}} diff --git a/src/components/messages.ts b/src/components/messages.ts index d6d5a87a..e0e8413e 100644 --- a/src/components/messages.ts +++ b/src/components/messages.ts @@ -338,6 +338,7 @@ export const labels = defineMessages({ location: { id: 'label.location', defaultMessage: 'Location' }, chart: { id: 'label.chart', defaultMessage: 'Chart' }, table: { id: 'label.table', defaultMessage: 'Table' }, + download: { id: 'label.download', defaultMessage: 'Download' }, traffic: { id: 'label.traffic', defaultMessage: 'Traffic' }, behavior: { id: 'label.behavior', defaultMessage: 'Behavior' }, growth: { id: 'label.growth', defaultMessage: 'Growth' }, diff --git a/src/components/metrics/EventsTable.tsx b/src/components/metrics/EventsTable.tsx index c8008922..2ee4dace 100644 --- a/src/components/metrics/EventsTable.tsx +++ b/src/components/metrics/EventsTable.tsx @@ -32,6 +32,7 @@ export function EventsTable({ onLabelClick, ...props }: EventsTableProps) { metric={formatMessage(labels.actions)} onDataLoad={handleDataLoad} renderLabel={renderLabel} + allowDownload={false} /> ); } diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index 45b6c9c5..c229b00c 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -1,4 +1,4 @@ -import { ReactNode, useEffect, useMemo, useState } from 'react'; +import { ReactNode, useMemo, useState } from 'react'; import { Icon, Text, SearchField, Row, Column } from '@umami/react-zen'; import { LinkButton } from '@/components/common/LinkButton'; import { DEFAULT_ANIMATION_DURATION } from '@/lib/constants'; @@ -7,6 +7,7 @@ import { useNavigation, useWebsiteMetricsQuery, useMessages, useFormat } from '@ import { Arrow } from '@/components/icons'; import { ListTable, ListTableProps } from './ListTable'; import { LoadingPanel } from '@/components/common/LoadingPanel'; +import { DownloadButton } from '@/components/input/DownloadButton'; export interface MetricsTableProps extends ListTableProps { websiteId: string; @@ -18,9 +19,8 @@ export interface MetricsTableProps extends ListTableProps { allowSearch?: boolean; searchFormattedValues?: boolean; showMore?: boolean; - params?: Record; - onDataLoad?: (data: any) => any; - className?: string; + params?: { [key: string]: any }; + allowDownload?: boolean; children?: ReactNode; } @@ -34,8 +34,7 @@ export function MetricsTable({ searchFormattedValues = false, showMore = true, params, - onDataLoad, - className, + allowDownload = true, children, ...props }: MetricsTableProps) { @@ -86,22 +85,17 @@ export function MetricsTable({ return []; }, [data, dataFilter, search, limit, formatValue, type]); - useEffect(() => { - if (data) { - onDataLoad?.(data); - } - }, [data]); - return ( {allowSearch && } - {children} + + {children} + {allowDownload && } + - {data && ( - - )} + {data && } {showMore && data && !error && limit && ( diff --git a/src/declaration.d.ts b/src/declaration.d.ts index 8c1820bb..6982f5b4 100644 --- a/src/declaration.d.ts +++ b/src/declaration.d.ts @@ -1,3 +1,6 @@ +declare module '*.css'; +declare module '*.svg'; +declare module '*.json'; declare module 'bcryptjs'; declare module 'chartjs-adapter-date-fns'; declare module 'cors'; @@ -6,5 +9,8 @@ declare module 'debug'; declare module 'fs-extra'; declare module 'jsonwebtoken'; declare module 'md5'; +declare module 'papaparse'; declare module 'prettier'; +declare module 'react-simple-maps'; declare module 'semver'; +declare module 'uuid'; diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts index 1b608cfc..1670546e 100644 --- a/src/lib/clickhouse.ts +++ b/src/lib/clickhouse.ts @@ -227,8 +227,7 @@ async function rawQuery( params: Record = {}, ): Promise { if (process.env.LOG_QUERY) { - log('QUERY:\n', query); - log('PARAMETERS:\n', params); + log({ query, params }); } await connect(); diff --git a/src/queries/index.ts b/src/queries/index.ts index 7ce5a54f..fba7e548 100644 --- a/src/queries/index.ts +++ b/src/queries/index.ts @@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues'; export * from '@/queries/sql/events/getEventDataStats'; export * from '@/queries/sql/events/getEventDataUsage'; export * from '@/queries/sql/events/getEventMetrics'; +export * from '@/queries/sql/events/getEventStats'; export * from '@/queries/sql/events/getWebsiteEvents'; export * from '@/queries/sql/events/getEventUsage'; export * from '@/queries/sql/events/saveEvent'; diff --git a/src/queries/sql/events/getEventMetrics.ts b/src/queries/sql/events/getEventMetrics.ts index 06c141ba..73af2278 100644 --- a/src/queries/sql/events/getEventMetrics.ts +++ b/src/queries/sql/events/getEventMetrics.ts @@ -1,9 +1,15 @@ import clickhouse from '@/lib/clickhouse'; -import { EVENT_TYPE } from '@/lib/constants'; +import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import { QueryFilters } from '@/lib/types'; +export interface WebsiteEventMetricParameters { + type: string; + limit?: string; + offset?: string; +} + export interface WebsiteEventMetricData { x: string; t: string; @@ -11,7 +17,7 @@ export interface WebsiteEventMetricData { } export async function getEventMetrics( - ...args: [websiteId: string, filters: QueryFilters] + ...args: [websiteId: string, parameters: WebsiteEventMetricParameters, filters: QueryFilters] ): Promise { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -19,29 +25,38 @@ export async function getEventMetrics( }); } -async function relationalQuery(websiteId: string, filters: QueryFilters) { - const { timezone = 'utc', unit = 'day' } = filters; - const { rawQuery, getDateSQL, parseFilters } = prisma; - const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({ - ...filters, - eventType: EVENT_TYPE.customEvent, - }); +async function relationalQuery( + websiteId: string, + parameters: WebsiteEventMetricParameters, + filters: QueryFilters, +) { + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters( + { + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }, + { joinSession: SESSION_COLUMNS.includes(type) }, + ); return rawQuery( ` - select - event_name x, - ${getDateSQL('website_event.created_at', unit, timezone)} t, - count(*) y + select ${column} x, + count(*) as y from website_event - ${joinSessionQuery} ${cohortQuery} + ${joinSessionQuery} where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} ${filterQuery} - group by 1, 2 - order by 2 + group by 1 + order by 2 desc + limit ${limit} + offset ${offset} `, queryParams, ); @@ -49,51 +64,32 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery( websiteId: string, + parameters: WebsiteEventMetricParameters, filters: QueryFilters, ): Promise { - const { timezone = 'UTC', unit = 'day' } = filters; - const { rawQuery, getDateSQL, parseFilters } = clickhouse; + const { type, limit = 500, offset = 0 } = parameters; + const column = FILTER_COLUMNS[type] || type; + const { rawQuery, parseFilters } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, eventType: EVENT_TYPE.customEvent, }); - let sql = ''; - - if (filterQuery || cohortQuery) { - sql = ` - select - event_name x, - ${getDateSQL('created_at', unit, timezone)} t, - count(*) y - from website_event - ${cohortQuery} - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - ${filterQuery} - group by x, t - order by t - `; - } else { - sql = ` - select - event_name x, - ${getDateSQL('created_at', unit, timezone)} t, - count(*) y - from ( - select arrayJoin(event_name) as event_name, - created_at - from website_event_stats_hourly as website_event - where website_id = {websiteId:UUID} - and created_at between {startDate:DateTime64} and {endDate:DateTime64} - and event_type = {eventType:UInt32} - ) as g - group by x, t - order by t - `; - } - - return rawQuery(sql, queryParams); + return rawQuery( + `select ${column} x, + count(*) as y + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x + order by y desc + limit ${limit} + offset ${offset} + `, + queryParams, + ); } diff --git a/src/queries/sql/events/getEventStats.ts b/src/queries/sql/events/getEventStats.ts new file mode 100644 index 00000000..ad6b155d --- /dev/null +++ b/src/queries/sql/events/getEventStats.ts @@ -0,0 +1,100 @@ +import clickhouse from '@/lib/clickhouse'; +import { EVENT_TYPE } from '@/lib/constants'; +import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; +import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +interface WebsiteEventMetric { + x: string; + t: string; + y: number; +} + +export async function getEventStats( + ...args: [websiteId: string, filters: QueryFilters] +): Promise { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, filters: QueryFilters) { + const { timezone = 'utc', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = prisma; + const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }); + + return rawQuery( + ` + select + event_name x, + ${getDateSQL('website_event.created_at', unit, timezone)} t, + count(*) y + from website_event + ${cohortQuery} + ${joinSessionQuery} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1, 2 + order by 2 + `, + queryParams, + ); +} + +async function clickhouseQuery( + websiteId: string, + filters: QueryFilters, +): Promise<{ x: string; t: string; y: number }[]> { + const { timezone = 'UTC', unit = 'day' } = filters; + const { rawQuery, getDateSQL, parseFilters } = clickhouse; + const { filterQuery, cohortQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + eventType: EVENT_TYPE.customEvent, + }); + + let sql = ''; + + if (filterQuery || cohortQuery) { + sql = ` + select + event_name x, + ${getDateSQL('created_at', unit, timezone)} t, + count(*) y + from website_event + ${cohortQuery} + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ${filterQuery} + group by x, t + order by t + `; + } else { + sql = ` + select + event_name x, + ${getDateSQL('created_at', unit, timezone)} t, + count(*) y + from ( + select arrayJoin(event_name) as event_name, + created_at + from website_event_stats_hourly website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + and event_type = {eventType:UInt32} + ) as g + group by x, t + order by t + `; + } + + return rawQuery(sql, queryParams); +} diff --git a/src/queries/sql/reports/getUTM.ts b/src/queries/sql/reports/getUTM.ts index 9fda7d83..f96c62d3 100644 --- a/src/queries/sql/reports/getUTM.ts +++ b/src/queries/sql/reports/getUTM.ts @@ -1,15 +1,16 @@ import clickhouse from '@/lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; -import { QueryFilters } from '@/lib/types'; - -export interface UTMParameters { - startDate: Date; - endDate: Date; -} export async function getUTM( - ...args: [websiteId: string, parameters: UTMParameters, filters: QueryFilters] + ...args: [ + websiteId: string, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, + ] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -19,12 +20,14 @@ export async function getUTM( async function relationalQuery( websiteId: string, - parameters: UTMParameters, - filters: QueryFilters, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, ) { - const { startDate, endDate } = parameters; - const { rawQuery, parseFilters } = prisma; - const { filterQuery, queryParams } = parseFilters({ ...filters, websiteId, startDate, endDate }); + const { startDate, endDate } = filters; + const { rawQuery } = prisma; return rawQuery( ` @@ -34,21 +37,26 @@ async function relationalQuery( and created_at between {{startDate}} and {{endDate}} and coalesce(url_query, '') != '' and event_type = 1 - ${filterQuery} group by 1 `, - queryParams, - ).then(result => parseParameters(result as any[])); + { + websiteId, + startDate, + endDate, + }, + ); } async function clickhouseQuery( websiteId: string, - parameters: UTMParameters, - filters: QueryFilters, + filters: { + startDate: Date; + endDate: Date; + timezone?: string; + }, ) { - const { startDate, endDate } = parameters; - const { rawQuery, parseFilters } = clickhouse; - const { filterQuery, queryParams } = parseFilters({ ...filters, websiteId, startDate, endDate }); + const { startDate, endDate } = filters; + const { rawQuery } = clickhouse; return rawQuery( ` @@ -58,34 +66,12 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and url_query != '' and event_type = 1 - ${filterQuery} group by 1 `, - queryParams, - ).then(result => parseParameters(result as any[])); -} - -function parseParameters(data: any[]) { - return data.reduce((obj, { url_query, num }) => { - try { - const searchParams = new URLSearchParams(url_query); - - for (const [key, value] of searchParams) { - if (key.match(/^utm_(\w+)$/)) { - const name = value; - if (!obj[key]) { - obj[key] = { [name]: Number(num) }; - } else if (!obj[key][name]) { - obj[key][name] = Number(num); - } else { - obj[key][name] += Number(num); - } - } - } - } catch { - // Ignore - } - - return obj; - }, {}); + { + websiteId, + startDate, + endDate, + }, + ); } diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.ts index 7e2393cf..55962310 100644 --- a/src/queries/sql/sessions/getSessionMetrics.ts +++ b/src/queries/sql/sessions/getSessionMetrics.ts @@ -6,8 +6,8 @@ import { QueryFilters } from '@/lib/types'; export interface SessionMetricsParameters { type: string; - limit: number | string; - offset: number | string; + limit?: number | string; + offset?: number | string; } export async function getSessionMetrics(