watson/umami
1
0
Fork 0
mirror of https://github.com/umami-software/umami.git synced 2026-02-12 16:45:35 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'dev' of https://github.com/umami-software/umami into feat/um-376-retention-report

Browse source
This commit is contained in:
Francis Cao 2023-08-04 13:10:33 -07:00
commit 8fff65728c
23 changed files with 364 additions and 207 deletions
Download patch file Download diff file Expand all files Collapse all files

12
.github/workflows/cd-manual.yml vendored
View file

@ -21,7 +21,7 @@ jobs:
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- uses: mr-smithers-excellent/docker-build-push@v6 - uses: mr-smithers-excellent/docker-build-push@v6
name: Build & push Docker image for ${{ matrix.db-type }} name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }}
with: with:
image: umami image: umami
tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest
@ -31,3 +31,13 @@ jobs:
platform: linux/amd64,linux/arm64 platform: linux/amd64,linux/arm64
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- uses: mr-smithers-excellent/docker-build-push@v6
name: Build & push Docker image to docker.io for ${{ matrix.db-type }}
with:
image: umamisoftware/umami
tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest
buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
registry: docker.io
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

12
.github/workflows/cd.yml vendored
View file

@ -19,7 +19,7 @@ jobs:
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- uses: mr-smithers-excellent/docker-build-push@v6 - uses: mr-smithers-excellent/docker-build-push@v6
name: Build & push Docker image for ${{ matrix.db-type }} name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }}
with: with:
image: umami image: umami
tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest
@ -29,3 +29,13 @@ jobs:
platform: linux/amd64,linux/arm64 platform: linux/amd64,linux/arm64
username: ${{ github.actor }} username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- uses: mr-smithers-excellent/docker-build-push@v6
name: Build & push Docker image to docker.io for ${{ matrix.db-type }}
with:
image: umamisoftware/umami
tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest
buildArgs: DATABASE_TYPE=${{ matrix.db-type }}
registry: docker.io
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

6
components/layout/AppLayout.js
View file

@ -2,9 +2,7 @@ import { Container } from 'react-basics';
import Head from 'next/head'; import Head from 'next/head';
import NavBar from 'components/layout/NavBar'; import NavBar from 'components/layout/NavBar';
import UpdateNotice from 'components/common/UpdateNotice'; import UpdateNotice from 'components/common/UpdateNotice';
import useRequireLogin from 'hooks/useRequireLogin'; import { useRequireLogin, useConfig } from 'hooks';
import useConfig from 'hooks/useConfig';
import { CURRENT_VERSION } from 'lib/constants';
import styles from './AppLayout.module.css'; import styles from './AppLayout.module.css';
export function AppLayout({ title, children }) { export function AppLayout({ title, children }) {
@ -16,7 +14,7 @@ export function AppLayout({ title, children }) {
} }
return ( return (
<div className={styles.layout} data-app-version={CURRENT_VERSION}> <div className={styles.layout}>
<UpdateNotice user={user} config={config} /> <UpdateNotice user={user} config={config} />
<Head> <Head>
<title>{title ? `${title} | umami` : 'umami'}</title> <title>{title ? `${title} | umami` : 'umami'}</title>

2
components/pages/reports/FieldSelectForm.js
View file

@ -13,7 +13,7 @@ export default function FieldSelectForm({ fields, onSelect }) {
return ( return (
<Item key={index} className={styles.item}> <Item key={index} className={styles.item}>
<div>{label || name}</div> <div>{label || name}</div>
<div className={styles.type}>{type}</div> {type && <div className={styles.type}>{type}</div>}
</Item> </Item>
); );
})} })}

2
components/pages/reports/ReportTemplates.js
View file

@ -33,14 +33,12 @@ export function ReportTemplates() {
const { formatMessage, labels } = useMessages(); const { formatMessage, labels } = useMessages();
const reports = [ const reports = [
/*
{ {
title: formatMessage(labels.insights), title: formatMessage(labels.insights),
description: 'Dive deeper into your data by using segments and filters.', description: 'Dive deeper into your data by using segments and filters.',
url: '/reports/insights', url: '/reports/insights',
icon: <Lightbulb />, icon: <Lightbulb />,
}, },
*/
{ {
title: formatMessage(labels.funnel), title: formatMessage(labels.funnel),
description: 'Understand the conversion and drop-off rate of users.', description: 'Understand the conversion and drop-off rate of users.',

7
components/pages/reports/funnel/FunnelParameters.js
View file

@ -16,6 +16,7 @@ import UrlAddForm from './UrlAddForm';
import { ReportContext } from 'components/pages/reports/Report'; import { ReportContext } from 'components/pages/reports/Report';
import BaseParameters from '../BaseParameters'; import BaseParameters from '../BaseParameters';
import ParameterList from '../ParameterList'; import ParameterList from '../ParameterList';
import PopupForm from '../PopupForm';
export function FunnelParameters() { export function FunnelParameters() {
const { report, runReport, updateReport, isRunning } = useContext(ReportContext); const { report, runReport, updateReport, isRunning } = useContext(ReportContext);
@ -53,7 +54,11 @@ export function FunnelParameters() {
</Icon> </Icon>
<Popup position="bottom" alignment="start"> <Popup position="bottom" alignment="start">
{(close, element) => { {(close, element) => {
return <UrlAddForm element={element} onAdd={handleAddUrl} onClose={close} />; return (
<PopupForm element={element} onClose={close}>
<UrlAddForm onAdd={handleAddUrl} />
</PopupForm>
);
}} }}
</Popup> </Popup>
</PopupTrigger> </PopupTrigger>

40
components/pages/reports/funnel/UrlAddForm.js
View file

@ -2,16 +2,14 @@ import { useState } from 'react';
import { useMessages } from 'hooks'; import { useMessages } from 'hooks';
import { Button, Form, FormRow, TextField, Flexbox } from 'react-basics'; import { Button, Form, FormRow, TextField, Flexbox } from 'react-basics';
import styles from './UrlAddForm.module.css'; import styles from './UrlAddForm.module.css';
import PopupForm from '../PopupForm';
export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) { export function UrlAddForm({ defaultValue = '', onAdd }) {
const [url, setUrl] = useState(defaultValue); const [url, setUrl] = useState(defaultValue);
const { formatMessage, labels } = useMessages(); const { formatMessage, labels } = useMessages();
const handleSave = () => { const handleSave = () => {
onAdd(url); onAdd(url);
setUrl(''); setUrl('');
onClose();
}; };
const handleChange = e => { const handleChange = e => {
@ -26,25 +24,23 @@ export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) {
}; };
return ( return (
<PopupForm element={element}> <Form>
<Form> <FormRow label={formatMessage(labels.url)}>
<FormRow label={formatMessage(labels.url)}> <Flexbox gap={10}>
<Flexbox gap={10}> <TextField
<TextField className={styles.input}
className={styles.input} value={url}
value={url} onChange={handleChange}
onChange={handleChange} autoFocus={true}
autoFocus={true} autoComplete="off"
autoComplete="off" onKeyDown={handleKeyDown}
onKeyDown={handleKeyDown} />
/> <Button variant="primary" onClick={handleSave}>
<Button variant="primary" onClick={handleSave}> {formatMessage(labels.add)}
{formatMessage(labels.add)} </Button>
</Button> </Flexbox>
</Flexbox> </FormRow>
</FormRow> </Form>
</Form>
</PopupForm>
); );
} }

36
components/pages/reports/insights/InsightsParameters.js
View file

@ -2,12 +2,28 @@ import { useContext, useRef } from 'react';
import { useMessages } from 'hooks'; import { useMessages } from 'hooks';
import { Form, FormRow, FormButtons, SubmitButton, PopupTrigger, Icon, Popup } from 'react-basics'; import { Form, FormRow, FormButtons, SubmitButton, PopupTrigger, Icon, Popup } from 'react-basics';
import { ReportContext } from 'components/pages/reports/Report'; import { ReportContext } from 'components/pages/reports/Report';
import { REPORT_PARAMETERS, WEBSITE_EVENT_FIELDS } from 'lib/constants'; import { REPORT_PARAMETERS } from 'lib/constants';
import Icons from 'components/icons'; import Icons from 'components/icons';
import BaseParameters from '../BaseParameters'; import BaseParameters from '../BaseParameters';
import FieldAddForm from '../FieldAddForm';
import ParameterList from '../ParameterList'; import ParameterList from '../ParameterList';
import styles from './InsightsParameters.module.css'; import styles from './InsightsParameters.module.css';
import FieldSelectForm from '../FieldSelectForm';
import PopupForm from '../PopupForm';
import FieldFilterForm from '../FieldFilterForm';
const fieldOptions = [
{ name: 'url', type: 'string' },
{ name: 'title', type: 'string' },
{ name: 'referrer', type: 'string' },
{ name: 'query', type: 'string' },
{ name: 'browser', type: 'string' },
{ name: 'os', type: 'string' },
{ name: 'device', type: 'string' },
{ name: 'country', type: 'string' },
{ name: 'region', type: 'string' },
{ name: 'city', type: 'string' },
{ name: 'language', type: 'string' },
];
export function InsightsParameters() { export function InsightsParameters() {
const { report, runReport, updateReport, isRunning } = useContext(ReportContext); const { report, runReport, updateReport, isRunning } = useContext(ReportContext);
@ -16,7 +32,6 @@ export function InsightsParameters() {
const { parameters } = report || {}; const { parameters } = report || {};
const { websiteId, dateRange, fields, filters, groups } = parameters || {}; const { websiteId, dateRange, fields, filters, groups } = parameters || {};
const queryEnabled = websiteId && dateRange && fields?.length; const queryEnabled = websiteId && dateRange && fields?.length;
const fieldOptions = Object.keys(WEBSITE_EVENT_FIELDS).map(key => WEBSITE_EVENT_FIELDS[key]);
const parameterGroups = [ const parameterGroups = [
{ label: formatMessage(labels.fields), group: REPORT_PARAMETERS.fields }, { label: formatMessage(labels.fields), group: REPORT_PARAMETERS.fields },
@ -57,13 +72,14 @@ export function InsightsParameters() {
<Popup position="bottom" alignment="start"> <Popup position="bottom" alignment="start">
{(close, element) => { {(close, element) => {
return ( return (
<FieldAddForm <PopupForm element={element} onClose={close}>
fields={fieldOptions} {group === REPORT_PARAMETERS.fields && (
group={group} <FieldSelectForm fields={fieldOptions} onSelect={handleAdd.bind(null, group)} />
element={element} )}
onAdd={handleAdd} {group === REPORT_PARAMETERS.filters && (
onClose={close} <FieldFilterForm fields={fieldOptions} onSelect={handleAdd.bind(null, group)} />
/> )}
</PopupForm>
); );
}} }}
</Popup> </Popup>

6
components/pages/reports/insights/InsightsTable.js
View file

@ -6,11 +6,13 @@ import { ReportContext } from '../Report';
export function InsightsTable() { export function InsightsTable() {
const { report } = useContext(ReportContext); const { report } = useContext(ReportContext);
const { formatMessage, labels } = useMessages(); const { formatMessage, labels } = useMessages();
const { fields = [] } = report?.parameters || {};
return ( return (
<GridTable data={report?.data || []}> <GridTable data={report?.data || []}>
<GridColumn name="field" label={formatMessage(labels.field)} /> {fields.map(({ name }) => {
<GridColumn name="value" label={formatMessage(labels.value)} /> return <GridColumn key={name} name={name} label={name} />;
})}
<GridColumn name="total" label={formatMessage(labels.total)} /> <GridColumn name="total" label={formatMessage(labels.total)} />
</GridTable> </GridTable>
); );

11
lib/clickhouse.ts
View file

@ -61,14 +61,17 @@ function getDateFormat(date) {
return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;
} }
function getFilterQuery(filters = {}, params = {}) { function getFilterQuery(filters = {}) {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
if (filter !== undefined) { if (filter !== undefined) {
const column = FILTER_COLUMNS[key] || key; const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column} = {${key}:String}`); arr.push(`and ${column} = {${key}:String}`);
params[key] = decodeURIComponent(filter); }
if (key === 'referrer') {
arr.push('and referrer_domain != {domain:String}');
} }
return arr; return arr;
@ -77,9 +80,9 @@ function getFilterQuery(filters = {}, params = {}) {
return query.join('\n'); return query.join('\n');
} }
function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) { function parseFilters(filters: WebsiteMetricFilter = {}) {
return { return {
filterQuery: getFilterQuery(filters, params), filterQuery: getFilterQuery(filters),
}; };
} }

41
lib/constants.ts
View file

@ -43,11 +43,6 @@ export const SESSION_COLUMNS = [
'city', 'city',
]; ];
export const COLLECTION_TYPE = {
event: 'event',
identify: 'identify',
};
export const FILTER_COLUMNS = { export const FILTER_COLUMNS = {
url: 'url_path', url: 'url_path',
referrer: 'referrer_domain', referrer: 'referrer_domain',
@ -57,6 +52,11 @@ export const FILTER_COLUMNS = {
region: 'subdivision1', region: 'subdivision1',
}; };
export const COLLECTION_TYPE = {
event: 'event',
identify: 'identify',
};
export const EVENT_TYPE = { export const EVENT_TYPE = {
pageView: 1, pageView: 1,
customEvent: 2, customEvent: 2,
@ -120,37 +120,6 @@ export const ROLE_PERMISSIONS = {
[ROLES.teamMember]: [], [ROLES.teamMember]: [],
} as const; } as const;
export const WEBSITE_EVENT_FIELDS = {
eventId: { name: 'event_id', type: 'uuid', label: 'Event ID' },
websiteId: { name: 'website_id', type: 'uuid', label: 'Website ID' },
sessionId: { name: 'session_id', type: 'uuid', label: 'Session ID' },
createdAt: { name: 'created_at', type: 'date', label: 'Created date' },
urlPath: { name: 'url_path', type: 'string', label: 'URL path' },
urlQuery: { name: 'url_query', type: 'string', label: 'URL query' },
referrerPath: { name: 'referrer_path', type: 'string', label: 'Referrer path' },
referrerQuery: { name: 'referrer_query', type: 'string', label: 'Referrer query' },
referrerDomain: { name: 'referrer_domain', type: 'string', label: 'Referrer domain' },
pageTitle: { name: 'page_title', type: 'string', label: 'Page title' },
eventType: { name: 'event_type', type: 'string', label: 'Event type' },
eventName: { name: 'event_name', type: 'string', label: 'Event name' },
};
export const SESSION_FIELDS = {
sessionId: { name: 'session_id', type: 'uuid' },
websiteId: { name: 'website_id', type: 'uuid' },
hostname: { name: 'hostname', type: 'string' },
browser: { name: 'browser', type: 'string' },
os: { name: 'os', type: 'string' },
device: { name: 'device', type: 'string' },
screen: { name: 'screen', type: 'string' },
language: { name: 'language', type: 'string' },
country: { name: 'country', type: 'string' },
subdivision1: { name: 'subdivision1', type: 'string' },
subdivision2: { name: 'subdivision2', type: 'string' },
city: { name: 'city', type: 'string' },
createdAt: { name: 'created_at', type: 'date' },
};
export const THEME_COLORS = { export const THEME_COLORS = {
light: { light: {
primary: '#2680eb', primary: '#2680eb',

28
lib/prisma.ts
View file

@ -1,7 +1,7 @@
import prisma from '@umami/prisma-client'; import prisma from '@umami/prisma-client';
import moment from 'moment-timezone'; import moment from 'moment-timezone';
import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';
import { FILTER_COLUMNS } from './constants'; import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants';
const MYSQL_DATE_FORMATS = { const MYSQL_DATE_FORMATS = {
minute: '%Y-%m-%d %H:%i:00', minute: '%Y-%m-%d %H:%i:00',
@ -64,14 +64,19 @@ function getTimestampIntervalQuery(field: string): string {
} }
} }
function getFilterQuery(filters = {}, params = []): string { function getFilterQuery(filters = {}): string {
const query = Object.keys(filters).reduce((arr, key) => { const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key]; const filter = filters[key];
if (filter !== undefined) { if (filter !== undefined) {
const column = FILTER_COLUMNS[key] || key; const column = FILTER_COLUMNS[key] || key;
arr.push(`and ${column}={{${key}}}`); arr.push(`and ${column}={{${key}}}`);
params.push(decodeURIComponent(filter)); }
if (key === 'referrer') {
arr.push(
'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)',
);
} }
return arr; return arr;
@ -80,19 +85,12 @@ function getFilterQuery(filters = {}, params = []): string {
return query.join('\n'); return query.join('\n');
} }
function parseFilters( function parseFilters(filters: { [key: string]: any } = {}) {
filters: { [key: string]: any } = {},
params = [],
sessionKey = 'session_id',
) {
const { os, browser, device, country, region, city } = filters;
return { return {
joinSession: joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key])
os || browser || device || country || region || city ? `inner join session on website_event.session_id = session.session_id`
? `inner join session on website_event.${sessionKey} = session.${sessionKey}` : '',
: '', filterQuery: getFilterQuery(filters),
filterQuery: getFilterQuery(filters, params),
}; };
} }

5
pages/api/auth/login.ts
View file

@ -7,6 +7,7 @@ import {
checkPassword, checkPassword,
createSecureToken, createSecureToken,
methodNotAllowed, methodNotAllowed,
forbidden,
} from 'next-basics'; } from 'next-basics';
import redis from '@umami/redis-client'; import redis from '@umami/redis-client';
import { getUserByUsername } from 'queries'; import { getUserByUsername } from 'queries';
@ -30,6 +31,10 @@ export default async (
req: NextApiRequestQueryBody<any, LoginRequestBody>, req: NextApiRequestQueryBody<any, LoginRequestBody>,
res: NextApiResponse<LoginResponse>, res: NextApiResponse<LoginResponse>,
) => { ) => {
if (process.env.DISABLE_LOGIN) {
return forbidden(res);
}
if (req.method === 'POST') { if (req.method === 'POST') {
const { username, password } = req.body; const { username, password } = req.body;

2
pages/api/reports/insights.ts
View file

@ -11,7 +11,7 @@ export interface InsightsRequestBody {
startDate: string; startDate: string;
endDate: string; endDate: string;
}; };
fields: string[]; fields: { name: string; type: string; value: string }[];
filters: string[]; filters: string[];
groups: string[]; groups: string[];
} }

53
pages/api/websites/[id]/metrics.ts
View file

@ -46,6 +46,7 @@ export default async (
country, country,
region, region,
city, city,
language,
} = req.query; } = req.query;
if (req.method === 'GET') { if (req.method === 'GET') {
@ -55,20 +56,27 @@ export default async (
const { startDate, endDate } = await parseDateRangeQuery(req); const { startDate, endDate } = await parseDateRangeQuery(req);
const filters = {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
language,
};
filters[type] = undefined;
const column = FILTER_COLUMNS[type] || type;
if (SESSION_COLUMNS.includes(type)) { if (SESSION_COLUMNS.includes(type)) {
const column = FILTER_COLUMNS[type] || type; const data = await getSessionMetrics(websiteId, {
const filters = {
os,
browser,
device,
country,
region,
city,
};
filters[type] = undefined;
let data = await getSessionMetrics(websiteId, {
startDate, startDate,
endDate, endDate,
column, column,
@ -88,30 +96,13 @@ export default async (
} }
} }
data = Object.values(combined); return ok(res, Object.values(combined));
} }
return ok(res, data); return ok(res, data);
} }
if (EVENT_COLUMNS.includes(type)) { if (EVENT_COLUMNS.includes(type)) {
const column = FILTER_COLUMNS[type] || type;
const filters = {
url,
referrer,
title,
query,
event,
os,
browser,
device,
country,
region,
city,
};
filters[type] = undefined;
const data = await getPageviewMetrics(websiteId, { const data = await getPageviewMetrics(websiteId, {
startDate, startDate,
endDate, endDate,

6
pages/api/websites/[id]/pageviews.ts
View file

@ -6,6 +6,7 @@ import { canViewWebsite } from 'lib/auth';
import { useAuth, useCors } from 'lib/middleware'; import { useAuth, useCors } from 'lib/middleware';
import { getPageviewStats } from 'queries'; import { getPageviewStats } from 'queries';
import { parseDateRangeQuery } from 'lib/query'; import { parseDateRangeQuery } from 'lib/query';
import { getSessionStats } from '../../../../queries/analytics/sessions/getSessionStats';
export interface WebsitePageviewRequestQuery { export interface WebsitePageviewRequestQuery {
id: string; id: string;
@ -62,7 +63,6 @@ export default async (
endDate, endDate,
timezone, timezone,
unit, unit,
count: '*',
filters: { filters: {
url, url,
referrer, referrer,
@ -75,14 +75,14 @@ export default async (
city, city,
}, },
}), }),
getPageviewStats(websiteId, { getSessionStats(websiteId, {
startDate, startDate,
endDate, endDate,
timezone, timezone,
unit, unit,
count: 'distinct website_event.',
filters: { filters: {
url, url,
referrer,
title, title,
os, os,
browser, browser,

4
queries/analytics/events/getEventMetrics.ts
View file

@ -47,11 +47,12 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite
order by 2 order by 2
`, `,
{ {
...filters,
websiteId, websiteId,
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.customEvent, eventType: EVENT_TYPE.customEvent,
...filters, domain: website.domain,
}, },
); );
} }
@ -82,6 +83,7 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.customEvent, eventType: EVENT_TYPE.customEvent,
domain: website.domain,
}, },
); );
} }

52
queries/analytics/pageviews/getPageviewMetrics.ts
View file

@ -34,22 +34,6 @@ async function relationalQuery(
const { startDate, endDate, filters = {}, column } = criteria; const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = prisma; const { rawQuery, parseFilters } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const params: any = {
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
...filters,
};
let excludeDomain = '';
if (column === 'referrer_domain') {
excludeDomain =
'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)';
params.domain = website.domain;
}
const { filterQuery, joinSession } = parseFilters(filters); const { filterQuery, joinSession } = parseFilters(filters);
@ -61,13 +45,19 @@ async function relationalQuery(
where website_event.website_id = {{websiteId::uuid}} where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}} and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}} and event_type = {{eventType}}
${excludeDomain}
${filterQuery} ${filterQuery}
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100 limit 100
`, `,
params, {
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }
@ -83,22 +73,8 @@ async function clickhouseQuery(
const { startDate, endDate, filters = {}, column } = criteria; const { startDate, endDate, filters = {}, column } = criteria;
const { rawQuery, parseFilters } = clickhouse; const { rawQuery, parseFilters } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const params = {
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: undefined,
};
let excludeDomain = ''; const { filterQuery } = parseFilters(filters);
if (column === 'referrer_domain') {
excludeDomain = 'and referrer_domain != {domain:String}';
params.domain = website.domain;
}
const { filterQuery } = parseFilters(filters, params);
return rawQuery( return rawQuery(
` `
@ -107,12 +83,18 @@ async function clickhouseQuery(
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery} ${filterQuery}
group by x group by x
order by y desc order by y desc
limit 100 limit 100
`, `,
params, {
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
domain: website.domain,
},
); );
} }

43
queries/analytics/pageviews/getPageviewStats.ts
View file

@ -10,9 +10,19 @@ export interface PageviewStatsCriteria {
endDate: Date; endDate: Date;
timezone?: string; timezone?: string;
unit?: string; unit?: string;
count?: string; filters: {
filters: object; url?: string;
sessionKey?: string; referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
} }
export async function getPageviewStats( export async function getPageviewStats(
@ -25,15 +35,7 @@ export async function getPageviewStats(
} }
async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) { async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const { const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
startDate,
endDate,
timezone = 'utc',
unit = 'day',
count = '*',
filters = {},
sessionKey = 'session_id',
} = criteria;
const { getDateQuery, parseFilters, rawQuery } = prisma; const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters); const { filterQuery, joinSession } = parseFilters(filters);
@ -42,7 +44,7 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
` `
select select
${getDateQuery('website_event.created_at', unit, timezone)} x, ${getDateQuery('website_event.created_at', unit, timezone)} x,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y count(*) y
from website_event from website_event
${joinSession} ${joinSession}
where website_event.website_id = {{websiteId::uuid}} where website_event.website_id = {{websiteId::uuid}}
@ -52,24 +54,18 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri
group by 1 group by 1
`, `,
{ {
...filters,
websiteId, websiteId,
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
...filters, domain: website.domain,
}, },
); );
} }
async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) { async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) {
const { const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
startDate,
endDate,
timezone = 'UTC',
unit = 'day',
count = '*',
filters = {},
} = criteria;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId); const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters); const { filterQuery } = parseFilters(filters);
@ -82,7 +78,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
from ( from (
select select
${getDateQuery('created_at', unit, timezone)} as t, ${getDateQuery('created_at', unit, timezone)} as t,
count(${count !== '*' ? 'distinct session_id' : count}) as y count(*) as y
from website_event from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime}
@ -98,6 +94,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
domain: website.domain,
}, },
); );
} }

84
queries/analytics/reports/getInsights.ts
View file

@ -1,11 +1,14 @@
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { maxDate } from 'lib/date';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
export interface GetInsightsCriteria { export interface GetInsightsCriteria {
startDate: Date; startDate: Date;
endDate: Date; endDate: Date;
fields: string[]; fields: { name: string; type: string; value: string }[];
filters: string[]; filters: string[];
groups: string[]; groups: string[];
} }
@ -26,7 +29,33 @@ async function relationalQuery(
y: number; y: number;
}[] }[]
> { > {
return null; const { startDate, endDate, filters = [] } = criteria;
const { parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const params = {};
const { filterQuery, joinSession } = parseFilters(params);
return rawQuery(
`
select
url_path,
count(*) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and website_event.event_type = {{eventType}}
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
} }
async function clickhouseQuery( async function clickhouseQuery(
@ -38,5 +67,54 @@ async function clickhouseQuery(
y: number; y: number;
}[] }[]
> { > {
return null; const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria;
const { parseFilters, rawQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const params = {};
const { filterQuery } = parseFilters(params);
const fieldsQuery = parseFields(fields);
return rawQuery(
`
select
${fieldsQuery}
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by ${fields.map(({ name }) => name).join(',')}
order by total desc
limit 500
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
},
);
}
function parseFields(fields) {
let count = false;
let distinct = false;
const query = fields.reduce((arr, field) => {
const { name, value } = field;
if (!count && value === 'total') {
count = true;
arr = arr.concat(`count(*) as views`);
} else if (!distinct && value === 'unique') {
distinct = true;
//arr = arr.concat(`count(distinct ${name})`);
}
return arr.concat(name);
}, []);
return query.join(',\n');
} }

15
queries/analytics/sessions/getSessionMetrics.ts
View file

@ -1,7 +1,7 @@
import prisma from 'lib/prisma'; import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse'; import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load'; import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date'; import { maxDate } from 'lib/date';
@ -28,17 +28,12 @@ async function relationalQuery(
return rawQuery( return rawQuery(
`select ${column} x, count(*) y `select ${column} x, count(*) y
from session as x
where x.session_id in (
select website_event.session_id
from website_event from website_event
join website ${joinSession}
on website_event.website_id = website.website_id where website_event.website_id = {{websiteId::uuid}}
${joinSession}
where website.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}} and website_event.created_at between {{startDate}} and {{endDate}}
${filterQuery} ${filterQuery}
) ) as t
group by 1 group by 1
order by 2 desc order by 2 desc
limit 100`, limit 100`,
@ -64,7 +59,7 @@ async function clickhouseQuery(
` `
select select
${column} x, count(distinct session_id) y ${column} x, count(distinct session_id) y
from website_event as x from website_event
where website_id = {websiteId:UUID} where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime} and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32} and event_type = {eventType:UInt32}

100
queries/analytics/sessions/getSessionStats.ts Normal file
View file

@ -0,0 +1,100 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { EVENT_TYPE } from 'lib/constants';
import { loadWebsite } from 'lib/load';
import { maxDate } from 'lib/date';
export interface SessionStatsCriteria {
startDate: Date;
endDate: Date;
timezone?: string;
unit?: string;
filters: {
url?: string;
referrer?: string;
title?: string;
browser?: string;
os?: string;
device?: string;
screen?: string;
language?: string;
country?: string;
region?: string;
city?: string;
};
}
export async function getSessionStats(
...args: [websiteId: string, criteria: SessionStatsCriteria]
) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) {
const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria;
const { getDateQuery, parseFilters, rawQuery } = prisma;
const website = await loadWebsite(websiteId);
const { filterQuery, joinSession } = parseFilters(filters);
return rawQuery(
`
select
${getDateQuery('website_event.created_at', unit, timezone)} x,
count(distinct website_event.session_id) y
from website_event
${joinSession}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
and event_type = {{eventType}}
${filterQuery}
group by 1
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
);
}
async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) {
const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const website = await loadWebsite(websiteId);
const { filterQuery } = parseFilters(filters);
return rawQuery(
`
select
${getDateStringQuery('g.t', unit)} as x,
g.y as y
from (
select
${getDateQuery('created_at', unit, timezone)} as t,
count(distinct session_id) as y
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
) as g
order by t
`,
{
...filters,
websiteId,
startDate: maxDate(startDate, website.resetAt),
endDate,
eventType: EVENT_TYPE.pageView,
domain: website.domain,
},
);
}

4
queries/analytics/stats/getWebsiteStats.ts
View file

@ -51,11 +51,12 @@ async function relationalQuery(
) as t ) as t
`, `,
{ {
...filters,
websiteId, websiteId,
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
...filters, domain: website.domain,
}, },
); );
} }
@ -97,6 +98,7 @@ async function clickhouseQuery(
startDate: maxDate(startDate, website.resetAt), startDate: maxDate(startDate, website.resetAt),
endDate, endDate,
eventType: EVENT_TYPE.pageView, eventType: EVENT_TYPE.pageView,
domain: website.domain,
}, },
); );
} }