From 8abeba4c0dddf5af92e066024bc212a1e8e8b4bb Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 10 Jul 2023 23:08:57 -0700 Subject: [PATCH 1/4] squash psql migration. add event_data migration --- components/pages/console/TestConsole.js | 1 + .../migrations/02_report_schema/migration.sql | 29 -------------- .../migration.sql | 39 +++++++++++++++++++ queries/analytics/eventData/saveEventData.ts | 5 ++- queries/analytics/session/saveSessionData.ts | 8 +--- yarn.lock | 9 ++++- 6 files changed, 53 insertions(+), 38 deletions(-) delete mode 100644 db/postgresql/migrations/02_report_schema/migration.sql rename db/postgresql/migrations/{03_session_data => 02_report_schema_session_data}/migration.sql (52%) diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.js index 3e907856..4f167b9a 100644 --- a/components/pages/console/TestConsole.js +++ b/components/pages/console/TestConsole.js @@ -33,6 +33,7 @@ export function TestConsole() { booleanError: 'true', time: new Date(), number: 1, + number2: Math.random() * 100, time2: new Date().toISOString(), nested: { test: 'test-data', diff --git a/db/postgresql/migrations/02_report_schema/migration.sql b/db/postgresql/migrations/02_report_schema/migration.sql deleted file mode 100644 index f4f3e885..00000000 --- a/db/postgresql/migrations/02_report_schema/migration.sql +++ /dev/null @@ -1,29 +0,0 @@ --- CreateTable -CREATE TABLE "report" ( - "report_id" UUID NOT NULL, - "user_id" UUID NOT NULL, - "website_id" UUID NOT NULL, - "type" VARCHAR(200) NOT NULL, - "name" VARCHAR(200) NOT NULL, - "description" VARCHAR(500) NOT NULL, - "parameters" VARCHAR(6000) NOT NULL, - "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - "updated_at" TIMESTAMPTZ(6), - - CONSTRAINT "report_pkey" PRIMARY KEY ("report_id") -); - --- CreateIndex -CREATE UNIQUE INDEX "report_report_id_key" ON "report"("report_id"); - --- CreateIndex -CREATE INDEX "report_user_id_idx" ON "report"("user_id"); - --- CreateIndex -CREATE INDEX "report_website_id_idx" ON "report"("website_id"); - --- CreateIndex -CREATE INDEX "report_type_idx" ON "report"("type"); - --- CreateIndex -CREATE INDEX "report_name_idx" ON "report"("name"); diff --git a/db/postgresql/migrations/03_session_data/migration.sql b/db/postgresql/migrations/02_report_schema_session_data/migration.sql similarity index 52% rename from db/postgresql/migrations/03_session_data/migration.sql rename to db/postgresql/migrations/02_report_schema_session_data/migration.sql index 0a93d39c..41c907cd 100644 --- a/db/postgresql/migrations/03_session_data/migration.sql +++ b/db/postgresql/migrations/02_report_schema_session_data/migration.sql @@ -21,6 +21,21 @@ CREATE TABLE "session_data" ( CONSTRAINT "session_data_pkey" PRIMARY KEY ("session_data_id") ); +-- CreateTable +CREATE TABLE "report" ( + "report_id" UUID NOT NULL, + "user_id" UUID NOT NULL, + "website_id" UUID NOT NULL, + "type" VARCHAR(200) NOT NULL, + "name" VARCHAR(200) NOT NULL, + "description" VARCHAR(500) NOT NULL, + "parameters" VARCHAR(6000) NOT NULL, + "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(6), + + CONSTRAINT "report_pkey" PRIMARY KEY ("report_id") +); + -- CreateIndex CREATE INDEX "session_data_created_at_idx" ON "session_data"("created_at"); @@ -29,3 +44,27 @@ CREATE INDEX "session_data_website_id_idx" ON "session_data"("website_id"); -- CreateIndex CREATE INDEX "session_data_session_id_idx" ON "session_data"("session_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "report_report_id_key" ON "report"("report_id"); + +-- CreateIndex +CREATE INDEX "report_user_id_idx" ON "report"("user_id"); + +-- CreateIndex +CREATE INDEX "report_website_id_idx" ON "report"("website_id"); + +-- CreateIndex +CREATE INDEX "report_type_idx" ON "report"("type"); + +-- CreateIndex +CREATE INDEX "report_name_idx" ON "report"("name"); + +-- EventData migration +UPDATE "event_data" +SET string_value = number_value +WHERE data_type = 2; + +UPDATE "event_data" +SET string_value = CONCAT(REPLACE(TO_CHAR(date_value, 'YYYY-MM-DD HH24:MI:SS:MS'), ' ', 'T'), 'Z') +WHERE data_type = 4; \ No newline at end of file diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index c889955f..f7cf06ee 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -37,8 +37,9 @@ async function relationalQuery(data: { websiteEventId: eventId, websiteId, eventKey: a.key, - stringValue: a.value.toString(), - numericValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, + stringValue: + a.dynamicDataType === DATA_TYPE.number ? parseFloat(a.value).toFixed(4) : a.value.toString(), + numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dynamicDataType, })); diff --git a/queries/analytics/session/saveSessionData.ts b/queries/analytics/session/saveSessionData.ts index 79104270..312f1f53 100644 --- a/queries/analytics/session/saveSessionData.ts +++ b/queries/analytics/session/saveSessionData.ts @@ -20,12 +20,8 @@ export async function saveSessionData(data: { sessionId, key: a.key, stringValue: - a.dynamicDataType === DATA_TYPE.string || - a.dynamicDataType === DATA_TYPE.boolean || - a.dynamicDataType === DATA_TYPE.array - ? a.value - : null, - numericValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, + a.dynamicDataType === DATA_TYPE.number ? parseFloat(a.value).toFixed(4) : a.value.toString(), + numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dynamicDataType, })); diff --git a/yarn.lock b/yarn.lock index 7dafda26..9d06a5c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8048,13 +8048,20 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== -semver@^7.3.4, semver@^7.3.5, semver@^7.3.6, semver@^7.3.7, semver@^7.3.8: +semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8: version "7.5.2" resolved "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz" integrity sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ== dependencies: lru-cache "^6.0.0" +semver@^7.5.2: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + serialize-javascript@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz" From 9726d02dcd530fafd61e5f026db621b3a2c5bd41 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 11 Jul 2023 08:55:58 -0700 Subject: [PATCH 2/4] remove ms from string date value, add mysql migration --- .../migrations/02_report_schema/migration.sql | 19 --------- .../migration.sql | 42 +++++++++++++------ .../migration.sql | 2 +- queries/analytics/eventData/saveEventData.ts | 6 ++- queries/analytics/session/saveSessionData.ts | 6 ++- 5 files changed, 40 insertions(+), 35 deletions(-) delete mode 100644 db/mysql/migrations/02_report_schema/migration.sql rename db/mysql/migrations/{03_session_data => 02_report_schema_session_data}/migration.sql (50%) diff --git a/db/mysql/migrations/02_report_schema/migration.sql b/db/mysql/migrations/02_report_schema/migration.sql deleted file mode 100644 index 0476e9f7..00000000 --- a/db/mysql/migrations/02_report_schema/migration.sql +++ /dev/null @@ -1,19 +0,0 @@ --- CreateTable -CREATE TABLE `report` ( - `report_id` VARCHAR(36) NOT NULL, - `user_id` VARCHAR(36) NOT NULL, - `website_id` VARCHAR(36) NOT NULL, - `type` VARCHAR(200) NOT NULL, - `name` VARCHAR(200) NOT NULL, - `description` VARCHAR(500) NOT NULL, - `parameters` VARCHAR(6000) NOT NULL, - `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), - `updated_at` TIMESTAMP(0) NULL, - - UNIQUE INDEX `report_report_id_key`(`report_id`), - INDEX `report_user_id_idx`(`user_id`), - INDEX `report_website_id_idx`(`website_id`), - INDEX `report_type_idx`(`type`), - INDEX `report_name_idx`(`name`), - PRIMARY KEY (`report_id`) -) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; diff --git a/db/mysql/migrations/03_session_data/migration.sql b/db/mysql/migrations/02_report_schema_session_data/migration.sql similarity index 50% rename from db/mysql/migrations/03_session_data/migration.sql rename to db/mysql/migrations/02_report_schema_session_data/migration.sql index 161b455f..49708899 100644 --- a/db/mysql/migrations/03_session_data/migration.sql +++ b/db/mysql/migrations/02_report_schema_session_data/migration.sql @@ -1,16 +1,3 @@ -/* - Warnings: - - - The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint. - - You are about to drop the column `event_data_type` on the `event_data` table. All the data in the column will be lost. - - You are about to drop the column `event_date_value` on the `event_data` table. All the data in the column will be lost. - - You are about to drop the column `event_id` on the `event_data` table. All the data in the column will be lost. - - You are about to drop the column `event_numeric_value` on the `event_data` table. All the data in the column will be lost. - - You are about to drop the column `event_string_value` on the `event_data` table. All the data in the column will be lost. - - Added the required column `data_type` to the `event_data` table without a default value. This is not possible if the table is not empty. - - Added the required column `event_data_id` to the `event_data` table without a default value. This is not possible if the table is not empty. - -*/ -- AlterTable ALTER TABLE `event_data` RENAME COLUMN `event_data_type` TO `data_type`; ALTER TABLE `event_data` RENAME COLUMN `event_date_value` TO `date_value`; @@ -35,3 +22,32 @@ CREATE TABLE `session_data` ( INDEX `session_data_session_id_idx`(`session_id`), PRIMARY KEY (`session_data_id`) ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- CreateTable +CREATE TABLE `report` ( + `report_id` VARCHAR(36) NOT NULL, + `user_id` VARCHAR(36) NOT NULL, + `website_id` VARCHAR(36) NOT NULL, + `type` VARCHAR(200) NOT NULL, + `name` VARCHAR(200) NOT NULL, + `description` VARCHAR(500) NOT NULL, + `parameters` VARCHAR(6000) NOT NULL, + `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0), + `updated_at` TIMESTAMP(0) NULL, + + UNIQUE INDEX `report_report_id_key`(`report_id`), + INDEX `report_user_id_idx`(`user_id`), + INDEX `report_website_id_idx`(`website_id`), + INDEX `report_type_idx`(`type`), + INDEX `report_name_idx`(`name`), + PRIMARY KEY (`report_id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- EventData migration +UPDATE event_data +SET string_value = number_value +WHERE data_type = 2; + +UPDATE event_data +SET string_value = CONCAT(REPLACE(DATE_FORMAT(date_value, '%Y-%m-%d %T'), ' ', 'T'), 'Z') +WHERE data_type = 4; \ No newline at end of file diff --git a/db/postgresql/migrations/02_report_schema_session_data/migration.sql b/db/postgresql/migrations/02_report_schema_session_data/migration.sql index 41c907cd..5fe6ef9d 100644 --- a/db/postgresql/migrations/02_report_schema_session_data/migration.sql +++ b/db/postgresql/migrations/02_report_schema_session_data/migration.sql @@ -66,5 +66,5 @@ SET string_value = number_value WHERE data_type = 2; UPDATE "event_data" -SET string_value = CONCAT(REPLACE(TO_CHAR(date_value, 'YYYY-MM-DD HH24:MI:SS:MS'), ' ', 'T'), 'Z') +SET string_value = CONCAT(REPLACE(TO_CHAR(date_value, 'YYYY-MM-DD HH24:MI:SS'), ' ', 'T'), 'Z') WHERE data_type = 4; \ No newline at end of file diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts index f7cf06ee..f30ceb90 100644 --- a/queries/analytics/eventData/saveEventData.ts +++ b/queries/analytics/eventData/saveEventData.ts @@ -38,7 +38,11 @@ async function relationalQuery(data: { websiteId, eventKey: a.key, stringValue: - a.dynamicDataType === DATA_TYPE.number ? parseFloat(a.value).toFixed(4) : a.value.toString(), + a.dynamicDataType === DATA_TYPE.number + ? parseFloat(a.value).toFixed(4) + : a.dynamicDataType === DATA_TYPE.date + ? a.value.split('.')[0] + 'Z' + : a.value.toString(), numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dynamicDataType, diff --git a/queries/analytics/session/saveSessionData.ts b/queries/analytics/session/saveSessionData.ts index 312f1f53..192053f1 100644 --- a/queries/analytics/session/saveSessionData.ts +++ b/queries/analytics/session/saveSessionData.ts @@ -20,7 +20,11 @@ export async function saveSessionData(data: { sessionId, key: a.key, stringValue: - a.dynamicDataType === DATA_TYPE.number ? parseFloat(a.value).toFixed(4) : a.value.toString(), + a.dynamicDataType === DATA_TYPE.number + ? parseFloat(a.value).toFixed(4) + : a.dynamicDataType === DATA_TYPE.date + ? a.value.split('.')[0] + 'Z' + : a.value.toString(), numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null, dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null, dataType: a.dynamicDataType, From 2baf952ab844330da1dd01a1ec320c38d4e0f166 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 11 Jul 2023 12:00:01 -0700 Subject: [PATCH 3/4] re-add strict-order. --- queries/analytics/pageview/getPageviewFunnel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index 7305e545..088a94a5 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -91,7 +91,7 @@ async function clickhouseQuery( count(*) AS count FROM ( SELECT session_id, - windowFunnel({window:UInt32}) + windowFunnel({window:UInt32}, 'strict_order') ( created_at ${columnsQuery} From 00a2137d0d235526f306b4c500a623e1b9becf38 Mon Sep 17 00:00:00 2001 From: Brian Cao Date: Tue, 11 Jul 2023 12:43:43 -0700 Subject: [PATCH 4/4] strict_increase --- queries/analytics/pageview/getPageviewFunnel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index 088a94a5..bfd921c1 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -91,7 +91,7 @@ async function clickhouseQuery( count(*) AS count FROM ( SELECT session_id, - windowFunnel({window:UInt32}, 'strict_order') + windowFunnel({window:UInt32}, 'strict_increase') ( created_at ${columnsQuery}