From 1bdaf4aca2c22466b8c8f7acd3275997fbad7807 Mon Sep 17 00:00:00 2001
From: Brian Cao <brian@umami.is>
Date: Mon, 10 Jul 2023 15:30:34 -0700
Subject: [PATCH 1/3] Always record string.

---
 lib/kafka.ts                                 |  4 ++--
 queries/analytics/eventData/saveEventData.ts | 15 +++------------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/lib/kafka.ts b/lib/kafka.ts
index 3d3e281c..8f5bb87f 100644
--- a/lib/kafka.ts
+++ b/lib/kafka.ts
@@ -61,8 +61,8 @@ async function getProducer(): Promise<Producer> {
   return producer;
 }
 
-function getDateFormat(date): string {
-  return dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss');
+function getDateFormat(date, format?): string {
+  return dateFormat(date, format ? format : 'UTC:yyyy-mm-dd HH:MM:ss');
 }
 
 async function sendMessage(
diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts
index 0ccc9421..9a7aa406 100644
--- a/queries/analytics/eventData/saveEventData.ts
+++ b/queries/analytics/eventData/saveEventData.ts
@@ -36,13 +36,8 @@ async function relationalQuery(data: {
     id: uuid(),
     websiteEventId: eventId,
     websiteId,
-    key: a.key,
-    stringValue:
-      a.dynamicDataType === DATA_TYPE.string ||
-      a.dynamicDataType === DATA_TYPE.boolean ||
-      a.dynamicDataType === DATA_TYPE.array
-        ? a.value
-        : null,
+    eventKey: a.key,
+    stringValue: a.value.toString(),
     numericValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
     dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null,
     dataType: a.dynamicDataType,
@@ -76,11 +71,7 @@ async function clickhouseQuery(data: {
     event_name: eventName,
     event_key: a.key,
     string_value:
-      a.dynamicDataType === DATA_TYPE.string ||
-      a.dynamicDataType === DATA_TYPE.boolean ||
-      a.dynamicDataType === DATA_TYPE.array
-        ? a.value
-        : null,
+      a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value, 'isoDateTime') : a.value,
     number_value: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
     date_value: a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value) : null,
     data_type: a.dynamicDataType,

From 3749e2a70d1db7ddec837d119be438afcd79a587 Mon Sep 17 00:00:00 2001
From: Brian Cao <brian@umami.is>
Date: Mon, 10 Jul 2023 16:54:42 -0700
Subject: [PATCH 2/3] update to isoutc string value.

---
 db/clickhouse/migrations/01_edit_keys.sql    | 13 ++++++++++++-
 queries/analytics/eventData/saveEventData.ts |  2 +-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/db/clickhouse/migrations/01_edit_keys.sql b/db/clickhouse/migrations/01_edit_keys.sql
index 952151ec..483a5650 100644
--- a/db/clickhouse/migrations/01_edit_keys.sql
+++ b/db/clickhouse/migrations/01_edit_keys.sql
@@ -6,4 +6,15 @@ ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";
 
 -- add job_id
 ALTER TABLE "website_event" ADD COLUMN "job_id" UUID AFTER "created_at";
-ALTER TABLE "event_data" ADD COLUMN "job_id" UUID AFTER "created_at";
\ No newline at end of file
+ALTER TABLE "event_data" ADD COLUMN "job_id" UUID AFTER "created_at";
+
+-- update event_data string
+alter table umami.event_data
+update string_value = number_value
+where number_value is not null
+    and string_value is null;
+
+alter table umami.event_data
+update string_value = replaceOne(concat(CAST(toDateTime(date_value, 'UTC'), 'String'),'Z'), ' ', 'T')
+where date_value is not null
+    and string_value is null;
\ No newline at end of file
diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.ts
index 9a7aa406..c889955f 100644
--- a/queries/analytics/eventData/saveEventData.ts
+++ b/queries/analytics/eventData/saveEventData.ts
@@ -71,7 +71,7 @@ async function clickhouseQuery(data: {
     event_name: eventName,
     event_key: a.key,
     string_value:
-      a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value, 'isoDateTime') : a.value,
+      a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value, 'isoUtcDateTime') : a.value,
     number_value: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
     date_value: a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value) : null,
     data_type: a.dynamicDataType,

From 1a793572e846ef2ccd4e0107dacb7415c01e5711 Mon Sep 17 00:00:00 2001
From: Brian Cao <brian@umami.is>
Date: Mon, 10 Jul 2023 22:29:38 -0700
Subject: [PATCH 3/3] update backfill script.

---
 db/clickhouse/migrations/01_edit_keys.sql | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/db/clickhouse/migrations/01_edit_keys.sql b/db/clickhouse/migrations/01_edit_keys.sql
index 483a5650..3fc7dd79 100644
--- a/db/clickhouse/migrations/01_edit_keys.sql
+++ b/db/clickhouse/migrations/01_edit_keys.sql
@@ -11,10 +11,8 @@ ALTER TABLE "event_data" ADD COLUMN "job_id" UUID AFTER "created_at";
 -- update event_data string
 alter table umami.event_data
 update string_value = number_value
-where number_value is not null
-    and string_value is null;
+where data_type = 2
 
 alter table umami.event_data
 update string_value = replaceOne(concat(CAST(toDateTime(date_value, 'UTC'), 'String'),'Z'), ' ', 'T')
-where date_value is not null
-    and string_value is null;
\ No newline at end of file
+where data_type = 4
\ No newline at end of file