bugfix: fix json parsing if there are newlines

devflowinc · Aug 6, 2024 · 469b4d7 · 469b4d7
1 parent 4b73773
commit 469b4d7
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 7 deletions.
diff --git a/server/src/data/models.rs b/server/src/data/models.rs
@@ -3856,17 +3856,26 @@ impl From<SearchQueryEventClickhouse> for SearchQueryEvent {
         SearchQueryEvent {
             id: uuid::Uuid::from_bytes(*clickhouse_response.id.as_bytes()),
             search_type: clickhouse_response.search_type,
-            query: clickhouse_response.query.replace("|q", "?"),
+            query: clickhouse_response
+                .query
+                .replace("|q", "?")
+                .replace('\n', ""),
             request_params: serde_json::from_str(
-                &clickhouse_response.request_params.replace("|q", "?"),
+                &clickhouse_response
+                    .request_params
+                    .replace("|q", "?")
+                    .replace('\n', ""),
             )
             .unwrap_or_default(),
             latency: clickhouse_response.latency,
             top_score: clickhouse_response.top_score,
             results: clickhouse_response
                 .results
                 .iter()
-                .map(|r| serde_json::from_str(&r.replace("|q", "?")).unwrap_or_default())
+                .map(|r| {
+                    serde_json::from_str(&r.replace("|q", "?").replace('\n', ""))
+                        .unwrap_or_default()
+                })
                 .collect::<Vec<serde_json::Value>>(),
             dataset_id: uuid::Uuid::from_bytes(*clickhouse_response.dataset_id.as_bytes()),
             created_at: clickhouse_response.created_at.to_string(),

diff --git a/server/src/operators/clickhouse_operator.rs b/server/src/operators/clickhouse_operator.rs
@@ -72,10 +72,12 @@ pub async fn send_to_clickhouse(
     for event in events {
         match event {
             ClickHouseEvent::SearchQueryEvent(mut event) => {
-                event
-                    .results
-                    .iter_mut()
-                    .for_each(|result| *result = result.replace('\'', "''").replace('?', "|q"));
+                event.results.iter_mut().for_each(|result| {
+                    *result = result
+                        .replace('\'', "''")
+                        .replace('?', "|q")
+                        .replace('\n', "")
+                });
                 search_queries_inserter.push_str(&format!(
                     " ('{}', '{}', '{}', '{}', embed_p('{}'), '{}', '{}', ['{}'], '{}', now()),",
                     event.id,