feat: ✨ add CSV repair option and test case

Author: pelikhan

docs/src/content/docs/reference/scripts/csv.md

@@ -3,10 +3,9 @@ title: CSV
 description: Learn how to parse and stringify CSV data using the CSV class in scripting.
 keywords: CSV parsing, CSV stringifying, CSV data, CSV manipulation, CSV utility
 sidebar:
-  order: 17
+    order: 17
 genaiscript:
-  files: src/samples/penguins.csv
-
+    files: src/samples/penguins.csv
 ---
 
 Parsing and stringifying of Comma Separated Values (CSV) data.
@@ -24,18 +23,18 @@ maps to the following array of objects:
 
 ```json
 [
-  {
-    "name": "A",
-    "value": 10
-  },
-  {
-    "name": "B",
-    "value": 2
-  },
-  {
-    "name": "C",
-    "value": 3
-  }
+    {
+        "name": "A",
+        "value": 10
+    },
+    {
+        "name": "B",
+        "value": 2
+    },
+    {
+        "name": "C",
+        "value": 3
+    }
 ]
 ```
 
@@ -51,9 +50,9 @@ def("DATA", env.files[0])
 
 ```js assistant=false
 def("DATA", env.files[0], {
-  sliceHead: 50, // take first 50
-  sliceTail: 25, // take last 25
-  sliceSample: 5 // take 5 at random
+    sliceHead: 50, // take first 50
+    sliceTail: 25, // take last 25
+    sliceSample: 5, // take 5 at random
 })
 ```
 
@@ -75,7 +74,7 @@ If the CSV file does not have a header row, you can specify the column names as
 ```js
 const rows = CSV.parse(csv, {
     delimiter: "|",
-    headers: ["name", "value"]
+    headers: ["name", "value"],
 })
 ```
 
@@ -122,3 +121,12 @@ The [parsers](/genaiscript/reference/scripts/parsers) also provide a parser for
 ```js
 const rows = parsers.CSV(env.files[0])
 ```
+
+
+## Repair
+
+You can specify the `repair: true` option to fix common LLM mistakes around CSV.
+
+```js
+const rows = CSV.parse(csv, { repair: true })
+```

packages/core/src/csv.test.ts

@@ -39,6 +39,14 @@ describe("CSVParse", () => {
             { name: "Jane", age: "25" },
         ])
     })
+    test("Parse CSV data with invalid quotes", () => {
+        const csv = "\"\\\"John\\\"\",30\nJane,25"
+        const result = CSVParse(csv, { headers: ["name", "age"], repair: true })
+        assert.deepEqual(result, [
+            { name: "\"John\"", age: "30" },
+            { name: "Jane", age: "25" },
+        ])
+    })
 
 })
 

packages/core/src/csv.ts

@@ -20,11 +20,17 @@ export function CSVParse(
     options?: {
         delimiter?: string
         headers?: ElementOrArray<string>
+        repair?: boolean
     }
 ): object[] {
     // Destructure options or provide defaults
-    const { delimiter, headers, ...rest } = options || {}
+    const { delimiter, headers, repair, ...rest } = options || {}
     const columns = headers ? arrayify(headers) : true
+
+    // common LLM escape errors
+    if (repair) {
+        text = text.replace(/\\"/g, '""').replace(/""""/g, '""')
+    }
     // Parse the CSV string based on the provided options
     return parse(text, {
         autoParse: true, // Automatically parse values to appropriate types
@@ -56,6 +62,7 @@ export function CSVTryParse(
     options?: {
         delimiter?: string
         headers?: ElementOrArray<string>
+        repair?: boolean
     } & TraceOptions
 ): object[] | undefined {
     const { trace } = options || {}

packages/core/src/types/prompt_template.d.ts

@@ -1097,6 +1097,7 @@ interface Tokenizer {
 interface CSVParseOptions {
     delimiter?: string
     headers?: string[]
+    repair?: boolean
 }
 
 interface TextChunk extends WorkspaceFile {
@@ -1939,13 +1940,7 @@ interface CSV {
      * @param options.headers - An array of headers to use. If not provided, headers will be inferred from the first row.
      * @returns An array of objects representing the parsed CSV data.
      */
-    parse(
-        text: string,
-        options?: {
-            delimiter?: string
-            headers?: string[]
-        }
-    ): object[]
+    parse(text: string, options?: CSVParseOptions): object[]
 
     /**
      * Converts an array of objects to a CSV string.