Fix performance of building row-level results (#577)

marcantony · web-flow · commit 3b1a3ec5d1aa · 2024-08-31T12:40:11.000-04:00
* Generate row-level results with withColumns

Iteratively using withColumn (singular) causes performance
issues when iterating over a large sequence of columns.

* Add back UNIQUENESS_ID
diff --git a/src/main/scala/com/amazon/deequ/VerificationResult.scala b/src/main/scala/com/amazon/deequ/VerificationResult.scala
@@ -98,9 +98,7 @@ object VerificationResult {
     val columnNamesToMetrics: Map[String, Column] = verificationResultToColumn(verificationResult)
 
     val dataWithID = data.withColumn(UNIQUENESS_ID, monotonically_increasing_id())
-    columnNamesToMetrics.foldLeft(dataWithID)(
-      (dataWithID, newColumn: (String, Column)) =>
-        dataWithID.withColumn(newColumn._1, newColumn._2)).drop(UNIQUENESS_ID)
+    dataWithID.withColumns(columnNamesToMetrics).drop(UNIQUENESS_ID)
   }
 
   def checkResultsAsJson(verificationResult: VerificationResult,

Original file line number	Diff line number	Diff line change
`@@ -98,9 +98,7 @@ object VerificationResult {`
`98`	`98`	`val columnNamesToMetrics: Map[String, Column] = verificationResultToColumn(verificationResult)`
`99`	`99`
`100`	`100`	`val dataWithID = data.withColumn(UNIQUENESS_ID, monotonically_increasing_id())`
`101`		`- columnNamesToMetrics.foldLeft(dataWithID)(`
`102`		`- (dataWithID, newColumn: (String, Column)) =>`
`103`		`- dataWithID.withColumn(newColumn._1, newColumn._2)).drop(UNIQUENESS_ID)`
	`101`	`+ dataWithID.withColumns(columnNamesToMetrics).drop(UNIQUENESS_ID)`
`104`	`102`	`}`
`105`	`103`
`106`	`104`	`def checkResultsAsJson(verificationResult: VerificationResult,`