Updated CI and Nightly pipelines with Spark 3.5, fixed incompatible t…

…ests
dotnet · Dec 29, 2024 · e4a378e · e4a378e
1 parent c7f4ae1
commit e4a378e
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 5 deletions.
diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml
@@ -31,7 +31,7 @@ variables:
   backwardCompatibleTestOptions_Linux_3_1: ""
   forwardCompatibleTestOptions_Linux_3_1: ""
 
-  # Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
+  # Skip all forward/backward compatibility tests since Spark 3.2 and 3.5 are not supported before this release.
   backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
   forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
@@ -41,6 +41,11 @@ variables:
   forwardCompatibleTestOptions_Windows_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
   backwardCompatibleTestOptions_Linux_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
   forwardCompatibleTestOptions_Linux_3_3: $(backwardCompatibleTestOptions_Windows_3_3)
+
+  backwardCompatibleTestOptions_Windows_3_5: "--filter FullyQualifiedName=NONE"
+  forwardCompatibleTestOptions_Windows_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  backwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  forwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
 
   # Azure DevOps variables are transformed into environment variables, with these variables we
   # avoid the first time experience and telemetry to speed up the build.
@@ -73,6 +78,11 @@ parameters:
   - '3.3.2'
   - '3.3.3'
   - '3.3.4'
+  - '3.5.0'
+  - '3.5.1'
+  - '3.5.2'
+  - '3.5.3'
+
 # List of OS types to run E2E tests, run each test in both 'Windows' and 'Linux' environments
 - name: listOfE2ETestsPoolTypes
   type: object

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -31,12 +31,17 @@ variables:
   backwardCompatibleTestOptions_Linux_3_1: ""
   forwardCompatibleTestOptions_Linux_3_1: ""
 
-  # Skip all forward/backward compatibility tests since Spark 3.2 is not supported before this release.
+  # Skip all forward/backward compatibility tests since Spark 3.2 and 3.5 are not supported before this release.
   backwardCompatibleTestOptions_Windows_3_2: "--filter FullyQualifiedName=NONE"
   forwardCompatibleTestOptions_Windows_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   backwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
   forwardCompatibleTestOptions_Linux_3_2: $(backwardCompatibleTestOptions_Windows_3_2)
 
+  backwardCompatibleTestOptions_Windows_3_5: "--filter FullyQualifiedName=NONE"
+  forwardCompatibleTestOptions_Windows_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  backwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+  forwardCompatibleTestOptions_Linux_3_5: $(backwardCompatibleTestOptions_Windows_3_5)
+
   # Azure DevOps variables are transformed into environment variables, with these variables we
   # avoid the first time experience and telemetry to speed up the build.
   DOTNET_CLI_TELEMETRY_OPTOUT: 1
@@ -413,3 +418,51 @@ stages:
         testOptions: ""
         backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_2)
         forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_2)
+    - version: '3.5.0'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.1'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.2'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
+    - version: '3.5.3'
+      enableForwardCompatibleTests: false
+      enableBackwardCompatibleTests: false
+      jobOptions:
+      - pool: 'Windows'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_3_5)
+      - pool: 'Linux'
+        testOptions: ""
+        backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_5)
+        forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_5)
diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/SparkContextTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/SparkContextTests.cs
@@ -57,16 +57,22 @@ public void TestSignaturesV2_4_X()
 
         /// <summary>
         /// Test signatures for APIs introduced in Spark 3.1.*.
+        /// In Spark 3.5 Spark throws an exception when trying to delete
+        /// archive.zip from temp folder, and causes failures of other tests
         /// </summary>
-        [SkipIfSparkVersionIsLessThan(Versions.V3_1_0)]
+        [SkipIfSparkVersionIsNotInRange(Versions.V3_1_0, Versions.V3_3_0)]
         public void TestSignaturesV3_1_X()
         {
             SparkContext sc = SparkContext.GetOrCreate(new SparkConf());
 
             string archivePath = $"{TestEnvironment.ResourceDirectory}archive.zip";
+
             sc.AddArchive(archivePath);
 
-            Assert.IsType<string[]>(sc.ListArchives().ToArray());
+            var archives = sc.ListArchives().ToArray();
+
+            Assert.IsType<string[]>(archives);
+            Assert.NotEmpty(archives.Where(a => a.EndsWith("archive.zip")));
         }
     }
 }
diff --git a/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/CatalogTests.cs b/src/csharp/Microsoft.Spark.E2ETest/IpcTests/Sql/CatalogTests.cs
@@ -59,7 +59,6 @@ public void TestSignaturesV2_4_X()
                 Assert.IsType<bool>(catalog.FunctionExists("functionname"));
                 Assert.IsType<Database>(catalog.GetDatabase("default"));
                 Assert.IsType<Function>(catalog.GetFunction("abs"));
-                Assert.IsType<Function>(catalog.GetFunction(null, "abs"));
                 Assert.IsType<Table>(catalog.GetTable("users"));
                 Assert.IsType<Table>(catalog.GetTable("default", "users"));
                 Assert.IsType<bool>(catalog.IsCached("users"));

diff --git a/src/csharp/Microsoft.Spark.Worker.UnitTest/PayloadWriter.cs b/src/csharp/Microsoft.Spark.Worker.UnitTest/PayloadWriter.cs
@@ -351,6 +351,7 @@ internal PayloadWriter Create(Version version = null)
                         new BroadcastVariableWriterV2_4_X(),
                         new CommandWriterV2_4_X());
                 case Versions.V3_3_0:
+                case Versions.V3_5_1:
                     return new PayloadWriter(
                         version,
                         new TaskContextWriterV3_3_X(),