From 9c24ba964fdf4b1bc7a74dc0b9d1e1a3ab3dcb63 Mon Sep 17 00:00:00 2001 From: prathit06 Date: Fri, 31 Jan 2025 10:48:07 +0530 Subject: [PATCH 1/5] SPARK-51048 | Support stop java spark context with exit code --- .../scala/org/apache/spark/api/java/JavaSparkContext.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 7311ef296363d..97fb5a9053d81 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -552,6 +552,11 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable { sc.stop() } + /** Shut down the SparkContext with exit code. */ + def stop(exitCode: Int): Unit = { + sc.stop(exitCode) + } + override def close(): Unit = stop() /** From bab84b8a659c55744f2959dbe1d8ddfce8a0230c Mon Sep 17 00:00:00 2001 From: prathit06 Date: Mon, 3 Feb 2025 13:24:23 +0530 Subject: [PATCH 2/5] SPARK-51048 | Support stop java spark context with exit code --- .../org/apache/spark/api/java/JavaSparkContext.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 97fb5a9053d81..74092eb138bd6 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -547,12 +547,16 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable { */ def broadcast[T](value: T): Broadcast[T] = sc.broadcast(value)(fakeClassTag) - /** Shut down the SparkContext. */ + /** Shut down the JavaSparkContext. */ def stop(): Unit = { sc.stop() } - /** Shut down the SparkContext with exit code. */ + /** + * Shut down the `JavaSparkContext` with given exit code that will be passed to scheduler backend. + * + * @param exitCode Specified exit code that will passed to scheduler backend in client mode. + */ def stop(exitCode: Int): Unit = { sc.stop(exitCode) } From b500324c94b069a5917a59c53da61e7d521d8262 Mon Sep 17 00:00:00 2001 From: prathit06 Date: Mon, 3 Feb 2025 13:25:01 +0530 Subject: [PATCH 3/5] SPARK-51048 | Support stop java spark context with exit code --- .../main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 74092eb138bd6..5d14ae92f111a 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -555,7 +555,7 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable { /** * Shut down the `JavaSparkContext` with given exit code that will be passed to scheduler backend. * - * @param exitCode Specified exit code that will passed to scheduler backend in client mode. + * @param exitCode Specified exit code that will be passed to scheduler backend in client mode. */ def stop(exitCode: Int): Unit = { sc.stop(exitCode) From 68d4c33edc5bc364a7b23f241b98c89b14cd59cf Mon Sep 17 00:00:00 2001 From: prathit06 Date: Mon, 3 Feb 2025 13:26:41 +0530 Subject: [PATCH 4/5] SPARK-51048 | Support stop java spark context with exit code --- .../scala/org/apache/spark/api/java/JavaSparkContext.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 5d14ae92f111a..1bf5930922a7a 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -547,13 +547,13 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable { */ def broadcast[T](value: T): Broadcast[T] = sc.broadcast(value)(fakeClassTag) - /** Shut down the JavaSparkContext. */ + /** Shut down the SparkContext. */ def stop(): Unit = { sc.stop() } /** - * Shut down the `JavaSparkContext` with given exit code that will be passed to scheduler backend. + * Shut down the `SparkContext` with given exit code that will be passed to scheduler backend. * * @param exitCode Specified exit code that will be passed to scheduler backend in client mode. */ From 2cce3c3aea6ed79c1ab5bc59fc5aa80702c3ebbb Mon Sep 17 00:00:00 2001 From: prathit06 Date: Tue, 4 Feb 2025 13:48:10 +0530 Subject: [PATCH 5/5] SPARK-51048 | updated javadoc --- .../org/apache/spark/api/java/JavaSparkContext.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala index 1bf5930922a7a..bb8b02616c93f 100644 --- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala +++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala @@ -553,9 +553,15 @@ class JavaSparkContext(val sc: SparkContext) extends Closeable { } /** - * Shut down the `SparkContext` with given exit code that will be passed to scheduler backend. - * - * @param exitCode Specified exit code that will be passed to scheduler backend in client mode. + * Shut down the SparkContext with exit code that will passed to scheduler backend. + * In client mode, client side may call `SparkContext.stop()` to clean up but exit with + * code not equal to 0. This behavior cause resource scheduler such as `ApplicationMaster` + * exit with success status but client side exited with failed status. Spark can call + * this method to stop SparkContext and pass client side correct exit code to scheduler backend. + * Then scheduler backend should send the exit code to corresponding resource scheduler + * to keep consistent. + * + * @param exitCode Specified exit code that will passed to scheduler backend in client mode. */ def stop(exitCode: Int): Unit = { sc.stop(exitCode)