This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 360adc2250b [SPARK-45197][CORE] Make `StandaloneRestServer` add `JavaModuleOptions` to drivers 360adc2250b is described below commit 360adc2250bccdb0fbe559dcd1fc4b6b4c7c1d7a Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Mon Sep 18 04:10:22 2023 -0700 [SPARK-45197][CORE] Make `StandaloneRestServer` add `JavaModuleOptions` to drivers ### What changes were proposed in this pull request? This PR aims to make `StandaloneRestServer` add `JavaModuleOptions` to drivers by default. ### Why are the changes needed? Since Apache Spark 3.3.0 (SPARK-36796, #34153), `SparkContext` adds `JavaModuleOptions` by default. We had better add `JavaModuleOptions` when `StandaloneRestServer` receives submissions via REST API, too. Otherwise, it fails like the following if the users don't set it manually. **SUBMISSION** ```bash $ SPARK_MASTER_OPTS="-Dspark.master.rest.enabled=true" sbin/start-master.sh $ curl -s -k -XPOST http://yourserver:6066/v1/submissions/create \ --header "Content-Type:application/json;charset=UTF-8" \ --data '{ "appResource": "", "sparkProperties": { "spark.master": "local[2]", "spark.app.name": "Test 1", "spark.submit.deployMode": "cluster", "spark.jars": "/Users/dongjoon/APACHE/spark-release/spark-3.5.0-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.0.jar" }, "clientSparkVersion": "", "mainClass": "org.apache.spark.examples.SparkPi", "environmentVariables": {}, "action": "CreateSubmissionRequest", "appArgs": [] }' ``` **DRIVER `stderr` LOG** ``` Exception in thread "main" java.lang.reflect.InvocationTargetException ... at org.apache.spark.deploy.worker.DriverWrapper.main(DriverWrapper.scala) Caused by: java.lang.IllegalAccessError: class org.apache.spark.storage.StorageUtils$ (in unnamed module 0x6d7a93c9) cannot access class sun.nio.ch.DirectBuffer (in module java.base) because module java.base does not export sun.nio.ch to unnamed module 0x6d7a93c9 ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with the newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #42975 from dongjoon-hyun/SPARK-45197. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../apache/spark/deploy/rest/StandaloneRestServer.scala | 14 +++++++++----- .../spark/deploy/rest/StandaloneRestSubmitSuite.scala | 12 ++++++++++++ .../java/org/apache/spark/launcher/JavaModuleOptions.java | 8 ++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala index c060ef9da8c..a298e4f6dbf 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestServer.scala @@ -24,7 +24,7 @@ import org.apache.spark.{SPARK_VERSION => sparkVersion, SparkConf} import org.apache.spark.deploy.{Command, DeployMessages, DriverDescription} import org.apache.spark.deploy.ClientArguments._ import org.apache.spark.internal.config -import org.apache.spark.launcher.SparkLauncher +import org.apache.spark.launcher.{JavaModuleOptions, SparkLauncher} import org.apache.spark.resource.ResourceUtils import org.apache.spark.rpc.RpcEndpointRef import org.apache.spark.util.Utils @@ -124,7 +124,10 @@ private[rest] class StandaloneSubmitRequestServlet( * fields used by python applications since python is not supported in standalone * cluster mode yet. */ - private def buildDriverDescription(request: CreateSubmissionRequest): DriverDescription = { + private[rest] def buildDriverDescription( + request: CreateSubmissionRequest, + masterUrl: String, + masterRestPort: Int): DriverDescription = { // Required fields, including the main class because python is not yet supported val appResource = Option(request.appResource).getOrElse { throw new SubmitRestMissingFieldException("Application jar is missing.") @@ -149,7 +152,6 @@ private[rest] class StandaloneSubmitRequestServlet( // the driver. val masters = sparkProperties.get("spark.master") val (_, masterPort) = Utils.extractHostPortFromSparkUrl(masterUrl) - val masterRestPort = this.conf.get(config.MASTER_REST_SERVER_PORT) val updatedMasters = masters.map( _.replace(s":$masterRestPort", s":$masterPort")).getOrElse(masterUrl) val appArgs = request.appArgs @@ -167,7 +169,8 @@ private[rest] class StandaloneSubmitRequestServlet( .getOrElse(Seq.empty) val extraJavaOpts = driverExtraJavaOptions.map(Utils.splitCommandString).getOrElse(Seq.empty) val sparkJavaOpts = Utils.sparkJavaOpts(conf) - val javaOpts = sparkJavaOpts ++ defaultJavaOpts ++ extraJavaOpts + val javaModuleOptions = JavaModuleOptions.defaultModuleOptionArray().toSeq + val javaOpts = javaModuleOptions ++ sparkJavaOpts ++ defaultJavaOpts ++ extraJavaOpts val command = new Command( "org.apache.spark.deploy.worker.DriverWrapper", Seq("{{WORKER_URL}}", "{{USER_JAR}}", mainClass) ++ appArgs, // args to the DriverWrapper @@ -194,7 +197,8 @@ private[rest] class StandaloneSubmitRequestServlet( responseServlet: HttpServletResponse): SubmitRestProtocolResponse = { requestMessage match { case submitRequest: CreateSubmissionRequest => - val driverDescription = buildDriverDescription(submitRequest) + val driverDescription = buildDriverDescription( + submitRequest, masterUrl, conf.get(config.MASTER_REST_SERVER_PORT)) val response = masterEndpoint.askSync[DeployMessages.SubmitDriverResponse]( DeployMessages.RequestSubmitDriver(driverDescription)) val submitResponse = new CreateSubmissionResponse diff --git a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala index 696765297e7..0ad14de4ab5 100644 --- a/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/rest/StandaloneRestSubmitSuite.scala @@ -413,6 +413,18 @@ class StandaloneRestSubmitSuite extends SparkFunSuite { assert(filteredVariables == Map("SPARK_VAR" -> "1", "MESOS_VAR" -> "1")) } + test("SPARK-45197: Make StandaloneRestServer add JavaModuleOptions to drivers") { + val request = new CreateSubmissionRequest + request.appResource = "" + request.mainClass = "" + request.appArgs = Array.empty[String] + request.sparkProperties = Map.empty[String, String] + request.environmentVariables = Map.empty[String, String] + val servlet = new StandaloneSubmitRequestServlet(null, null, null) + val desc = servlet.buildDriverDescription(request, "spark://master:7077", 6066) + assert(desc.command.javaOpts.exists(_.startsWith("--add-opens"))) + } + /* --------------------- * | Helper methods | * --------------------- */ diff --git a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java index 013dde2766f..77895a09020 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java +++ b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java @@ -50,4 +50,12 @@ public class JavaModuleOptions { public static String defaultModuleOptions() { return String.join(" ", DEFAULT_MODULE_OPTIONS); } + + /** + * Returns the default Java option array related to `--add-opens' and + * `-XX:+IgnoreUnrecognizedVMOptions` used by Spark. + */ + public static String[] defaultModuleOptionArray() { + return DEFAULT_MODULE_OPTIONS; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org