This is an automated email from the ASF dual-hosted git repository.
felixcheung pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6c18d8d [SPARK-26642][K8S] Add --num-executors option to spark-submit
for Spark on K8S.
6c18d8d is described below
commit 6c18d8d8079ac4d2d6dc7539601ab83fc5b51760
Author: Luca Canali <luca.can...@cern.ch>
AuthorDate: Sun Jan 20 12:43:34 2019 -0800
[SPARK-26642][K8S] Add --num-executors option to spark-submit for Spark on
K8S.
## What changes were proposed in this pull request?
This PR proposes to extend the spark-submit option --num-executors to be
applicable to Spark on K8S too. It is motivated by convenience, for example
when migrating jobs written for YARN to run on K8S.
## How was this patch tested?
Manually tested on a K8S cluster.
Author: Luca Canali <luca.can...@cern.ch>
Closes #23573 from LucaCanali/addNumExecutorsToK8s.
---
core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 4 ++--
.../main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala | 6 +++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index b403cc4..d5e17ff 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -537,14 +537,14 @@ private[spark] class SparkSubmit extends Logging {
// Yarn only
OptionAssigner(args.queue, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.queue"),
- OptionAssigner(args.numExecutors, YARN, ALL_DEPLOY_MODES,
- confKey = EXECUTOR_INSTANCES.key),
OptionAssigner(args.pyFiles, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.pyFiles"),
OptionAssigner(args.jars, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.jars"),
OptionAssigner(args.files, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.files"),
OptionAssigner(args.archives, YARN, ALL_DEPLOY_MODES, confKey =
"spark.yarn.dist.archives"),
// Other options
+ OptionAssigner(args.numExecutors, YARN | KUBERNETES, ALL_DEPLOY_MODES,
+ confKey = EXECUTOR_INSTANCES.key),
OptionAssigner(args.executorCores, STANDALONE | YARN | KUBERNETES,
ALL_DEPLOY_MODES,
confKey = EXECUTOR_CORES.key),
OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN |
KUBERNETES, ALL_DEPLOY_MODES,
diff --git
a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index f5e4c4a..9692d2a 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -585,15 +585,15 @@ private[deploy] class SparkSubmitArguments(args:
Seq[String], env: Map[String, S
| in standalone mode).
|
| Spark on YARN and Kubernetes only:
+ | --num-executors NUM Number of executors to launch (Default:
2).
+ | If dynamic allocation is enabled, the
initial number of
+ | executors will be at least NUM.
| --principal PRINCIPAL Principal to be used to login to KDC.
| --keytab KEYTAB The full path to the file that contains
the keytab for the
| principal specified above.
|
| Spark on YARN only:
| --queue QUEUE_NAME The YARN queue to submit to (Default:
"default").
- | --num-executors NUM Number of executors to launch (Default:
2).
- | If dynamic allocation is enabled, the
initial number of
- | executors will be at least NUM.
| --archives ARCHIVES Comma separated list of archives to be
extracted into the
| working directory of each executor.
""".stripMargin
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
