Repository: spark Updated Branches: refs/heads/branch-1.2 1770c51b2 -> 755f9cc40
[SPARK-5130][Deploy]Take yarn-cluster as cluster mode in spark-submit https://issues.apache.org/jira/browse/SPARK-5130 Author: WangTaoTheTonic <[email protected]> Closes #3929 from WangTaoTheTonic/SPARK-5130 and squashes the following commits: c490648 [WangTaoTheTonic] take yarn-cluster as cluster mode in spark-submit (cherry picked from commit 0760787da885187b0c6dcd5c28753f0ab014d5ed) Signed-off-by: Andrew Or <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/755f9cc4 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/755f9cc4 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/755f9cc4 Branch: refs/heads/branch-1.2 Commit: 755f9cc403357da9e5673b5035ebec100b1d83e6 Parents: 1770c51 Author: WangTaoTheTonic <[email protected]> Authored: Thu Jan 8 11:45:42 2015 -0800 Committer: Andrew Or <[email protected]> Committed: Thu Jan 8 11:46:53 2015 -0800 ---------------------------------------------------------------------- bin/spark-submit | 5 +++++ bin/spark-submit2.cmd | 6 ++++++ 2 files changed, 11 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/755f9cc4/bin/spark-submit ---------------------------------------------------------------------- diff --git a/bin/spark-submit b/bin/spark-submit index f92d90c..aefd38a 100755 --- a/bin/spark-submit +++ b/bin/spark-submit @@ -38,11 +38,16 @@ while (($#)); do export SPARK_SUBMIT_CLASSPATH=$2 elif [ "$1" = "--driver-java-options" ]; then export SPARK_SUBMIT_OPTS=$2 + elif [ "$1" = "--master" ]; then + export MASTER=$2 fi shift done DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf" +if [ "$MASTER" == "yarn-cluster" ]; then + SPARK_SUBMIT_DEPLOY_MODE=cluster +fi export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"} export SPARK_SUBMIT_PROPERTIES_FILE=${SPARK_SUBMIT_PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"} http://git-wip-us.apache.org/repos/asf/spark/blob/755f9cc4/bin/spark-submit2.cmd ---------------------------------------------------------------------- diff --git a/bin/spark-submit2.cmd b/bin/spark-submit2.cmd index cf6046d..daf0284 100644 --- a/bin/spark-submit2.cmd +++ b/bin/spark-submit2.cmd @@ -45,11 +45,17 @@ if [%1] == [] goto continue set SPARK_SUBMIT_CLASSPATH=%2 ) else if [%1] == [--driver-java-options] ( set SPARK_SUBMIT_OPTS=%2 + ) else if [%1] == [--master] ( + set MASTER=%2 ) shift goto loop :continue +if [%MASTER%] == [yarn-cluster] ( + set SPARK_SUBMIT_DEPLOY_MODE=cluster +) + rem For client mode, the driver will be launched in the same JVM that launches rem SparkSubmit, so we may need to read the properties file for any extra class rem paths, library paths, java options and memory early on. Otherwise, it will --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
