Merge pull request #69 from KarthikTunga/master Fix for issue SPARK-627. Implementing --config argument in the scripts.
This code fix is for issue SPARK-627. I added code to consider --config arguments in the scripts. In case the <conf-dir> is not a directory the scripts exit. I removed the --hosts argument. It can be achieved by giving a different config directory. Let me know if an explicit --hosts argument is required. (cherry picked from commit fc26e5b8320556b9edb93741391b759813b4079b) Signed-off-by: Reynold Xin <r...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/df21ac8d Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/df21ac8d Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/df21ac8d Branch: refs/heads/branch-0.8 Commit: df21ac8ddcebd1ad69d241da7d25c8ac59d5fe4e Parents: b531552 Author: Matei Zaharia <ma...@eecs.berkeley.edu> Authored: Thu Oct 17 13:21:07 2013 -0700 Committer: Reynold Xin <r...@apache.org> Committed: Thu Oct 17 18:38:46 2013 -0700 ---------------------------------------------------------------------- bin/slaves.sh | 19 ++++++++++++++++++- bin/spark-daemon.sh | 21 ++++++++++++++++++++- bin/spark-daemons.sh | 2 +- 3 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/df21ac8d/bin/slaves.sh ---------------------------------------------------------------------- diff --git a/bin/slaves.sh b/bin/slaves.sh index 752565b..c367c2f 100755 --- a/bin/slaves.sh +++ b/bin/slaves.sh @@ -28,7 +28,7 @@ # SPARK_SSH_OPTS Options passed to ssh when running remote commands. ## -usage="Usage: slaves.sh [--config confdir] command..." +usage="Usage: slaves.sh [--config <conf-dir>] command..." # if no args specified, show usage if [ $# -le 0 ]; then @@ -46,6 +46,23 @@ bin=`cd "$bin"; pwd` # spark-env.sh. Save it here. HOSTLIST=$SPARK_SLAVES +# Check if --config is passed as an argument. It is an optional parameter. +# Exit if the argument is not a directory. +if [ "$1" == "--config" ] +then + shift + conf_dir=$1 + if [ ! -d "$conf_dir" ] + then + echo "ERROR : $conf_dir is not a directory" + echo $usage + exit 1 + else + export SPARK_CONF_DIR=$conf_dir + fi + shift +fi + if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then . "${SPARK_CONF_DIR}/spark-env.sh" fi http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/df21ac8d/bin/spark-daemon.sh ---------------------------------------------------------------------- diff --git a/bin/spark-daemon.sh b/bin/spark-daemon.sh index 5bfe967..a0c0d44 100755 --- a/bin/spark-daemon.sh +++ b/bin/spark-daemon.sh @@ -29,7 +29,7 @@ # SPARK_NICENESS The scheduling priority for daemons. Defaults to 0. ## -usage="Usage: spark-daemon.sh [--config <conf-dir>] [--hosts hostlistfile] (start|stop) <spark-command> <spark-instance-number> <args...>" +usage="Usage: spark-daemon.sh [--config <conf-dir>] (start|stop) <spark-command> <spark-instance-number> <args...>" # if no args specified, show usage if [ $# -le 1 ]; then @@ -43,6 +43,25 @@ bin=`cd "$bin"; pwd` . "$bin/spark-config.sh" # get arguments + +# Check if --config is passed as an argument. It is an optional parameter. +# Exit if the argument is not a directory. + +if [ "$1" == "--config" ] +then + shift + conf_dir=$1 + if [ ! -d "$conf_dir" ] + then + echo "ERROR : $conf_dir is not a directory" + echo $usage + exit 1 + else + export SPARK_CONF_DIR=$conf_dir + fi + shift +fi + startStop=$1 shift command=$1 http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/df21ac8d/bin/spark-daemons.sh ---------------------------------------------------------------------- diff --git a/bin/spark-daemons.sh b/bin/spark-daemons.sh index 354eb90..64286cb 100755 --- a/bin/spark-daemons.sh +++ b/bin/spark-daemons.sh @@ -19,7 +19,7 @@ # Run a Spark command on all slave hosts. -usage="Usage: spark-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command instance-number args..." +usage="Usage: spark-daemons.sh [--config <conf-dir>] [start|stop] command instance-number args..." # if no args specified, show usage if [ $# -le 1 ]; then
