Repository: systemml Updated Branches: refs/heads/master 13a017583 -> 8f786aa22
[SYSTEMML-1945] added --deploy-mode param to python scripts Closes #681 Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8f786aa2 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8f786aa2 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8f786aa2 Branch: refs/heads/master Commit: 8f786aa227d536558ed684060cef0e628bf3247f Parents: 13a0175 Author: Krishna Kalyan <krishnakaly...@gmail.com> Authored: Wed Oct 11 09:20:40 2017 -0700 Committer: Nakul Jindal <naku...@gmail.com> Committed: Wed Oct 11 09:20:40 2017 -0700 ---------------------------------------------------------------------- bin/systemml-spark-submit.py | 9 +++++---- scripts/perftest/python/run_perftest.py | 3 ++- scripts/perftest/python/utils_misc.py | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/bin/systemml-spark-submit.py ---------------------------------------------------------------------- diff --git a/bin/systemml-spark-submit.py b/bin/systemml-spark-submit.py index b4da801..2546236 100755 --- a/bin/systemml-spark-submit.py +++ b/bin/systemml-spark-submit.py @@ -41,8 +41,8 @@ def default_jars(systemml_home): return target_jars, systemml_jar -def spark_submit_entry(master, driver_memory, num_executors, executor_memory, - executor_cores, conf, +def spark_submit_entry(master, deploy_mode, driver_memory, num_executors, + executor_memory, executor_cores, conf, nvargs, args, config, explain, debug, stats, gpu, f): """ This function is responsible for the execution of arguments via @@ -100,7 +100,7 @@ def spark_submit_entry(master, driver_memory, num_executors, executor_memory, # stats, explain, target_jars cmd_spark = [spark_path, '--class', 'org.apache.sysml.api.DMLScript', - '--master', master, + '--master', master, '--deploy-mode', deploy_mode, '--driver-memory', driver_memory, '--conf', default_conf, '--jars', cuda_jars, systemml_jars] @@ -129,7 +129,8 @@ if __name__ == '__main__': cparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='System-ML Spark Submit Script') # SPARK-SUBMIT Options - cparser.add_argument('--master', default='local[*]', help='local, yarn-client, yarn-cluster', metavar='') + cparser.add_argument('--master', default='local[*]', help='local, yarn', metavar='') + cparser.add_argument('--deploy-mode', help='client, cluster', default='client', metavar='') cparser.add_argument('--driver-memory', default='8G', help='Memory for driver (e.g. 512M, 1G)', metavar='') cparser.add_argument('--num-executors', nargs=1, help='Number of executors to launch', metavar='') cparser.add_argument('--executor-memory', nargs=1, help='Memory per executor', metavar='') http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/scripts/perftest/python/run_perftest.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/run_perftest.py b/scripts/perftest/python/run_perftest.py index 6c016a8..6e87261 100755 --- a/scripts/perftest/python/run_perftest.py +++ b/scripts/perftest/python/run_perftest.py @@ -355,7 +355,8 @@ if __name__ == '__main__': 'set <force> option to skip conservative memory estimates ' 'and use GPU wherever possible', nargs='?', const='no_option') # Spark Configuration Option - cparser.add_argument('--master', help='local, yarn-client, yarn-cluster', metavar='') + cparser.add_argument('--master', help='local, yarn', metavar='') + cparser.add_argument('--deploy-mode', help='client, cluster', metavar='') cparser.add_argument('--driver-memory', help='Memory for driver (e.g. 512M)', metavar='') cparser.add_argument('--num-executors', help='Number of executors to launch', metavar='') cparser.add_argument('--executor-memory', help='Memory per executor', metavar='') http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/scripts/perftest/python/utils_misc.py ---------------------------------------------------------------------- diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py index ebc4376..63d8c1b 100755 --- a/scripts/perftest/python/utils_misc.py +++ b/scripts/perftest/python/utils_misc.py @@ -84,6 +84,9 @@ def split_config_args(args): if args['master'] is not None: backend_args_dict['--master'] = args['master'] + if args['deploy_mode'] is not None: + backend_args_dict['--deploy-mode'] = args['master'] + if args['num_executors'] is not None: backend_args_dict['--num-executors'] = args['num_executors']
