SINGA-81 Add Python Helper update singa-run.sh to support running in a single process without job.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/2a6645cf Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/2a6645cf Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/2a6645cf Branch: refs/heads/master Commit: 2a6645cf2b7b625795b58b619e6cc544c08b63ff Parents: d47ed4b Author: WANG Sheng <[email protected]> Authored: Thu Dec 31 11:02:22 2015 +0800 Committer: chonho <[email protected]> Committed: Fri Jan 1 15:59:15 2016 +0800 ---------------------------------------------------------------------- bin/singa-run.sh | 44 +++++++++++++++++++++++--------------------- src/utils/cluster_rt.cc | 26 ++++++++++++++------------ src/utils/tool.cc | 2 +- 3 files changed, 38 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/bin/singa-run.sh ---------------------------------------------------------------------- diff --git a/bin/singa-run.sh b/bin/singa-run.sh index 548f8f1..836342e 100755 --- a/bin/singa-run.sh +++ b/bin/singa-run.sh @@ -23,16 +23,14 @@ # run a singa job # -usage="Usage: singa-run.sh -conf <job config file> [ other arguments ]\n - -resume : if want to recover a job\n - -exec <path to mysinga> : if want to use own singa driver\n - ### NOTICE ###\n - if you are using model.conf + cluster.conf,\n - please see how to combine them to a job.conf:\n - http://singa.incubator.apache.org/quick-start.html"; +usage="Usage: singa-run.sh [ arguments ]\n + -exec <binary or python script> : if want to use own singa driver\n + -conf <job config file> : need cluster conf if train in a cluster + -resume : if want to recover a job" # parse arguments -# make sure we have '-conf' and remove '-exec' +# extract and remove '-exec' and '-conf' +# other arguments remain untouched exe=./singa while [ $# != 0 ]; do if [ $1 == "-exec" ]; then @@ -46,23 +44,23 @@ while [ $# != 0 ]; do fi shift done -if [ -z $conf ]; then - echo -e $usage - exit 1 -fi # get environment variables . `dirname "${BASH_SOURCE-$0}"`/singa-env.sh # change conf to an absolute path -conf_dir=`dirname "$conf"` -conf_dir=`cd "$conf_dir">/dev/null; pwd` -conf_base=`basename "$conf"` -job_conf=$conf_dir/$conf_base -if [ ! -f $job_conf ]; then - echo $job_conf not exists - exit 1 +if [ ! -z $conf ]; then + conf_dir=`dirname "$conf"` + conf_dir=`cd "$conf_dir">/dev/null; pwd` + conf_base=`basename "$conf"` + job_conf=$conf_dir/$conf_base + if [ ! -f $job_conf ]; then + echo $job_conf not exists + exit 1 + fi fi + +# go to singa home to execute binary cd $SINGA_HOME # generate unique job id @@ -81,9 +79,13 @@ host_file=$log_dir/job.hosts ./singatool genhost $job_conf 1>$host_file || exit 1 # set command to run singa -singa_run="$exe $args -conf $job_conf \ +singa_run="$exe $args \ -singa_conf $SINGA_HOME/conf/singa.conf \ - -singa_job $job_id" + -singa_job $job_id" +# add -conf if exists +if [ ! -z $job_conf ]; then + singa_run="$singa_run -conf $job_conf" +fi singa_sshrun="cd $SINGA_HOME; $singa_run" # ssh and start singa processes http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/cluster_rt.cc ---------------------------------------------------------------------- diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc index 5ac3812..cdf8aab 100644 --- a/src/utils/cluster_rt.cc +++ b/src/utils/cluster_rt.cc @@ -367,19 +367,21 @@ bool JobManager::GenerateJobID(int* id) { bool JobManager::GenerateHostList(const char* host_file, const char* job_file, vector<string>* list) { + int nprocs = 1; // compute required #process from job conf - ClusterProto cluster; - google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file), - &cluster); - int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group() - / cluster.nworkers_per_procs(); - int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group() - / cluster.nservers_per_procs(); - int nprocs = 0; - if (cluster.server_worker_separate()) - nprocs = nworker_procs + nserver_procs; - else - nprocs = std::max(nworker_procs, nserver_procs); + if (job_file != nullptr) { + ClusterProto cluster; + google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file), + &cluster); + int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group() + / cluster.nworkers_per_procs(); + int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group() + / cluster.nservers_per_procs(); + if (cluster.server_worker_separate()) + nprocs = nworker_procs + nserver_procs; + else + nprocs = std::max(nworker_procs, nserver_procs); + } // get available host list from global conf std::ifstream hostfile(host_file); if (!hostfile.is_open()) { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/tool.cc ---------------------------------------------------------------------- diff --git a/src/utils/tool.cc b/src/utils/tool.cc index 18df826..4b50214 100644 --- a/src/utils/tool.cc +++ b/src/utils/tool.cc @@ -147,7 +147,7 @@ int main(int argc, char **argv) { else if (!strcmp(argv[1], "create")) stat = create(); else if (!strcmp(argv[1], "genhost")) - stat = (argc > 2) ? genhost(argv[2]) : ARG_ERR; + stat = (argc > 2) ? genhost(argv[2]) : genhost(nullptr); else if (!strcmp(argv[1], "list")) stat = list(false); else if (!strcmp(argv[1], "listall"))
