Repository: incubator-singa Updated Branches: refs/heads/master 2498ff135 -> f2b0aef12
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/configure.ac ---------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 277eb96..35c6d61 100644 --- a/configure.ac +++ b/configure.ac @@ -26,9 +26,6 @@ AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [ AC_SEARCH_LIBS([zmsg_new], [czmq], [], [ AC_MSG_ERROR([unable to find zmsg_new() function]) ]) -AC_CHECK_LIB([gflags], [main], [], [ - AC_MSG_ERROR([unable to find gflags library]) - ]) AC_CHECK_LIB([glog], [main], [], [ AC_MSG_ERROR([unable to find glog library]) ]) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/examples/mnist/create_shard.cc ---------------------------------------------------------------------- diff --git a/examples/mnist/create_shard.cc b/examples/mnist/create_shard.cc index 09229ac..635f901 100644 --- a/examples/mnist/create_shard.cc +++ b/examples/mnist/create_shard.cc @@ -7,7 +7,6 @@ // The MNIST dataset could be downloaded at // http://yann.lecun.com/exdb/mnist/ -#include <gflags/gflags.h> #include <glog/logging.h> #include <cstdint> #include <iostream> http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/include/singa.h ---------------------------------------------------------------------- diff --git a/include/singa.h b/include/singa.h index 52d1f90..20f941d 100644 --- a/include/singa.h +++ b/include/singa.h @@ -2,7 +2,6 @@ #define SINGA_SINGA_H_ #include <cblas.h> -#include <gflags/gflags.h> #include <glog/logging.h> #include <string> #include "communication/socket.h" @@ -78,9 +77,16 @@ class Driver { * launching script. */ inline int job_id() const { return job_id_; } + /** + * @return job conf path which is passed by users at the command line. It + * should at least contains the cluster configuration. + */ + inline JobProto job_conf() const { return job_conf_; } private: int job_id_; + JobProto job_conf_; + SingaProto singa_conf_; }; } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/include/utils/common.h ---------------------------------------------------------------------- diff --git a/include/utils/common.h b/include/utils/common.h index c4ff4eb..acdb20e 100644 --- a/include/utils/common.h +++ b/include/utils/common.h @@ -2,7 +2,6 @@ #define SINGA_UTILS_COMMON_H_ #include <google/protobuf/message.h> -#include <gflags/gflags.h> #include <stdlib.h> #include <unordered_map> #include <sstream> @@ -10,11 +9,6 @@ #include <vector> #include "proto/common.pb.h" -#ifndef GFLAGS_GFLAGS_H_ - namespace gflags = google; -#endif // GFLAGS_GFLAGS_H_ - - namespace singa { using std::vector; @@ -30,6 +24,16 @@ void ReadProtoFromBinaryFile(const char* filename, void WriteProtoToBinaryFile(const google::protobuf::Message& proto, const char* filename); +/** + * Locate the position of the arg in arglist. + * + * @param argc total num of arguments + * @param arglist all arguments + * @param the searched argument + * @return the position of arg in the arglist; -1 if not found. + */ +int ArgPos(int argc, char** arglist, const char* arg); + const std::string CurrentDateTime(); void CreateFolder(const std::string name); /** http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/driver.cc ---------------------------------------------------------------------- diff --git a/src/driver.cc b/src/driver.cc index 5469583..6d41fc5 100644 --- a/src/driver.cc +++ b/src/driver.cc @@ -2,17 +2,24 @@ namespace singa { -/** - * the job and singa_conf arguments are passed by the singa script which is - * transparent to users - */ -DEFINE_int32(job, -1, "Unique job ID generated from singa-run.sh"); -DEFINE_string(singa_conf, "conf/singa.conf", "Global config file"); - void Driver::Init(int argc, char **argv) { google::InitGoogleLogging(argv[0]); - gflags::ParseCommandLineFlags(&argc, &argv, true); - job_id_ = FLAGS_job; + + // unique job ID generated from singa-run.sh, passed in as "-singa_job <id>" + int arg_pos = ArgPos(argc, argv, "-singa_job"); + job_id_ = (arg_pos != -1) ? atoi(argv[arg_pos+1]) : -1; + + // global signa conf passed by singa-run.sh as "-singa_conf <path>" + arg_pos = ArgPos(argc, argv, "-singa_conf"); + if (arg_pos != -1) + ReadProtoFromTextFile(argv[arg_pos+1], &singa_conf_); + else + ReadProtoFromTextFile("conf/singa.conf", &singa_conf_); + + // job conf passed by users as "-conf <path>" + arg_pos = ArgPos(argc, argv, "-conf"); + CHECK_NE(arg_pos, -1); + ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_); // register layers RegisterLayer<BridgeDstLayer>(kBridgeDst); @@ -82,11 +89,9 @@ int Driver::RegisterWorker(int type) { } void Driver::Submit(bool resume, const JobProto& jobConf) { - SingaProto singaConf; - ReadProtoFromTextFile(FLAGS_singa_conf.c_str(), &singaConf); - if (singaConf.has_log_dir()) - SetupLog(singaConf.log_dir(), std::to_string(FLAGS_job) - + "-" + jobConf.name()); + if (singa_conf_.has_log_dir()) + SetupLog(singa_conf_.log_dir(), std::to_string(job_id_) + + "-" + jobConf.name()); if (jobConf.num_openblas_threads() != 1) LOG(WARNING) << "openblas with " << jobConf.num_openblas_threads() << " threads"; @@ -96,7 +101,7 @@ void Driver::Submit(bool resume, const JobProto& jobConf) { job.CopyFrom(jobConf); job.set_id(job_id_); Trainer trainer; - trainer.Start(resume, singaConf, &job); + trainer.Start(resume, singa_conf_, &job); } } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/main.cc ---------------------------------------------------------------------- diff --git a/src/main.cc b/src/main.cc index 692fe2c..1f3f4a4 100644 --- a/src/main.cc +++ b/src/main.cc @@ -13,6 +13,9 @@ * Optionally, users can register their own implemented classes, e.g., layer, * updater, through the registration func provided by the Driver. * + * Users must pass at least one argument to the singa-run.sh, i.e., the job + * configuration file which includes the cluster topology setting. Other fields + * e.g, neuralnet, updater can be configured in main.cc. * * TODO * Add helper functions for users to generate their configurations easily. @@ -20,21 +23,21 @@ * or, MLP(layer1_size, layer2_size, tanh, loss); */ -DEFINE_bool(resume, false, "Resume from checkpoint passed at cmd line"); -DEFINE_string(conf, "./job.conf", "job conf passed at cmd line"); - int main(int argc, char **argv) { // must create driver at the beginning and call its Init method. singa::Driver driver; driver.Init(argc, argv); + // if -resume in argument list, set resume to true; otherwise false + int resume_pos = singa::ArgPos(argc, argv, "-resume"); + bool resume = (resume_pos != -1); + // users can register new subclasses of layer, updater, etc. - // prepare job conf; - singa::JobProto jobConf; - singa::ReadProtoFromTextFile(FLAGS_conf.c_str(), &jobConf); + // get the job conf, and custmize it if need + singa::JobProto jobConf = driver.job_conf(); // submit the job - driver.Submit(FLAGS_resume, jobConf); + driver.Submit(resume, jobConf); return 0; } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/utils/common.cc ---------------------------------------------------------------------- diff --git a/src/utils/common.cc b/src/utils/common.cc index 3bba398..01ee317 100644 --- a/src/utils/common.cc +++ b/src/utils/common.cc @@ -64,7 +64,14 @@ void Debug() { while (0 == i) sleep(5); } - +int ArgPos(int argc, char** arglist, const char* arg) { + for (int i = 0; i < argc; i++) { + if (strcmp(arglist[i], arg) == 0) { + return i; + } + } + return -1; +} void CreateFolder(const std::string name) { struct stat buffer; if (stat(name.c_str(), &buffer) != 0) { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/src/utils/tool.cc ---------------------------------------------------------------------- diff --git a/src/utils/tool.cc b/src/utils/tool.cc index ebc0022..084ebea 100644 --- a/src/utils/tool.cc +++ b/src/utils/tool.cc @@ -1,4 +1,3 @@ -#include <gflags/gflags.h> #include <glog/logging.h> #include <algorithm> #include <fstream> @@ -8,12 +7,6 @@ #include "utils/cluster_rt.h" #include "utils/common.h" -#ifndef GFLAGS_GFLAGS_H_ -namespace gflags = google; -#endif // GFLAGS_GFLAGS_H_ - -DEFINE_string(confdir, "conf", "Global config dir"); - singa::SingaProto global; const int SUCCESS = 0; const int ARG_ERR = 1; @@ -54,9 +47,9 @@ int genhost(char* job_conf) { nprocs = std::max(nworker_procs, nserver_procs); // get available host list from global conf - std::fstream hostfile(FLAGS_confdir+"/hostfile"); + std::fstream hostfile("conf/hostfile"); if (!hostfile.is_open()) { - LOG(ERROR) << "Cannot open file: " << FLAGS_confdir+"/hostfile"; + LOG(ERROR) << "Cannot open file: " << "conf/hostfile"; return RUN_ERR; } std::vector<std::string> hosts; @@ -129,21 +122,20 @@ int cleanup() { int main(int argc, char **argv) { std::string usage = "Usage: singatool <command> <args>\n" - " getlogdir : show log dir in global config\n" - " create : generate a unique job id\n" - " genhost JOB_CONF : generate a host list\n" - " list : list running singa jobs\n" - " listall : list all singa jobs\n" - " view JOB_ID : view procs of a singa job\n" - " remove JOB_ID : remove a job path in zookeeper\n" - " removeall : remova all job paths in zookeeper\n" - " cleanup : clean all singa data in zookeeper\n"; - // set logging level to ERROR and log to STDERR - FLAGS_logtostderr = 1; - FLAGS_minloglevel = 2; + " getlogdir : show log dir in global config\n" + " create : generate a unique job id\n" + " genhost <job conf> : generate a host list\n" + " list : list running singa jobs\n" + " listall : list all singa jobs\n" + " view <job id> : view procs of a singa job\n" + " remove <job id> : remove a job path in zookeeper\n" + " removeall : remova all job paths in zookeeper\n" + " cleanup : clean all singa data in zookeeper\n"; + // set logging level to ERROR and log to STDERR only + google::LogToStderr(); + google::SetStderrLogging(google::ERROR); google::InitGoogleLogging(argv[0]); - gflags::ParseCommandLineFlags(&argc, &argv, true); - singa::ReadProtoFromTextFile((FLAGS_confdir+"/singa.conf").c_str(), &global); + singa::ReadProtoFromTextFile("conf/singa.conf", &global); // stat code: ARG_ERR for wrong argument, RUN_ERR for runtime error int stat = (argc <= 1) ? ARG_ERR : SUCCESS; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/f2b0aef1/thirdparty/install.sh ---------------------------------------------------------------------- diff --git a/thirdparty/install.sh b/thirdparty/install.sh index 04321d2..a70b964 100755 --- a/thirdparty/install.sh +++ b/thirdparty/install.sh @@ -84,40 +84,6 @@ function install_czmq() return 0; } -function install_gflags() -{ - if [ ! -e "gflags-2.1.1.tar.gz" ] - then - wget http://www.comp.nus.edu.sg/~dbsystem/singa/assets/file/thirdparty/gflags-2.1.1.tar.gz; - fi - rm -rf gflags-2.1.1; - tar zxvf gflags-2.1.1.tar.gz && cd gflags-2.1.1; - mkdir build && cd build; - - if [ $# == 1 ] - then - echo "install gflags in $1" - cmake .. -DCMAKE_INSTALL_PREFIX=$1 -DCMAKE_CXX_FLAGS=-fPIC; - make && make install; - elif [ $# == 0 ] - then - echo "install gflags in default path"; - cmake .. -DCMAKE_CXX_FLAGS=-fPIC; - make && sudo make install; - else - echo "gflags is done"; - fi - - if [ $? -ne 0 ] - then - cd ../..; - return -1; - fi - - cd ../..; - return 0; -} - function install_glog() { if [ ! -e "glog-0.3.3.tar.gz" ] @@ -448,27 +414,6 @@ do shift fi ;; - "gflags") - echo "install gflags"; - if [[ $2 == */* ]];then - install_gflags $2; - if [ $? -ne 0 ] - then - echo "ERROR during gflags installation" ; - exit; - fi - shift - shift - else - install_gflags; - if [ $? -ne 0 ] - then - echo "ERROR during gflags installation" ; - exit; - fi - shift - fi - ;; "glog") echo "install glog"; if [[ $2 == */* ]];then @@ -637,12 +582,6 @@ do echo "ERROR during czmq installation" ; exit; fi - install_gflags $2; - if [ $? -ne 0 ] - then - echo "ERROR during gflags installation" ; - exit; - fi install_glog $2; if [ $? -ne 0 ] then @@ -700,12 +639,6 @@ do echo "ERROR during czmq installation" ; exit; fi - install_gflags; - if [ $? -ne 0 ] - then - echo "ERROR during gflags installation" ; - exit; - fi install_glog; if [ $? -ne 0 ] then @@ -750,7 +683,6 @@ do echo " MISSING_LIBRARY_NAME can be: " echo " cmake" echo " czmq" - echo " gflags" echo " glog" echo " lmdb" echo " OpenBLAS"
