Repository: incubator-singa Updated Branches: refs/heads/master 79a241c8b -> 0c6e5c692
SINGA-21 Code review 5 review neuralnet.h, neuralnet.cc - format code - remove unused varaibles: data_layers_, parser_layers, loss_layers_ TODO: - re-implement src-dst connection function to make it clear in logic Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/31611759 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/31611759 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/31611759 Branch: refs/heads/master Commit: 31611759c474826cbc4592841ad653e8aed04bc1 Parents: 79a241c Author: wang sheng <[email protected]> Authored: Tue Sep 22 14:06:50 2015 +0800 Committer: wang sheng <[email protected]> Committed: Tue Sep 22 17:28:25 2015 +0800 ---------------------------------------------------------------------- include/neuralnet/neuralnet.h | 62 +++------ include/trainer/worker.h | 6 + src/neuralnet/neuralnet.cc | 270 ++++++++++++++++++------------------- 3 files changed, 155 insertions(+), 183 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/31611759/include/neuralnet/neuralnet.h ---------------------------------------------------------------------- diff --git a/include/neuralnet/neuralnet.h b/include/neuralnet/neuralnet.h index f3b091a..06fd977 100644 --- a/include/neuralnet/neuralnet.h +++ b/include/neuralnet/neuralnet.h @@ -22,28 +22,24 @@ #ifndef SINGA_NEURALNET_NEURALNET_H_ #define SINGA_NEURALNET_NEURALNET_H_ -#include <vector> #include <map> #include <memory> #include <string> +#include <vector> -#include "proto/job.pb.h" #include "neuralnet/layer.h" +#include "proto/job.pb.h" #include "utils/factory.h" #include "utils/graph.h" namespace singa { -using std::vector; -using std::string; -using std::map; -using std::shared_ptr; /** * The neural network is constructed from user configurations in NetProto. * * Some layers, e.g., SplitLayer and BridgeSrcLayer/BridgeDstLayer * will be added implicitly to partition the neural network. - * TODO create wrappers for popular models, e.g., MLP, CNN. + * TODO(wangwei) create wrappers for popular models, e.g., MLP, CNN. */ class NeuralNet { public: @@ -53,20 +49,20 @@ class NeuralNet { * Parameters for test/validation net can share those from training after * setup (done outside of this funcion). * - * @param np proto for the neural network + * @param net_conf proto for the neural network * @param phase test/training/validation - * @param num num of partitions, do partitioning if num > 1 + * @param npartitions num of partitions, do partitioning if num > 1 * @return shared pointer to a neural net */ - static shared_ptr<NeuralNet> Create(const NetProto& np, Phase phase, int num); + static std::shared_ptr<NeuralNet> Create(const NetProto& net_conf, + Phase phase, int npartitions); - public: /** * construct the net structure from protocol buffer. * @param netproto neural net config * @param npartitions num of partitions. 1 for no partitioning. */ - explicit NeuralNet(NetProto netproto, int npartitions = 1); + NeuralNet(NetProto netproto, int npartitions); ~NeuralNet(); /** * To display the adjacency layers @@ -75,35 +71,16 @@ class NeuralNet { /** * Share memory of parameter values from other neuralnet */ - void ShareParamsFrom(shared_ptr<NeuralNet> other); - - const std::vector<Layer*>& layers() { - return layers_; - } - const std::vector<ParserLayer*>& parserlayers() const { - LOG(FATAL)<< " not implemented"; - return parserlayers_; - } - const std::vector<LossLayer*>& losslayers() const { - LOG(FATAL)<< " not implemented"; - return losslayers_; - } - const std::vector<DataLayer*>& datalayers() const { - LOG(FATAL)<< " not implemented"; - return datalayers_; - } - const std::vector<Param*>& params() const { - return params_; - } - Layer* name2layer(string name) const { + void ShareParamsFrom(std::shared_ptr<NeuralNet> other); + inline const std::vector<Layer*>& layers() { return layers_; } + inline const std::vector<Param*>& params() const { return params_; } + inline Layer* name2layer(std::string name) const { if (name2layer_.find(name) != name2layer_.end()) return name2layer_.at(name); else return nullptr; } - Param* paramid2param(int id) const { - return paramid2param_.at(id); - } + inline Param* paramid2param(int id) const { return paramid2param_.at(id); } protected: /** @@ -126,14 +103,13 @@ class NeuralNet { void PrepareDataStructures(); protected: - vector<Layer*> layers_; - vector<ParserLayer*> parserlayers_; - vector<LossLayer*> losslayers_; - vector<DataLayer*> datalayers_; - vector<Param*> params_; + std::vector<Layer*> layers_; + std::vector<Param*> params_; - map<string, Layer*> name2layer_; - map<int, Param*> paramid2param_; + std::map<std::string, Layer*> name2layer_; + std::map<int, Param*> paramid2param_; }; + } // namespace singa + #endif // SINGA_NEURALNET_NEURALNET_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/31611759/include/trainer/worker.h ---------------------------------------------------------------------- diff --git a/include/trainer/worker.h b/include/trainer/worker.h index 0607385..679435c 100644 --- a/include/trainer/worker.h +++ b/include/trainer/worker.h @@ -26,6 +26,12 @@ #include "communication/socket.h" namespace singa { + +using std::map; +using std::shared_ptr; +using std::string; +using std::vector; + //!< sleep 5 milliseconds if the Param is not updated to the expected version const int kCollectSleepTime=5; /** http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/31611759/src/neuralnet/neuralnet.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc index 2f50ec5..286d273 100644 --- a/src/neuralnet/neuralnet.cc +++ b/src/neuralnet/neuralnet.cc @@ -26,10 +26,14 @@ #include "utils/singleton.h" namespace singa { -shared_ptr<NeuralNet> NeuralNet::Create( - const NetProto& net_conf, - Phase phase, - int npartitions) { + +using std::map; +using std::shared_ptr; +using std::string; +using std::vector; + +shared_ptr<NeuralNet> NeuralNet::Create(const NetProto& net_conf, Phase phase, + int npartitions) { NetProto conf; conf.CopyFrom(net_conf); conf.clear_layer(); @@ -43,22 +47,21 @@ shared_ptr<NeuralNet> NeuralNet::Create( if (p == phase) include = false; } - if (include) { - LayerProto* layer_conf = conf.add_layer(); - layer_conf->CopyFrom(layer); - // using net partition if layer partition is not set - if (!layer_conf->has_partition_dim()) - layer_conf->set_partition_dim(net_conf.partition_dim()); - for (int i = 0; i < layer_conf->param_size(); i++) { - ParamProto* param = layer_conf->mutable_param(i); - if (param->has_name() && param->name() != "") { - CHECK(name2param.find(param->name()) == name2param.end()) - << "param name is repeated: " << param->name(); - name2param[param->name()] = param; - } - if (param->has_share_from() && param->share_from() != "") - shares.push_back(param); + if (include == false) continue; + LayerProto* layer_conf = conf.add_layer(); + layer_conf->CopyFrom(layer); + // using net partition if layer partition is not set + if (!layer_conf->has_partition_dim()) + layer_conf->set_partition_dim(net_conf.partition_dim()); + for (int i = 0; i < layer_conf->param_size(); i++) { + ParamProto* param = layer_conf->mutable_param(i); + if (param->has_name() && param->name() != "") { + CHECK(name2param.find(param->name()) == name2param.end()) + << "param name is repeated: " << param->name(); + name2param[param->name()] = param; } + if (param->has_share_from() && param->share_from() != "") + shares.push_back(param); } } for (auto param : shares) { @@ -71,19 +74,12 @@ shared_ptr<NeuralNet> NeuralNet::Create( param->set_name(name); param->set_share_from(from); } - LOG(INFO) << "NeuralNet config is\n" << conf.DebugString(); - // TODO(wangwei) create net based on net type, e.g., directed, undirected, etc auto net = std::make_shared<NeuralNet>(conf, npartitions); return net; } -NeuralNet::~NeuralNet() { - for (auto layer : layers_) - delete layer; -} - NeuralNet::NeuralNet(NetProto netproto, int npartitions) { LOG(INFO) << "Constructing Neural Net..."; auto graph = CreateGraph(netproto, npartitions); @@ -95,82 +91,34 @@ NeuralNet::NeuralNet(NetProto netproto, int npartitions) { LOG(INFO) << "Neural net constructed"; } -void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) { - // create one layer per node - for (Node* node : graph->nodes()) { - auto proto_ptr = static_cast<LayerProto*>(node->proto); - auto layer = Layer::Create(*proto_ptr); - layers_.push_back(layer); - name2layer_[node->name] = layer; - } - // connect layers - for (Node* node : graph->nodes()) { - auto layer = name2layer_[node->name]; - layer->clear_dstlayers(); - for (Node* dst : node->dstnodes) - layer->add_dstlayer(name2layer_[dst->name]); - layer->clear_srclayers(); - for (Node* src : node->srcnodes) - layer->add_srclayer(name2layer_[src->name]); - } - // setup layers - int paramid = 0; - map<string, string> layerinfo; - map<string, vector<Layer*>> share_param_layers; - for (Node* node : graph->nodes()) { - auto layer = name2layer_[node->name]; - layer->Setup(*(static_cast<LayerProto*>(node->proto)), npartitions); - LOG(INFO) << "constructing graph: " << layer->name(); - layerinfo[layer->name()] = IntVecToString(layer->data(nullptr).shape()); - string param_name = "$"; - for (auto param : layer->GetParams()) { - param->set_id(paramid++); - // if user does not name the param, then name it based on layer name. - if (param->name() == "") { - param->set_name(layer->name() + param_name); - param_name += "$"; - } - } - if (layer->partition_dim() == 0) - share_param_layers[node->origin].push_back(layer); - } - LOG(INFO) << "Neural net structure\n" << graph->ToJson(layerinfo); +NeuralNet::~NeuralNet() { + for (auto layer : layers_) + delete layer; +} - // create map from param name to param ptr - std::unordered_map<string, Param*> name2param; - for (auto layer : layers_) { - for (auto param : layer->GetParams()) { - name2param[param->name()] = param; - } - } - for (auto & entry : share_param_layers) { - // overwrite entries for replicated params due to layer partition (dim 0). - for (auto *param : entry.second.front()->GetParams()) - name2param.at(param->name()) = param; +std::string NeuralNet::ToAdjacency() { + string disp = ""; + for (auto& layer : layers_) { + disp += layer->name()+": "; + for (const auto& dst : layer->dstlayers()) + disp += dst->name()+", "; + disp += "\n"; } - // share params based on share_from field - for (auto & entry : name2param) { - Param* param = entry.second; - const string share_from = param->share_from(); - if (param->share_from() != "") { - if(name2param.find(share_from) != name2param.end()) { - param->ShareFrom(*name2param.at(param->share_from())); - } else { - LOG(FATAL) << "No param with the name (share_from) " << share_from; + return disp; +} + +void NeuralNet::ShareParamsFrom(shared_ptr<NeuralNet> other) { + for (auto& layer : layers_) { + auto otherlayer = other->name2layer(layer->name()); + if (otherlayer != nullptr) { + const auto& otherparams = otherlayer->GetParams(); + const auto& params = layer->GetParams(); + CHECK_EQ(params.size(), otherparams.size()); + for (size_t i = 0; i < params.size(); i++) { + params[i]->ShareFrom(*otherparams[i]); } } } - // share Params for layers generated (partitioned) from the same origin layer - for (auto & entry : share_param_layers) { - const auto& owner = entry.second.begin(); - const auto& owner_params = (*owner)->GetParams(); - for (auto it = owner + 1; it != entry.second.end(); it++) { - auto params = (*it)->GetParams(); - CHECK_EQ(params.size(), owner_params.size()); - for (size_t i = 0; i < params.size(); i++) - params.at(i)->ShareFrom(*owner_params.at(i)); - } - } } // add a node for SliceLayer between srcnode and dstnodes @@ -314,23 +262,26 @@ Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) { for (Node* node : nodes) ConcateNodes(graph, srcnodes, node); } else if ((src_pdim == 1 && pdim == 0) || (src_pdim == 0 && pdim == 1)) { + // TODO(wangwei) rewrite the whole src-dst construction in a clear way + LOG(FATAL) << "not implemented"; // the most complext scenario - vector<Node*> nodes; - for (Node* srcnode : srcnodes) - nodes.push_back(SliceNode(graph, srcnode, nodes, false)); - for (Node* node : nodes) - ConcateNodes(graph, nodes, node); + // vector<Node*> nodes; + // for (Node* srcnode : srcnodes) + // nodes.push_back(SliceNode(graph, srcnode, nodes, false)); + // for (Node* node : nodes) + // ConcateNodes(graph, nodes, node); } else if ((src_pdim == 0 && pdim == 0)|| (src_pdim == 1 && pdim == 1 && connection == kOneToOne)) { CHECK_EQ(srcnodes.size(), nodes.size()); for (size_t i = 0; i < srcnodes.size(); i++) graph->AddEdge(srcnodes[i], nodes[i]); + } else { + LOG(FATAL) << "in wrong branch, not implemented"; } } } // must do topology sort, because we have added new nodes. graph->Sort(); - // add nodes for SplitLayer vector<Node*> oldnodes = graph->nodes(); for (Node* node : oldnodes) { @@ -344,7 +295,6 @@ Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) { } delete layer; } - // add nodes for bridge layers for (Node* node : oldnodes) { vector<Node*> dstnodes = node->dstnodes; @@ -363,54 +313,94 @@ Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) { return graph; } +void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) { + // create one layer per node + for (Node* node : graph->nodes()) { + auto proto_ptr = static_cast<LayerProto*>(node->proto); + auto layer = Layer::Create(*proto_ptr); + layers_.push_back(layer); + name2layer_[node->name] = layer; + } + // connect layers + for (Node* node : graph->nodes()) { + auto layer = name2layer_[node->name]; + layer->clear_dstlayers(); + for (Node* dst : node->dstnodes) + layer->add_dstlayer(name2layer_[dst->name]); + layer->clear_srclayers(); + for (Node* src : node->srcnodes) + layer->add_srclayer(name2layer_[src->name]); + } + // setup layers + int paramid = 0; + map<string, string> layerinfo; + map<string, vector<Layer*>> share_param_layers; + for (Node* node : graph->nodes()) { + auto layer = name2layer_[node->name]; + layer->Setup(*(static_cast<LayerProto*>(node->proto)), npartitions); + LOG(INFO) << "constructing graph: " << layer->name(); + layerinfo[layer->name()] = IntVecToString(layer->data(nullptr).shape()); + string param_name = "$"; + for (auto param : layer->GetParams()) { + param->set_id(paramid++); + // if user does not name the param, then name it based on layer name. + if (param->name() == "") { + param->set_name(layer->name() + param_name); + param_name += "$"; + } + } + if (layer->partition_dim() == 0) + share_param_layers[node->origin].push_back(layer); + } + LOG(INFO) << "Neural net structure\n" << graph->ToJson(layerinfo); + // create map from param name to param ptr + std::unordered_map<string, Param*> name2param; + for (auto layer : layers_) { + for (auto param : layer->GetParams()) { + name2param[param->name()] = param; + } + } + for (auto & entry : share_param_layers) { + // overwrite entries for replicated params due to layer partition (dim 0). + for (auto *param : entry.second.front()->GetParams()) + name2param.at(param->name()) = param; + } + // share params based on share_from field + for (auto & entry : name2param) { + Param* param = entry.second; + const string share_from = param->share_from(); + if (param->share_from() != "") { + if (name2param.find(share_from) != name2param.end()) { + param->ShareFrom(*name2param.at(param->share_from())); + } else { + LOG(FATAL) << "No param with the name (share_from) " << share_from; + } + } + } + // share Params for layers generated (partitioned) from the same origin layer + for (auto & entry : share_param_layers) { + const auto& owner = entry.second.begin(); + const auto& owner_params = (*owner)->GetParams(); + for (auto it = owner + 1; it != entry.second.end(); it++) { + auto params = (*it)->GetParams(); + CHECK_EQ(params.size(), owner_params.size()); + for (size_t i = 0; i < params.size(); i++) + params.at(i)->ShareFrom(*owner_params.at(i)); + } + } +} void NeuralNet::PrepareDataStructures() { - parserlayers_.clear(); - losslayers_.clear(); - datalayers_.clear(); params_.clear(); paramid2param_.clear(); name2layer_.clear(); - for (auto& layer : layers_) { name2layer_[layer->name()] = layer; - /* - if (layer->is_parserlayer()) - parserlayers_.push_back(static_cast<ParserLayer*>(layer)); - if (layer->is_losslayer()) - losslayers_.push_back(static_cast<LossLayer*>(layer)); - if (layer->is_datalayer()) - datalayers_.push_back(static_cast<DataLayer*>(layer)); - */ for (Param* p : layer->GetParams()) { paramid2param_[p->id()] = p; params_.push_back(p); } } } -std::string NeuralNet::ToAdjacency() { - string disp = ""; - for (auto& layer : layers_) { - disp += layer->name()+": "; - for (const auto& dst : layer->dstlayers()) - disp += dst->name()+", "; - disp += "\n"; - } - return disp; -} - -void NeuralNet::ShareParamsFrom(shared_ptr<NeuralNet> other) { - for (auto& layer : layers_) { - auto otherlayer = other->name2layer(layer->name()); - if (otherlayer != nullptr) { - const auto& otherparams = otherlayer->GetParams(); - const auto& params = layer->GetParams(); - CHECK_EQ(params.size(), otherparams.size()); - for (size_t i = 0; i < params.size(); i++) { - params[i]->ShareFrom(*otherparams[i]); - } - } - } -} } // namespace singa
