http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/connection_layer/slice.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc deleted file mode 100644 index 3cca3fd..0000000 --- a/src/neuralnet/connection_layer/slice.cc +++ /dev/null @@ -1,166 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/connection_layer.h" -#include "singa/utils/math_blob.h" -#include "singa/utils/singleton.h" -#include "singa/utils/context.h" - -namespace singa { - -using std::vector; - -SliceLayer::~SliceLayer() { - for (size_t i = 1; i < datavec_.size(); ++i) { - if (datavec_[i] != nullptr) delete datavec_[i]; - if (gradvec_[i] != nullptr) delete gradvec_[i]; - } -} - -void SliceLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - Layer::Setup(conf, srclayers); - vector<int> shape = srclayers[0]->data(this).shape(); - slice_dim_ = conf.slice_conf().slice_dim(); - num_slices_ = conf.slice_conf().num_slices(); - CHECK_GE(slice_dim_, 0); - CHECK_LT(slice_dim_, shape.size()); - CHECK_GT(num_slices_, 0); - // add num_slices-1 more blobs - for (int i = 1; i < num_slices_; ++i) { - datavec_.push_back(new Blob<float>()); - gradvec_.push_back(new Blob<float>()); - } - // TODO(wangsh): remove equal-size restrict later - CHECK_EQ(shape[slice_dim_] % num_slices_, 0); - shape[slice_dim_] /= num_slices_; - for (int i = 0; i < num_slices_; ++i) { - // if (i == slice_num - 1) shape[slice_dim] += remain; - datavec_[i]->Reshape(shape); - gradvec_[i]->Reshape(shape); - } -} - -void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - const Blob<float>& blob = srclayers[0]->data(this); - // calculate step for each memcpy - int step = datavec_[0]->shape()[slice_dim_]; - for (unsigned i = slice_dim_ + 1; i < datavec_[0]->shape().size(); ++i) - step *= datavec_[0]->shape()[i]; - int srclayer_offset = 0; - int slice_offset = 0; - auto context = Singleton<Context>::Instance(); - int device = context->device_id(std::this_thread::get_id()); - while (srclayer_offset < blob.count()) { - for (int i = 0; i < num_slices_; ++i) { - if (device < 0) { - const float* src = blob.cpu_data() + srclayer_offset; - float* dst = datavec_[i]->mutable_cpu_data() + slice_offset; - memcpy(dst, src, step * sizeof(float)); - } else { -#ifdef USE_GPU - const float* src = blob.gpu_data() + srclayer_offset; - float* dst = datavec_[i]->mutable_gpu_data() + slice_offset; - cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault); -#else - LOG(FATAL) << "GPU is not supported"; -#endif - } - srclayer_offset += step; - } - slice_offset += step; - } -} - -void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - Blob<float>* blob = srclayers[0]->mutable_grad(this); - // calculate step for each memcpy - int step = gradvec_[0]->shape()[slice_dim_]; - for (size_t i = slice_dim_ + 1; i < gradvec_[0]->shape().size(); ++i) - step *= gradvec_[0]->shape()[i]; - int srclayer_offset = 0; - int slice_offset = 0; - auto context = Singleton<Context>::Instance(); - int device = context->device_id(std::this_thread::get_id()); - while (srclayer_offset < blob->count()) { - for (int i = 0; i < num_slices_; ++i) { - if (device < 0) { - const float* src = gradvec_[i]->cpu_data() + slice_offset; - float* dst = blob->mutable_cpu_data() + srclayer_offset; - memcpy(dst, src, step * sizeof(float)); - } else { -#ifdef USE_GPU - const float* src = gradvec_[i]->gpu_data() + slice_offset; - float* dst = blob->mutable_gpu_data() + srclayer_offset; - cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault); -#else - LOG(FATAL) << "GPU is not supported"; -#endif - } - srclayer_offset += step; - } - slice_offset += step; - } -} - -const Blob<float>& SliceLayer::data(const Layer* from) { - int idx = from ? layer_idx_.Get(from) : 0; - CHECK_LT(idx, num_slices_); - return *datavec_[idx]; -} - -const Blob<float>& SliceLayer::grad(const Layer* from) { - int idx = from ? layer_idx_.Get(from) : 0; - CHECK_LT(idx, num_slices_); - return *gradvec_[idx]; -} - -Blob<float>* SliceLayer::mutable_data(const Layer* from) { - CHECK(from); - int idx = layer_idx_.Get(from); - CHECK_LT(idx, num_slices_); - return datavec_[idx]; -} - -Blob<float>* SliceLayer::mutable_grad(const Layer* from) { - CHECK(from); - int idx = layer_idx_.Get(from); - CHECK_LT(idx, num_slices_); - return gradvec_[idx]; -} -const std::string SliceLayer::ToString(bool debug, int flag) { - if (!debug) - return ""; - string ret = ""; - if ((flag & kForward) == kForward && data_.count() !=0) { - for (unsigned k = 0; k < datavec_.size(); k++) - ret += StringPrintf("data-%u :%e ", k, Asum(*datavec_.at(k))); - } - if ((flag & kBackward) == kBackward && grad_.count() != 0) { - for (unsigned k = 0; k < gradvec_.size(); k++) - ret += StringPrintf("grad-%u:%e ", k, Asum(*gradvec_.at(k))); - } - return ret; -} -} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/connection_layer/split.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc deleted file mode 100644 index e46b902..0000000 --- a/src/neuralnet/connection_layer/split.cc +++ /dev/null @@ -1,91 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/connection_layer.h" -#include "singa/utils/math_blob.h" - -namespace singa { - -using std::vector; - -SplitLayer::~SplitLayer() { - for (size_t i = 1; i < gradvec_.size(); ++i) { - if (gradvec_[i] != nullptr) delete gradvec_[i]; - } -} - -void SplitLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - Layer::Setup(conf, srclayers); - data_.Reshape(srclayers[0]->data(this).shape()); - data_.ShareData(srclayers[0]->mutable_data(this), false); - num_splits_ = conf.split_conf().num_splits(); - CHECK_GT(num_splits_, 0); - // add num_splits-1 more grad blobs - for (int i = 1; i < num_splits_; ++i) { - gradvec_.push_back(new Blob<float>()); - } - for (int i = 0; i < num_splits_; ++i) - gradvec_[i]->Reshape(srclayers[0]->data(this).shape()); -} - -void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - // data is shared from its source, - // nothing to do in compute feature phase -} - -void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - // aggregate all gradients to grad_[0] - for (int i = 1; i < num_splits_; ++i) - AXPY<float>(1.0, *gradvec_[i], gradvec_[0]); - // copy grad_[0] to srclayer's grad - Copy(*gradvec_[0], srclayers[0]->mutable_grad(this)); -} - -const Blob<float>& SplitLayer::grad(const Layer* from) { - CHECK(from); - int idx = layer_idx_.Get(from); - CHECK_LT(idx, num_splits_); - return *gradvec_[idx]; -} - -Blob<float>* SplitLayer::mutable_grad(const Layer* from) { - CHECK(from); - int idx = layer_idx_.Get(from); - CHECK_LT(idx, num_splits_); - return gradvec_[idx]; -} -const std::string SplitLayer::ToString(bool debug, int flag) { - if (!debug) - return ""; - string ret = ""; - if ((flag & kForward) == kForward && data_.count() !=0) { - ret += StringPrintf("data:%13.9f ", Asum(data_)); - } - if ((flag & kBackward) == kBackward && grad_.count() != 0) { - for (unsigned k = 0; k < gradvec_.size(); k++) - ret += StringPrintf("grad-%u:%13.9f ", k, Asum(*gradvec_.at(k))); - } - return ret; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/char_rnn.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/char_rnn.cc b/src/neuralnet/input_layer/char_rnn.cc deleted file mode 100644 index 8a56711..0000000 --- a/src/neuralnet/input_layer/char_rnn.cc +++ /dev/null @@ -1,93 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ -#include <sstream> -#include <fstream> -#include "singa/neuralnet/input_layer.h" -namespace singa { - -void CharRNNInputLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - InputLayer::Setup(conf, srclayers); - batchsize_ = conf.char_rnn_conf().batchsize(); - path_ = conf.char_rnn_conf().path(); - vocab_path_ = conf.char_rnn_conf().vocab_path(); - unroll_len_ = conf.char_rnn_conf().unroll_len(); - datavec_.clear(); - // each unroll layer has a input blob - for (int i = 0; i <= unroll_len_; i++) { - datavec_.push_back(new Blob<float>(batchsize_)); - } -} - -void CharRNNInputLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - if (buf_.size() == 0) { - - // read the vocab - { - std::ifstream fin; - fin.open(vocab_path_); - CHECK(fin.is_open()) << "Can't open vocab_path = " << vocab_path_; - std::stringstream stream; - stream << fin.rdbuf(); - string vocab = stream.str(); - LOG(ERROR) << "Vocab_size = " << vocab.length(); - for (char c : vocab) - char2index_[c] = char2index_.size() - 1; - fin.close(); - } - - // read the whole text file - { - std::ifstream fin; - fin.open(path_); - CHECK(fin.is_open()) << "Can't open filepath = " << path_; - std::stringstream stream; - stream << fin.rdbuf(); - buf_ = stream.str(); - fin.close(); - } - - // decide the start pos of each instance in one mini-batch - int max_offset = buf_.length() / batchsize_; - CHECK_GT(max_offset, unroll_len_); - LOG(ERROR) << "Max iteration per epoch = " << max_offset / unroll_len_; - for (int i = 0; i < batchsize_; i ++) { - start_.push_back(i * max_offset); - } - } - - for (int l = 0; l < unroll_len_ + 1; l++) { - float* ptr = datavec_[l]->mutable_cpu_data(); - for (int i = 0; i < batchsize_; i++) { - ptr[i] = static_cast<float>(char2index_.at(buf_[start_[i] + offset_ + l])); - } - } - offset_ += unroll_len_; - if (offset_ >= buf_.length() / batchsize_) { -// unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); -// std::mt19937 g(seed); -// std::shuffle(start_.begin(), start_.end(), g); - offset_ = 0; - // return -1; - } -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/csv.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/csv.cc b/src/neuralnet/input_layer/csv.cc deleted file mode 100644 index 53cabff..0000000 --- a/src/neuralnet/input_layer/csv.cc +++ /dev/null @@ -1,67 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer.h" -#include "singa/utils/tokenizer.h" - -namespace singa { - -void CSVInputLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - SingleLabelRecordLayer::Setup(conf, srclayers); - sep_ = conf.store_conf().separator(); -} - -void CSVInputLayer::LoadRecord(const string& backend, - const string&path, Blob<float>* to) { - io::Store* store = io::OpenStore(backend, path, io::kRead); - string key, val; - CHECK(store->Read(&key, &val)); - float* ptr = to->mutable_cpu_data(); - Tokenizer t(val, sep_); - string x; - for (int i = 0; i< to->count(); i++) { - t >> x; - ptr[i] = stof(x); - } - CHECK(!t.Valid()); - delete store; -} - -bool CSVInputLayer::Parse(int k, int flag, const string& key, - const string& value) { - float* ptr = data_.mutable_cpu_data() + k * data_.count() / batchsize_; - Tokenizer t(value, sep_); - string x; - // parse label if not deploy phase and has_label is set. - if ((flag & kDeploy) == 0 && layer_conf_.store_conf().has_label()) { - t >> x; - aux_data_[k] = stoi(x); - } - for (int i = 0; i< data_.count() / batchsize_; i++) { - t >> x; - ptr[i] = stof(x); - } - CHECK(!t.Valid()); - return true; -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/deprecated.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc deleted file mode 100644 index d2901f7..0000000 --- a/src/neuralnet/input_layer/deprecated.cc +++ /dev/null @@ -1,373 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include <random> -#include "singa/neuralnet/input_layer.h" -#include "singa/utils/context.h" -#include "singa/utils/singleton.h" -#include "mshadow/tensor.h" -namespace singa { - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape4; -using mshadow::Tensor; - -using std::string; -using std::vector; - -ShardDataLayer::~ShardDataLayer() { - if (shard_ != nullptr) - delete shard_; - shard_ = nullptr; -} - -void ShardDataLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - shard_ = new DataShard(proto.sharddata_conf().path(), DataShard::kRead); - string key; - shard_->Next(&key, &sample_); - delete shard_; - shard_ = nullptr; - batchsize_ = proto.sharddata_conf().batchsize(); - if (partition_dim() == 0) - batchsize_ /= proto.num_partitions(); - records_.resize(batchsize_); - random_skip_ = proto.sharddata_conf().random_skip(); -} - -void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - if (shard_ == nullptr) - shard_ = new DataShard(layer_conf_.sharddata_conf().path(), - DataShard::kRead); - if (random_skip_) { - std::uniform_int_distribution<int> distribution(0, random_skip_); - auto generator = Singleton<Context>::Instance()->rand_generator(); - int nskip = distribution(*generator); - LOG(INFO) << "Random Skip " << nskip << " records, there are " - << shard_->Count() << " records in total"; - string key; - for (int i = 0; i < nskip; i++) { - shard_->Next(&key, &sample_); - } - random_skip_ = 0; - } - for (auto& record : records_) { - string key; - if (!shard_->Next(&key, &record)) { - shard_->SeekToFirst(); - CHECK(shard_->Next(&key, &record)); - } - } -} - -/*****************LMDB data layer*******************/ -#ifdef USE_LMDB -LMDBDataLayer::~LMDBDataLayer() { - mdb_cursor_close(mdb_cursor_); - mdb_txn_abort(mdb_txn_); - mdb_cursor_ = nullptr; -} - -void LMDBDataLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - OpenLMDB(proto.lmdbdata_conf().path()); - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT), - MDB_SUCCESS); - mdb_cursor_close(mdb_cursor_); - mdb_txn_abort(mdb_txn_); - mdb_cursor_ = nullptr; - CaffeDatum datum; - datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); - SingleLabelImageRecord* record = sample_.mutable_image(); - ConvertCaffeDatumToRecord(datum, record); - batchsize_ = proto.lmdbdata_conf().batchsize(); - if (partition_dim() == 0) - batchsize_ /= proto.num_partitions(); - records_.resize(batchsize_); - random_skip_ = proto.lmdbdata_conf().random_skip(); -} - -void LMDBDataLayer::OpenLMDB(const std::string& path) { - CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed"; - CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB - CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(), - MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path; - CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS) - << "mdb_txn_begin failed"; - CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS) - << "mdb_open failed"; - CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS) - << "mdb_cursor_open failed"; - LOG(INFO) << "Opening lmdb " << path; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST), - MDB_SUCCESS) << "mdb_cursor_get failed"; -} - -void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - if (mdb_cursor_ == nullptr) - OpenLMDB(layer_conf_.lmdbdata_conf().path()); - if (random_skip_) { - std::uniform_int_distribution<int> distribution(0, random_skip_); - auto generator = - Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id()); - int nskip = distribution(*generator); - - int n = 0; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - while (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) == MDB_SUCCESS) - n++; - LOG(INFO) << "Random Skip " << nskip << " records of total " - << n << "records"; - // We have reached the end. Restart from the first. - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - for (int i = 0; i < nskip; i++) { - if (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { - // We have reached the end. Restart from the first. - DLOG(INFO) << "Restarting data prefetching from start."; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - } - } - random_skip_ = 0; - } - CaffeDatum datum; - for (auto& record : records_) { - SingleLabelImageRecord* image = record.mutable_image(); - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS); - datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size); - ConvertCaffeDatumToRecord(datum, image); - if (mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_NEXT) != MDB_SUCCESS) { - // We have reached the end. Restart from the first. - DLOG(INFO) << "Restarting data prefetching from start."; - CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, - &mdb_value_, MDB_FIRST), MDB_SUCCESS); - } - } -} - -void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum, - SingleLabelImageRecord* record) { - record->set_label(datum.label()); - record->clear_shape(); - if (datum.has_channels()) - record->add_shape(datum.channels()); - if (datum.has_height()) - record->add_shape(datum.height()); - if (datum.has_width()) - record->add_shape(datum.width()); - if (datum.has_data()) - record->set_pixel(datum.data()); - if (datum.float_data_size()) { - record->clear_data(); - for (float x : datum.float_data()) - record->add_data(x); - } -} -#endif - -/***************Parser layer*******************/ -void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin()); - ParseRecords(flag, datalayer->records(), &data_); -} - -/**********Mnist Layer************/ -void MnistLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - LOG_IF(ERROR, records.size() == 0) << "Empty records to parse"; - int ndim = records.at(0).image().shape_size(); - int inputsize = records.at(0).image().shape(ndim-1); - CHECK_EQ(inputsize, blob->shape()[2]); - - float* dptr = blob->mutable_cpu_data(); - for (const Record& record : records) { - const SingleLabelImageRecord& imagerecord = record.image(); - if (imagerecord.pixel().size()) { - string pixel = imagerecord.pixel(); - for (int i = 0, k = 0; i < inputsize; i++) { - for (int j = 0; j < inputsize; j++) { - // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of - // time to debug this - float x = static_cast<float>(static_cast<uint8_t>(pixel[k++])); - x = x / norm_a_-norm_b_; - *dptr = x; - dptr++; - } - } - } else { - for (int i = 0, k = 0; i < inputsize; i++) { - for (int j = 0; j < inputsize; j++) { - *dptr = imagerecord.data(k++) / norm_a_ - norm_b_; - dptr++; - } - } - } - } - CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count()); -} - -void MnistLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); - norm_a_ = proto.mnist_conf().norm_a(); - norm_b_ = proto.mnist_conf().norm_b(); - int ndim = sample.image().shape_size(); - CHECK_GE(ndim, 2); - int s = sample.image().shape(ndim - 1); - CHECK_EQ(s, sample.image().shape(ndim - 2)); - data_.Reshape(vector<int>{batchsize, 1, s, s}); -} - -/**********RGB image layer****************/ -void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - const vector<int>& s = blob->shape(); - Tensor<cpu, 4> images(data_.mutable_cpu_data(), - Shape4(s[0], s[1], s[2], s[3])); - const SingleLabelImageRecord& r = records.at(0).image(); - Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2))); - AllocSpace(raw_image); - Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3])); - if (cropsize_) - AllocSpace(croped_image); - int rid = 0; - const float* meandptr = mean_.cpu_data(); - - std::uniform_int_distribution<int> distribution(0, r.shape(0) - cropsize_); - auto generator = - Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id()); - for (const Record& record : records) { - auto image = images[rid]; - bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain); - bool do_mirror = mirror_ - && (distribution(*generator) % 2) - && ((flag & kTrain) == kTrain); - float* dptr = nullptr; - if (do_crop || do_mirror) - dptr = raw_image.dptr; - else - dptr = image.dptr; - if (record.image().pixel().size()) { - string pixel = record.image().pixel(); - for (size_t i = 0; i < pixel.size(); i++) - dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i])); - } else { - memcpy(dptr, record.image().data().data(), - sizeof(float) * record.image().data_size()); - } - for (int i = 0; i < mean_.count(); i++) - dptr[i] -= meandptr[i]; - if (do_crop) { - int hoff = distribution(*generator); - int woff = distribution(*generator); - Shape<2> cropshape = Shape2(cropsize_, cropsize_); - if (do_mirror) { - croped_image = expr::crop(raw_image, cropshape, hoff, woff); - image = expr::mirror(croped_image); - } else { - image = expr::crop(raw_image, cropshape, hoff, woff); - } - } else if (do_mirror) { - image = expr::mirror(raw_image); - } - rid++; - } - if (scale_) - images = images * scale_; - FreeSpace(raw_image); - if (cropsize_) - FreeSpace(croped_image); -} - -void RGBImageLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - ParserLayer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - scale_ = proto.rgbimage_conf().scale(); - cropsize_ = proto.rgbimage_conf().cropsize(); - mirror_ = proto.rgbimage_conf().mirror(); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample(); - vector<int> shape; - shape.push_back(batchsize); - for (int x : sample.image().shape()) { - shape.push_back(x); - } - CHECK_EQ(shape.size(), 4); - if (cropsize_) { - shape[2] = cropsize_; - shape[3] = cropsize_; - } - data_.Reshape(shape); - mean_.Reshape({shape[1], shape[2], shape[3]}); - if (proto.rgbimage_conf().has_meanfile()) { - if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) { - CaffeBlob mean; - ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); - CHECK_EQ(mean_.count(), mean.data_size()); - memcpy(mean_.mutable_cpu_data(), mean.data().data(), - sizeof(float)*mean.data_size()); - } else { - SingleLabelImageRecord mean; - ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean); - CHECK_EQ(mean_.count(), mean.data_size()); - memcpy(mean_.mutable_cpu_data(), mean.data().data(), - sizeof(float)*mean.data_size()); - } - } else { - memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count()); - } -} - -/*************Label layer *************/ - -void LabelLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - Layer::Setup(proto, srclayers); - CHECK_EQ(srclayers.size(), 1); - int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize(); - data_.Reshape(vector<int>{batchsize}); -} - -void LabelLayer::ParseRecords(int flag, const vector<Record>& records, - Blob<float>* blob) { - int rid = 0; - float *label = blob->mutable_cpu_data(); - for (const Record& record : records) { - label[rid++] = record.image().label(); - // CHECK_LT(record.image().label(),10); - } - CHECK_EQ(rid, blob->shape()[0]); -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/image_preprocess.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/image_preprocess.cc b/src/neuralnet/input_layer/image_preprocess.cc deleted file mode 100644 index 6f2e094..0000000 --- a/src/neuralnet/input_layer/image_preprocess.cc +++ /dev/null @@ -1,78 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer.h" -#include "singa/utils/image_transform.h" -#include "singa/utils/context.h" -#include "singa/utils/singleton.h" - -namespace singa { - -using std::vector; - -void ImagePreprocessLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 1); - InputLayer::Setup(conf, srclayers); - scale_ = conf.rgbimage_conf().scale(); - cropsize_ = conf.rgbimage_conf().cropsize(); - mirror_ = conf.rgbimage_conf().mirror(); - const auto& src = srclayers.at(0)->data(this); - const auto& shape = src.shape(); - CHECK_EQ(shape.size(), 4); - CHECK_EQ(shape.at(2), shape.at(3)); - if (cropsize_ && (cropsize_ != shape.at(2) || cropsize_ != shape.at(3))) { - data_.Reshape(vector<int>{shape.at(0), shape.at(1), cropsize_, cropsize_}); - } else { - data_ = src; - } -} - -void ImagePreprocessLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - const auto& srcdata = srclayers.at(0)->data(this); - int batchsize = srcdata.shape(0), channel = srcdata.shape(1); - int height = srcdata.shape(2), width = srcdata.shape(3); - int srcimage_size = channel * height * width; - int image_size = channel * data_.shape(2) * data_.shape(3); - std::uniform_int_distribution<int> rand1(0, height - cropsize_); - std::uniform_int_distribution<int> rand2(0, width - cropsize_); - auto generator = Singleton<Context>::Instance()->rand_generator(); - - const float* srcdptr = srcdata.cpu_data(); - float* dptr = data_.mutable_cpu_data(); - - for (int k = 0; k < batchsize; k++) { - int h_offset = 0, w_offset = 0; - if (cropsize_> 0 && (flag & kTrain)) { - h_offset = rand1(*generator); - w_offset = rand2(*generator); - } - bool do_mirror = mirror_ - && (rand1(*generator) % 2) - && (flag & kTrain); - ImageTransform(srcdptr + k * srcimage_size, nullptr, do_mirror, cropsize_, - cropsize_, h_offset, w_offset, channel, height, width, - scale_, dptr + k * image_size); - } -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/onehot.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/onehot.cc b/src/neuralnet/input_layer/onehot.cc deleted file mode 100644 index 4b83705..0000000 --- a/src/neuralnet/input_layer/onehot.cc +++ /dev/null @@ -1,40 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ -#include "singa/neuralnet/input_layer.h" - -namespace singa { -void OneHotLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - InputLayer::Setup(conf, srclayers); - batchsize_ = srclayers.at(0)->data(unroll_index()).shape(0); - dim_ = conf.onehot_conf().vocab_size(); - data_.Reshape(batchsize_, dim_); -} - -void OneHotLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - float* ptr = data_.mutable_cpu_data(); - memset(ptr, 0, sizeof(float) * data_.count()); - const float* idx = srclayers[0]->data(unroll_index()).cpu_data(); - for (int i = 0; i < batchsize_; i++) { - ptr[i * dim_ + static_cast<int>(idx[i])] = 1; - } -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/record.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/record.cc b/src/neuralnet/input_layer/record.cc deleted file mode 100644 index b14fc80..0000000 --- a/src/neuralnet/input_layer/record.cc +++ /dev/null @@ -1,73 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer.h" -namespace singa { - -using std::string; -using std::vector; - -void RecordInputLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - SingleLabelRecordLayer::Setup(conf, srclayers); - encoded_ = conf.store_conf().encoded(); -} - -void RecordInputLayer::LoadRecord(const string& backend, - const string& path, Blob<float>* to) { - io::Store* store = io::OpenStore(backend, path, io::kRead); - string key, val; - CHECK(store->Read(&key, &val)); - RecordProto image; - image.ParseFromString(val); - CHECK_EQ(to->count(), image.data_size()); - float* ptr = to->mutable_cpu_data(); - for (int i = 0; i< to->count(); i++) - ptr[i] = image.data(i); - delete store; -} - -bool RecordInputLayer::Parse(int k, int flag, const string& key, - const string& value) { - RecordProto image; - image.ParseFromString(value); - int size = data_.count() / batchsize_; - if (image.data_size()) { - CHECK_EQ(size, image.data_size()); - float* ptr = data_.mutable_cpu_data() + k * size; - for (int i = 0; i< size; i++) - ptr[i] = image.data(i); - } else if (image.pixel().size()) { - CHECK_EQ(size, image.pixel().size()); - float* ptr = data_.mutable_cpu_data() + k * size; - string pixel = image.pixel(); - for (int i = 0; i < size; i++) - ptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i])); - } else { - LOG(ERROR) << "not pixel nor pixel"; - } - if ((flag & kDeploy) == 0) { // deploy mode does not have label - aux_data_.at(k) = image.label(); - } - return true; -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/rnn_label.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/rnn_label.cc b/src/neuralnet/input_layer/rnn_label.cc deleted file mode 100644 index 4924d87..0000000 --- a/src/neuralnet/input_layer/rnn_label.cc +++ /dev/null @@ -1,35 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer.h" -namespace singa { -void RNNLabelLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - InputLayer::Setup(proto, srclayers); - aux_data_.resize(srclayers[0]->data(unroll_index() + 1).shape(0)); -} -void RNNLabelLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - const float* input = srclayers[0]->data(unroll_index() + 1).cpu_data(); - for (unsigned i = 0; i < aux_data_.size(); i++) { - aux_data_[i] = static_cast<int>(input[i]); - } -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/store.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer/store.cc b/src/neuralnet/input_layer/store.cc deleted file mode 100644 index 32f1887..0000000 --- a/src/neuralnet/input_layer/store.cc +++ /dev/null @@ -1,162 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/input_layer.h" -#include "singa/utils/context.h" -#include "singa/utils/singleton.h" -namespace singa { - -using std::thread; - -StoreInputLayer::~StoreInputLayer() { - if (thread_ != nullptr) { - thread_->join(); - delete thread_; - } - if (store_ != nullptr) { - delete store_; - } -} - -void StoreInputLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - InputLayer::Setup(conf, srclayers); - const auto& batchsize = conf.store_conf().batchsize(); - CHECK(batchsize.size()); - if (conf.partition_dim() == 0) { - if (batchsize.size() == 1) // equal partition - batchsize_ = batchsize.Get(0) / conf.num_partitions(); - else // manual partition - batchsize_ = batchsize.Get(conf.partition_id()); - } else { - batchsize_ = conf.store_conf().batchsize(0); - } - - vector<int> shape {batchsize_}; - for (int s : conf.store_conf().shape()) - shape.push_back(s); - data_.Reshape(shape); - aux_data_.resize(batchsize_); -} - -void StoreInputLayer::fetch_data() { - if (store_ == nullptr) { - store_ = io::OpenStore(layer_conf_.store_conf().backend(), - layer_conf_.store_conf().path(), - io::kRead); - if (layer_conf_.store_conf().random_skip() > 0) { - std::uniform_int_distribution<int> - distribution(0, layer_conf_.store_conf().random_skip()); - auto generator = Singleton<Context>::Instance()->rand_generator( - std::this_thread::get_id()); - random_skip_ = distribution(*generator); - } - - string key, val; - while (random_skip_ > 0) { - if (!store_->Read(&key, &val)) { - store_->SeekToFirst(); - CHECK(store_->Read(&key, &val)); - } - random_skip_--; - } - buf_keys_.resize(batchsize_); - buf_vals_.resize(batchsize_); - } - for (int k = 0; k < batchsize_; k++) { - if (!store_->Read(&buf_keys_[k], &buf_vals_[k])) { - store_->SeekToFirst(); - CHECK(store_->Read(&buf_keys_[k], &buf_vals_[k])); - } - } -} - -void StoreInputLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - - // if prefetching, wait for the thread to finish - if (layer_conf_.store_conf().prefetching()) { - if (thread_ == nullptr) { - thread_ = new thread(&StoreInputLayer::fetch_data, this); - } - thread_->join(); - delete thread_; - thread_ = nullptr; - } else { - fetch_data(); - } - for (int k = 0; k < batchsize_; k++) - Parse(k, flag, buf_keys_[k], buf_vals_[k]); - if (layer_conf_.store_conf().prefetching()) - thread_ = new thread(&StoreInputLayer::fetch_data, this); -} - -void SingleLabelRecordLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - StoreInputLayer::Setup(conf, srclayers); -} - -void SingleLabelRecordLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - - StoreInputLayer::ComputeFeature(flag, srclayers); - auto& store_conf = layer_conf_.store_conf(); - - if (store_conf.has_mean_file() && mean_.count() == 0) { - mean_.Reshape(vector<int>{data_.count() / batchsize_}); - LoadRecord(store_conf.backend(), store_conf.mean_file(), &mean_); - } else if (store_conf.has_mean_value() && mean_.count() == 0) { - mean_.Reshape(vector<int>{data_.count() / batchsize_}); - for (int i = 0; i < data_.count() / batchsize_; i++) - mean_.mutable_cpu_data()[i] = store_conf.mean_value(); - } - if (store_conf.has_std_file() && std_.count() == 0) { - std_.Reshape(vector<int>{data_.count() / batchsize_}); - LoadRecord(store_conf.backend(), store_conf.std_file(), &std_); - // TODO(wangwei) check std[i] != 0 - } else if (store_conf.has_std_value() && std_.count() == 0) { - std_.Reshape(vector<int>{data_.count() / batchsize_}); - CHECK_NE(store_conf.std_value(), 0); - for (int i = 0; i < data_.count() / batchsize_; i++) - std_.mutable_cpu_data()[i] = store_conf.std_value(); - } - - if (mean_.count()) { - const float* mean = mean_.cpu_data(); - for (int k = 0; k < batchsize_; k++) { - float* dptr = data_.mutable_cpu_data() + k * mean_.count(); - for (int i = 0; i < mean_.count(); i++) { - dptr[i] -= mean[i]; - } - } - } - if (std_.count()) { - const float* std = std_.cpu_data(); - for (int k = 0; k < batchsize_; k++) { - float* dptr = data_.mutable_cpu_data() + k * std_.count(); - for (int i = 0; i < std_.count(); i++) { - dptr[i] /= std[i]; - } - } - } -} - -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/layer.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc deleted file mode 100644 index ef1629f..0000000 --- a/src/neuralnet/layer.cc +++ /dev/null @@ -1,82 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/worker.h" -#include "singa/neuralnet/layer.h" -#include "singa/neuralnet/input_layer.h" -#include "singa/neuralnet/neuron_layer.h" -#include "singa/neuralnet/loss_layer.h" - -#include <cblas.h> -#include <glog/logging.h> -#include <math.h> -#include <cfloat> -#include "singa/utils/factory.h" -#include "singa/utils/singleton.h" -#include "singa/utils/math_blob.h" - -namespace singa { - -using std::string; - -void Layer::SetupLayer(Layer* layer, const string str, const vector<Layer*>& srclayers) { - LayerProto layer_conf; - layer_conf.ParseFromString(str); - layer->Setup(layer_conf, srclayers); - for (auto param : layer->GetParams()) - param->InitValues(); -} - -Layer* Layer::CreateLayer(const string str) { - LayerProto layer_conf; - layer_conf.ParseFromString(str); - return Layer::Create(layer_conf); -} - -Layer* Layer::Create(const LayerProto& proto) { - auto* factory = Singleton<Factory<Layer>>::Instance(); - Layer* layer = nullptr; - if (proto.has_user_type()) - layer = factory->Create(proto.user_type()); - else - layer = factory->Create(proto.type()); - return layer; -} - -const std::string Layer::ToString(bool debug, int flag) { - if (!debug) - return ""; - string ret = ""; - if ((flag & kForward) == kForward && data_.count() !=0) { - ret += StringPrintf("data:%e ", Asum(data_)); - for (Param* p : GetParams()) - ret += StringPrintf("%s:%13.9f ", - p->name().c_str(), Asum(p->data())); - } - if ((flag & kBackward) == kBackward && grad_.count() != 0) { - ret += StringPrintf("grad:%e ", Asum(grad_)); - for (Param* p : GetParams()) - ret += StringPrintf("%s:%13.9f ", - p->name().c_str(), Asum(p->grad())); - } - return ret; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/cudnn_softmaxloss.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc deleted file mode 100644 index 0d4ba45..0000000 --- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc +++ /dev/null @@ -1,83 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include "singa/neuralnet/loss_layer.h" -#include "singa/utils/blob.h" -#include "singa/utils/math_blob.h" -#include "singa/utils/math_kernel.h" - -namespace singa { -void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - LossLayer::Setup(conf, srclayers); - softmax_.Setup(conf, vector<Layer*> {srclayers.at(0)}); - data_.Reshape(softmax_.data(this).shape()); - data_.ShareData(softmax_.mutable_data(this), false); - batchsize_ = data_.shape(0); - dim_ = data_.count() / batchsize_; -} -void CudnnSoftmaxLossLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - softmax_.ComputeFeature(flag, srclayers); - Blob<int> label(batchsize_); - int *labelptr = label.mutable_cpu_data(); - // aux_data: vector<int>, convert vector to int array. - for (int i = 0; i < batchsize_; ++i) { - labelptr[i] = srclayers[1]->aux_data(this)[i]; - } - - Blob<float> loss(batchsize_); - singa_gpu_softmaxloss_forward(batchsize_, dim_, data_.gpu_data(), - label.gpu_data(), loss.mutable_gpu_data()); - loss_ += Asum(loss); - counter_++; -} - -void CudnnSoftmaxLossLayer::ComputeGradient(int flag, - const vector<Layer*>& srclayers) { - Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this); - Copy(data_, gsrcblob); - // gsrcblob->CopyFrom(data_); - float* gsrcptr = gsrcblob->mutable_gpu_data(); - - Blob<int> label(batchsize_); - int *labelptr = label.mutable_cpu_data(); - - // aux_data: vector<int>, convert vector to int array. - for (int i = 0; i < batchsize_; ++i) { - labelptr[i] = srclayers[1]->aux_data(this)[i]; - } - - singa_gpu_softmaxloss_backward(batchsize_, dim_, 1.0f, label.gpu_data(), - gsrcptr); - Scale(1.0f / batchsize_, gsrcblob); -} - -const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) { - if (debug) - return Layer::ToString(debug, flag); - - string disp = "Loss = " + std::to_string(loss_ / counter_); - counter_ = 0; - loss_ = 0; - return disp; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/euclidean.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc deleted file mode 100644 index 67c0cd5..0000000 --- a/src/neuralnet/loss_layer/euclidean.cc +++ /dev/null @@ -1,80 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include <glog/logging.h> -#include "singa/neuralnet/loss_layer.h" -#include "mshadow/tensor.h" - -namespace singa { - -using namespace mshadow; -using mshadow::cpu; - -using mshadow::Shape1; -using mshadow::Tensor; - -using std::vector; - -void EuclideanLossLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 2); - Layer::Setup(conf, srclayers); -} - -void EuclideanLossLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - int count = srclayers[0]->data(this).count(); - CHECK_EQ(count, srclayers[1]->data(this).count()); - const float* reconstruct_dptr = srclayers[0]->data(this).cpu_data(); - const float* input_dptr = srclayers[1]->data(this).cpu_data(); - float loss = 0; - for (int i = 0; i < count; i++) { - loss += (input_dptr[i] - reconstruct_dptr[i]) * - (input_dptr[i] - reconstruct_dptr[i]); - } - loss_ += loss / srclayers[0]->data(this).shape()[0]; - counter_++; -} - -void EuclideanLossLayer::ComputeGradient(int flag, - const vector<Layer*>& srclayers) { - int count = srclayers[0]->data(this).count(); - CHECK_EQ(count, srclayers[1]->data(this).count()); - const float* reconstruct_dptr = srclayers[0]->data(this).cpu_data(); - const float* input_dptr = srclayers[1]->data(this).cpu_data(); - Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this); - float* gsrcptr = gsrcblob->mutable_cpu_data(); - for (int i = 0; i < count; i++) { - gsrcptr[i] = reconstruct_dptr[i]-input_dptr[i]; - } - Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count())); - gsrc /= srclayers[0]->data(this).shape()[0]; -} -const std::string EuclideanLossLayer::ToString(bool debug, int flag) { - if (debug) - return Layer::ToString(debug, flag); - - string disp = "Loss = " + std::to_string(loss_ / counter_); - counter_ = 0; - loss_ = 0; - return disp; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/softmax.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc deleted file mode 100644 index 9d0cb1d..0000000 --- a/src/neuralnet/loss_layer/softmax.cc +++ /dev/null @@ -1,112 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - - -#include <glog/logging.h> -#include <algorithm> -#include "singa/neuralnet/loss_layer.h" -#include "mshadow/tensor.h" -#include "singa/utils/math_blob.h" - -namespace singa { - -using namespace mshadow; -using mshadow::cpu; - -using mshadow::Shape; -using mshadow::Shape1; -using mshadow::Shape2; -using mshadow::Tensor; - -using std::vector; - -void SoftmaxLossLayer::Setup(const LayerProto& proto, - const vector<Layer*>& srclayers) { - CHECK_EQ(srclayers.size(), 2); - LossLayer::Setup(proto, srclayers); - data_.Reshape(srclayers[0]->data(this).shape()); - batchsize_ = data_.shape()[0]; - dim_ = data_.count() / batchsize_; - topk_ = proto.softmaxloss_conf().topk(); - scale_ = proto.softmaxloss_conf().scale(); -} - -void SoftmaxLossLayer::ComputeFeature(int flag, - const vector<Layer*>& srclayers) { - Shape<2> s = Shape2(batchsize_, dim_); - Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s); - Tensor<cpu, 2> src(srclayers[0]->mutable_data(this)->mutable_cpu_data(), s); - Softmax(prob, src); - const auto& label = srclayers[1]->aux_data(this); - const float* probptr = prob.dptr; - float loss = 0, precision = 0; - for (int n = 0; n < batchsize_; n++) { - int ilabel = static_cast<int>(label[n]); - // CHECK_LT(ilabel,10); - CHECK_GE(ilabel, 0); - float prob_of_truth = probptr[ilabel]; - loss -= log(std::max(prob_of_truth, FLT_MIN)); - vector<std::pair<float, int> > probvec; - for (int j = 0; j < dim_; ++j) { - probvec.push_back(std::make_pair(probptr[j], j)); - } - std::partial_sort(probvec.begin(), probvec.begin() + topk_, probvec.end(), - std::greater<std::pair<float, int> >()); - // check if true label is in top k predictions - for (int k = 0; k < topk_; k++) { - if (probvec[k].second == static_cast<int>(label[n])) { - precision++; - break; - } - } - probptr += dim_; - } - CHECK_EQ(probptr, prob.dptr + prob.shape.Size()); - loss_ += loss * scale_ / (1.0f * batchsize_); - accuracy_ += precision * scale_ / (1.0f * batchsize_); - counter_++; -} - -void SoftmaxLossLayer::ComputeGradient(int flag, - const vector<Layer*>& srclayers) { - const auto& label = srclayers[1]->aux_data(); - Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this); - Copy(data_, gsrcblob); -// gsrcblob->CopyFrom(data_); - float* gsrcptr = gsrcblob->mutable_cpu_data(); - for (int n = 0; n < batchsize_; n++) { - gsrcptr[n*dim_ + static_cast<int>(label[n])] -= 1.0f; - } - Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count())); - gsrc *= scale_ / (1.0f * batchsize_); -} - -const std::string SoftmaxLossLayer::ToString(bool debug, int flag) { - if (debug) - return Layer::ToString(debug, flag); - - string disp = "Loss = " + std::to_string(loss_ / counter_) - + ", accuracy = " + std::to_string(accuracy_ / counter_); - counter_ = 0; - loss_ = accuracy_ = 0; - return disp; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/neuralnet.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc deleted file mode 100644 index b045e06..0000000 --- a/src/neuralnet/neuralnet.cc +++ /dev/null @@ -1,644 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - - -#include "singa/neuralnet/neuralnet.h" -#include <unordered_map> -#include <algorithm> -#include <queue> -#include "singa/utils/singleton.h" - -namespace singa { - -using std::map; -using std::string; -using std::vector; - -/** - * Check user defined net config and make some preprocessing, e.g., assing names - * to params. - * TODO(wnagwei) implement the following functions. - * 1. layer and paramname should not include '@', '+' and '#'. '@<suffix>' - * is used for identifying layer location/partition. '<prefix>#' is used for - * identifying the unrolled Param in RNN models. - * 2. assign names to unnamed Param, e.g., p<param_id>+<layer_name>. - */ -const NetProto NetConfPreprocess(const NetProto& conf) { - /* - string param_name = "$"; - // if user does not name the param, then name it based on layer name. - if (param->name() == "") { - param->set_name(layer->name() + param_name); - param_name += "$"; - } - */ - NetProto proto = conf; - for (int i = 0; i < proto.layer_size(); i++) { - if (!proto.layer(i).has_unroll_len()) - proto.mutable_layer(i)->set_unroll_len(proto.unroll_len()); - } - return proto; -} - -NeuralNet* NeuralNet::Create(const NetProto& net_conf, Phase phase, - int npartitions) { - const NetProto& full_net_conf = NetConfPreprocess(net_conf); - NetProto conf = full_net_conf; - conf.clear_layer(); - // flag=0: neither exclude nor include field appears - // flag=1: exclude field appears - // flag=2: include field appears - int flag = 0; - // exclude layers according to phase - // exclude field is deprecated - // please use include field instead - for (const auto& layer : full_net_conf.layer()) { - bool include = true; - for (auto p : layer.exclude()) { - // check whether both exclude and include field - // appear in the same .conf file - CHECK(flag == 0 || flag == 1) << "Don't use include and exclude together"; - if (p == phase) - include = false; - flag = 1; - } - // neural net only include the specified layer in the include field - for (auto p : layer.include()) { - CHECK(flag == 0 || flag == 2) << "Don't use include and exclude together"; - if (p == phase) { - include = true; - break; - } - include = false; - flag = 2; - } - if (include == false) continue; - LayerProto* layer_conf = conf.add_layer(); - layer_conf->CopyFrom(layer); - // using net partition if layer partition is not set - if (!layer_conf->has_partition_dim()) - layer_conf->set_partition_dim(net_conf.partition_dim()); - } - // LOG(INFO) << "Before unrolling: \n" << conf.DebugString(); - conf = Unrolling(conf); - - // Copy shared parameters for sharing param conf - std::vector<ParamProto*> shares; - std::unordered_map<string, ParamProto*> name2param; - for (int index = 0; index < conf.layer_size(); index++) { - LayerProto* layer = conf.mutable_layer(index); - for (int i = 0; i < layer->param_size(); i++) { - ParamProto* param = layer->mutable_param(i); - CHECK(name2param.find(param->name()) == name2param.end()) - << "Repeated param = " << param->name(); - name2param[param->name()] = param; - if (param->has_share_from() && param->share_from() != "") - shares.push_back(param); - } - } - for (auto param : shares) { - const std::string from = param->share_from(); - const std::string name = param->name(); - CHECK(name2param.find(from) != name2param.end()) - << "can't find share_from = " << from; - // CopyFrom will overwrite the name and share_from fields - param->CopyFrom(*name2param.at(from)); - param->set_name(name); - param->set_share_from(from); - } - LOG(INFO) << "Initial NeuralNet Config is\n" << conf.DebugString(); - // TODO(wangwei) create net based on net type, e.g., directed, undirected. - return new NeuralNet(conf, npartitions); -} - -const NetProto NeuralNet::Unrolling(const NetProto& net_conf) { - // Step 1: Unroll each layer & set parameter sharing - NetProto conf; - - std::vector<std::vector<int>> layer_groups; - std::unordered_map<string, int> org_layer_names; - for (int index = 0; index < net_conf.layer_size(); index ++) { - const LayerProto& org_layer = net_conf.layer(index); - org_layer_names[org_layer.name()] = index; // layer_name -> index - - std::vector<int> layer_group; - for (int i = 0; i < org_layer.unroll_len(); i ++) { // unroll - LayerProto* unroll_layer = conf.add_layer(); - unroll_layer->CopyFrom(org_layer); // create a new layer conf - // update layer names - std::stringstream sstm; - sstm << i << '#' << unroll_layer->name(); - unroll_layer->set_name(sstm.str()); - unroll_layer->set_unroll_index(i); - // update layer parameter sharing - for (int j = 0; j < unroll_layer->param_size(); j ++) { - ParamProto* param = unroll_layer->mutable_param(j); - if (i > 0) { - param->set_share_from("0#" + param->name()); - } - std::stringstream sstm1; - sstm1 << i << '#' << param->name(); - param->set_name(sstm1.str()); - } - // clear unrolling related fields - unroll_layer->clear_unroll_len(); - unroll_layer->clear_unroll_conn_type(); - unroll_layer->clear_shift(); - unroll_layer->clear_srclayers(); - - layer_group.push_back(conf.layer_size() - 1); - // LOG(ERROR) << "unrolling layer " << unroll_layer->name(); - } - layer_groups.push_back(layer_group); - } - // Step 2: Connect unrolled layers by setting `srclayers` - for (int index = 0; index < net_conf.layer_size(); index ++) { - const LayerProto& org_layer = net_conf.layer(index); - if (org_layer.srclayers_size() == 0) - continue; // no src layer - for (int i = 0; i < org_layer.srclayers_size(); i ++) { - const string& org_layer_src = org_layer.srclayers(i); - singa::UnrollConnType unroll_conn_type = kUnrollOneToOne; - if (i < org_layer.unroll_conn_type_size()) - unroll_conn_type = org_layer.unroll_conn_type(i); - unsigned int shift = 0; - if (i < org_layer.shift_size()) - shift = org_layer.shift(i); - - const std::vector<int> unroll_layer_srcs - = layer_groups[org_layer_names[org_layer_src]]; - - for (unsigned int j = 0; j < layer_groups[index].size(); j ++) { - LayerProto* unroll_layer = conf.mutable_layer(layer_groups[index][j]); - // Update src layers of `unroll_layer` by considering the types - if (unroll_conn_type == kUnrollOneToAll) { - for (int unroll_layer_src : unroll_layer_srcs) { - unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name()); - } - } else if (unroll_conn_type == kUnrollOneToOne) { - if (j < shift) continue; // no need to connect with the src - int unroll_layer_src = unroll_layer_srcs[j - shift]; - unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name()); - } else if (unroll_conn_type == kUnrollFirstToLast) { - if (j > 0) break; - int unroll_layer_src = - unroll_layer_srcs[unroll_layer_srcs.size() - 1]; - unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name()); - } - } - } - - // TODO(fanju): add LSTM when it is ready - if (org_layer.type() == kGRU) { // connect GRU layers - for (unsigned int j = 1; j < layer_groups[index].size(); j ++) { - LayerProto* unroll_layer = conf.mutable_layer(layer_groups[index][j]); - string srcname = conf.layer(layer_groups[index][j-1]).name(); - unroll_layer->add_srclayers(srcname); - } - } - } - return conf; -} - - -NeuralNet::NeuralNet(NetProto netproto, int npartitions) { - LOG(INFO) << "Constructing NeuralNet..."; - auto graph = CreateGraph(netproto, npartitions); - CreateNetFromGraph(graph); - PrepareDataStructures(); - - for (Node* node : graph->nodes()) - delete static_cast<LayerProto*>(node->proto); - delete graph; - LOG(INFO) << "NeuralNet Constructed"; - unroll_len_ = netproto.unroll_len(); -} - -NeuralNet::~NeuralNet() { - for (auto layer : layers_) - delete layer; -} -void NeuralNet::Load(const vector<string>& paths) { - unordered_map<string, Param*> params; - for (auto p : params_) { - params[p->name()] = p; - } - Load(paths, params); -} -void NeuralNet::Load(const vector<string>& paths, - const unordered_map<string, Param*>& params) { - for (const auto path : paths) { - LOG(ERROR) << "Load from checkpoint file " << path; - BlobProtos bps; - // TODO(wangwei) extend to read checkpoint from HDFS - ReadProtoFromBinaryFile(path.c_str(), &bps); - for (int i = 0; i < bps.name_size(); i++) { - if (params.find(bps.name(i)) != params.end()) { - // LOG(ERROR) << "Loading param = " << bps.name(i); - params.at(bps.name(i))->FromProto(bps.blob(i)); - params.at(bps.name(i))->set_version(bps.version(i)); - } - } - } -} - -void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) { - for (auto& layer : layers_) { - auto otherlayer = other->name2layer(layer->name()); - if (otherlayer != nullptr) { - const auto& otherparams = otherlayer->GetParams(); - const auto& params = layer->GetParams(); - CHECK_EQ(params.size(), otherparams.size()); - for (size_t i = 0; i < params.size(); i++) { - params[i]->ShareDataFrom(otherparams[i], cpu_only); - } - } - } -} - -// name of connection layers -string splitName(const string& layer) { return "split("+layer+")"; } -string sliceName(const string& layer) { return "slice("+layer+")"; } -string concateName(const string& layer) { return "concate("+layer+")"; } -string bridgeName(const string& src, const string& dst) { return src+"->"+dst; } -string bridgeSrcName(const string& src, const string& dst) { - return "bridge_src("+bridgeName(src, dst)+")"; -} -string bridgeDstName(const string& src, const string& dst) { - return "bridge_dst("+bridgeName(src, dst)+")"; -} - -ConnectionType dstLayerConnection(const LayerProto& proto) { - auto layer = Layer::Create(proto); - auto ret = layer->dst_layer_connection(); - delete layer; - return ret; -} - -ConnectionType srcNeuronConnection(const LayerProto& proto) { - auto layer = Layer::Create(proto); - auto ret = layer->src_neuron_connection(0); - delete layer; - return ret; -} - -NetProto NeuralNet::AddModelSplitLayers(const NetProto& netproto) { - NetProto net_w_split; - net_w_split.CopyFrom(netproto); - net_w_split.clear_layer(); - // calculate number of dst-layers for each layer - map<string, int> dst_count; - for (const LayerProto& layer : netproto.layer()) - for (const string& src_name : layer.srclayers()) - ++dst_count[src_name]; - // tag to add split layer if: - // dst_count[] > 1 && dst_layer_connection() = OneToOne - for (const LayerProto& layer : netproto.layer()) - if ((dst_count[layer.name()] > 1 && dstLayerConnection(layer) == kOneToOne)) - dst_count[layer.name()] = -dst_count[layer.name()]; - // add orginal layers and adjust srclayers - for (const LayerProto& layer : netproto.layer()) { - LayerProto* proto = net_w_split.add_layer(); - proto->CopyFrom(layer); - proto->clear_srclayers(); - for (const string& src_name : layer.srclayers()) - if (dst_count[src_name] < 0) - proto->add_srclayers(splitName(src_name)); - else - proto->add_srclayers(src_name); - } - // add split layers - for (const LayerProto& layer : netproto.layer()) { - if (dst_count[layer.name()] < 0) { - LayerProto* split_proto = net_w_split.add_layer(); - split_proto->set_name(splitName(layer.name())); - split_proto->set_type(kSplit); - split_proto->set_partition_dim(layer.partition_dim()); - split_proto->add_srclayers(layer.name()); - split_proto->mutable_split_conf() - ->set_num_splits(-dst_count[layer.name()]); - } - } - // LOG(INFO) << "NeuralNet Config After Model Split is\n" - // << net_w_split.DebugString(); - return net_w_split; -} - -NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto, - int npartitions) { - CHECK_GT(npartitions, 0); - NetProto net_w_connection; - net_w_connection.CopyFrom(netproto); - // if npartitions is 1, no need to add connection layers - if (npartitions == 1) return net_w_connection; - // add original layers, but remove all edges first - net_w_connection.clear_layer(); - map<string, LayerProto*> name2proto; - for (const LayerProto& layer : netproto.layer()) { - LayerProto* layer_proto = net_w_connection.add_layer(); - layer_proto->CopyFrom(layer); - layer_proto->clear_srclayers(); - name2proto[layer_proto->name()] = layer_proto; - } - /* - * Add Slice, Concate, Split Layers for Model Partition - * - * All cases are as follows: - * src_pdim | dst_pdim | connection_type | Action - * 0 | 0 | OneToOne | Direct Connection - * 1 | 1 | OneToOne | Direct Connection - * 0 | 0 | OneToAll | Direct Connection - * 1 | 0 | OneToOne | Slice -> Concate - * 0 | 1 | OneToOne | Slice -> Concate - * 1 | 0 | OneToAll | Slice -> Concate - * 0 | 1 | OneToAll | Split -> Concate - * 1 | 1 | OneToAll | Split -> Concate - * - * Logic: - * dst_pdim = 1 && OneToAll ? - * (YES) Split -> Concate - * (NO) src_pdim = dst_pdim ? - * (YES) Direct Connection - * (NO) Slice -> Concate - */ - for (const LayerProto& origin_layer : netproto.layer()) { - LayerProto* dst_layer = name2proto[origin_layer.name()]; - int dst_pdim = dst_layer->partition_dim(); - ConnectionType connection = srcNeuronConnection(*dst_layer); - for (const string& src_name : origin_layer.srclayers()) { - LayerProto* src_layer = name2proto[src_name]; - int src_pdim = src_layer->partition_dim(); - // dst_pdim = 1 && OneToAll ? - if (dst_pdim == 1 && connection == kOneToAll) { - // add split layer - LayerProto* split_layer = net_w_connection.add_layer(); - split_layer->set_name(splitName(src_layer->name())); - split_layer->set_type(kSplit); - split_layer->set_partition_dim(src_layer->partition_dim()); - split_layer->add_srclayers(src_layer->name()); - split_layer->mutable_split_conf()->set_num_splits(npartitions); - // add concate layer - LayerProto* concate_layer = net_w_connection.add_layer(); - concate_layer->set_name(concateName(split_layer->name())); - concate_layer->set_type(kConcate); - concate_layer->set_partition_dim(dst_layer->partition_dim()); - // concate on src_pdim - concate_layer->mutable_concate_conf() - ->set_concate_dim(src_layer->partition_dim()); - concate_layer->mutable_concate_conf()->set_num_concates(npartitions); - concate_layer->add_srclayers(split_layer->name()); - // connect dst_layer to concate layer - dst_layer->add_srclayers(concate_layer->name()); - } else { - // src_pdim = dst_pdim ? - if (dst_pdim == src_pdim) { - // direct connection - dst_layer->add_srclayers(src_layer->name()); - } else { - // add slice layer - LayerProto* slice_layer = net_w_connection.add_layer(); - slice_layer->set_name(sliceName(src_layer->name())); - slice_layer->set_type(kSlice); - slice_layer->set_partition_dim(src_layer->partition_dim()); - // slice on dst_pdim - slice_layer->mutable_slice_conf() - ->set_slice_dim(dst_layer->partition_dim()); - slice_layer->mutable_slice_conf()->set_num_slices(npartitions); - slice_layer->add_srclayers(src_layer->name()); - // add concate layer - LayerProto* concate_layer = net_w_connection.add_layer(); - concate_layer->set_name(concateName(slice_layer->name())); - concate_layer->set_type(kConcate); - concate_layer->set_partition_dim(dst_layer->partition_dim()); - // concate on src_pdim - concate_layer->mutable_concate_conf() - ->set_concate_dim(src_layer->partition_dim()); - concate_layer->mutable_concate_conf()->set_num_concates(npartitions); - concate_layer->add_srclayers(slice_layer->name()); - // connect dst_layer to concate layer - dst_layer->add_srclayers(concate_layer->name()); - } - } - } - } - LOG(INFO) << "NeuralNet Config After Adding Connection Layers is\n" - << net_w_connection.DebugString(); - return net_w_connection; -} - -Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) { - NetProto net_w_split = AddModelSplitLayers(netproto); - NetProto net_w_connection = - AddPartitionConnectionLayers(net_w_split, npartitions); - // for each original layer proto, create #npartitions of nodes - Graph* graph = new Graph(); - map<string, vector<Node*>> name2nodes; - map<string, const LayerProto*> name2proto; - for (const LayerProto& layer : net_w_connection.layer()) { - vector<Node*> nodes; - for (int i = 0; i < npartitions; i++) { - LayerProto *proto = new LayerProto(layer); - // differentiate partitions - string nodename = layer.name() + "@" + std::to_string(i); - proto->set_name(nodename); - proto->set_type(layer.type()); - proto->set_partition_dim(layer.partition_dim()); - proto->set_partition_id(i); - proto->set_num_partitions(npartitions); - Node* node = graph->AddNode(nodename, layer.name(), i, proto); - nodes.push_back(node); - // TODO(wangwei) update param name - } - name2nodes[layer.name()] = nodes; - name2proto[layer.name()] = &layer; - } - // connect layers, add bridge layers if partition id is different - for (const LayerProto& origin_layer : net_w_connection.layer()) { - vector<Node*> dst_nodes = name2nodes[origin_layer.name()]; - for (const string& src_name : origin_layer.srclayers()) { - vector<Node*> src_nodes = name2nodes[src_name]; - if (origin_layer.type() != kConcate) { - for (size_t i = 0; i < src_nodes.size(); ++i) { - CHECK_EQ(src_nodes[i]->partition_id, i); - CHECK_EQ(dst_nodes[i]->partition_id, i); - graph->AddEdge(src_nodes[i], dst_nodes[i]); - } - } else { - // need to add bridge layers - for (size_t i = 0; i < src_nodes.size(); ++i) { - CHECK_EQ(src_nodes[i]->partition_id, i); - for (size_t j = 0; j < dst_nodes.size(); ++j) { - CHECK_EQ(dst_nodes[j]->partition_id, j); - if (i == j) { // in same partition, no bridge needed - graph->AddEdge(src_nodes[i], dst_nodes[j]); - } else { // add bridges - // bridge src && dst layer - LayerProto *proto_bsrc = new LayerProto(); - LayerProto *proto_bdst = new LayerProto(); - string bsrc_name = bridgeSrcName(src_nodes[i]->name, - dst_nodes[j]->name); - string bdst_name = bridgeDstName(src_nodes[i]->name, - dst_nodes[j]->name); - proto_bsrc->set_name(bsrc_name); - proto_bdst->set_name(bdst_name); - proto_bsrc->set_type(kBridgeSrc); - proto_bdst->set_type(kBridgeDst); - proto_bsrc->set_partition_dim(origin_layer.partition_dim()); - proto_bdst->set_partition_dim(origin_layer.partition_dim()); - proto_bsrc->set_partition_id(src_nodes[i]->partition_id); - proto_bdst->set_partition_id(dst_nodes[j]->partition_id); - proto_bsrc->set_num_partitions(npartitions); - proto_bdst->set_num_partitions(npartitions); - Node* bsrc_node = graph->AddNode(bsrc_name, bsrc_name, i, - proto_bsrc); - Node* bdst_node = graph->AddNode(bdst_name, bdst_name, j, - proto_bdst); - graph->AddEdge(src_nodes[i], bsrc_node); - graph->AddEdge(bsrc_node, bdst_node); - graph->AddEdge(bdst_node, dst_nodes[j]); - } - } - } - } - } - } - graph->Sort(); - // DLOG(INFO) << "Pure graph structure\n" << graph->ToJson(); - return graph; -} - -void NeuralNet::CreateNetFromGraph(Graph* graph) { - // create one layer per node - for (Node* node : graph->nodes()) { - auto proto_ptr = static_cast<LayerProto*>(node->proto); - auto layer = Layer::Create(*proto_ptr); - layers_.push_back(layer); - name2layer_[node->name] = layer; - } - // connect layers - for (Node* node : graph->nodes()) { - auto layer = name2layer(node->name); - src_map_[layer] = vector<Layer*>{}; - for (Node* src : node->srcnodes) - src_map_[layer].push_back(name2layer(src->name)); - } - // setup layers - int paramid = 0; - map<string, string> layerinfo; - map<string, vector<Layer*>> share_param_layers; - for (Node* node : graph->nodes()) { - LOG(INFO) << "constructing graph: " << node->name; - auto layer = name2layer(node->name); - layer->Setup(*(static_cast<LayerProto*>(node->proto)), srclayers(layer)); - DLOG(INFO) << "constructing graph: " << layer->name(); - layerinfo[layer->name()] = IntVecToString(layer->data(nullptr).shape()); - for (auto param : layer->GetParams()) { - param->set_id(paramid++); - } - if (layer->partition_dim() == 0) - share_param_layers[node->origin].push_back(layer); - } - // create map from param name to param ptr - std::unordered_map<string, Param*> name2param; - for (auto layer : layers_) { - for (auto param : layer->GetParams()) { - name2param[param->name()] = param; - } - } - for (auto & entry : share_param_layers) { - // overwrite entries for replicated params due to layer partition (dim 0). - for (auto *param : entry.second.front()->GetParams()) - name2param.at(param->name()) = param; - } - // share params based on share_from field - for (auto & entry : name2param) { - Param* param = entry.second; - const string share_from = param->share_from(); - if (param->share_from() != "") { - if (name2param.find(share_from) != name2param.end()) { - param->ShareDataFrom(name2param.at(param->share_from()), false); - } else { - LOG(FATAL) << "No param with the name (share_from) " << share_from; - } - } - } - - // share params due to laye unrolling - for (auto & entry : name2param) { - Param* param = entry.second; - auto pos = param->name().find("#"); - if (pos != std::string::npos && param->owner() != param->id()) { - string from = "0" + param->name().substr(pos); - CHECK(name2param.find(from) != name2param.end()) - << "Can't find owner = " << from << " for param = " << param->name(); - Param* owner = name2param.at(from); - param->ShareFrom(owner); - } - } - // share Params for layers generated (partitioned) from the same origin layer - for (auto & entry : share_param_layers) { - const auto& owner = entry.second.begin(); - const auto& owner_params = (*owner)->GetParams(); - for (auto it = owner + 1; it != entry.second.end(); it++) { - auto params = (*it)->GetParams(); - CHECK_EQ(params.size(), owner_params.size()); - for (size_t i = 0; i < params.size(); i++) - params.at(i)->ShareDataFrom(owner_params.at(i), true); - } - } -} - -void NeuralNet::PrepareDataStructures() { - params_.clear(); - paramid2param_.clear(); - name2layer_.clear(); - for (auto& layer : layers_) { - name2layer_[layer->name()] = layer; - for (Param* p : layer->GetParams()) { - paramid2param_[p->id()] = p; - params_.push_back(p); - } - } -} - -const Graph NeuralNet::ToGraph(bool include_shape) const { - Graph g; - map<string, string> attrs; - attrs["shape"] = "box"; - vector<string> colors {"black", "red", "yellow", "blue"}; - for (auto layer : layers_) { - LOG_IF(WARNING, layer->partition_id() >= static_cast<int>(colors.size())) - << "Too many partitions for displaying"; - attrs["color"] = colors[layer->partition_id() % colors.size()]; - if (include_shape) { - attrs["label"] = "shape: "; - for (const auto& x : layer->data(nullptr).shape()) - attrs["label"] += std::to_string(x) + " "; - } - g.AddNode(layer->name(), attrs); - } - - for (auto layer : layers_) - for (auto src : src_map_.at(layer)) - g.AddEdge(src->name(), layer->name()); - return g; -} -} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/neuron_layer/activation.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/neuron_layer/activation.cc b/src/neuralnet/neuron_layer/activation.cc deleted file mode 100644 index f75961e..0000000 --- a/src/neuralnet/neuron_layer/activation.cc +++ /dev/null @@ -1,87 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ -#include "singa/neuralnet/neuron_layer.h" -#include "singa/utils/math_blob.h" -#include "singa/utils/singa_op.h" -#include "singa/proto/job.pb.h" -namespace singa { - -void ActivationLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - NeuronLayer::Setup(conf, srclayers); - data_.ReshapeLike(srclayers[0]->data(this)); - grad_.ReshapeLike(data_); - if (conf.share_src_blobs()) { - data_.ShareData(srclayers[0]->mutable_data(this), false); - grad_.ShareData(srclayers[0]->mutable_grad(this), false); - } -} -void -ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - switch (layer_conf_.activation_conf().type()) { - case RELU: - Map<op::Relu<float>, float>(srclayers[0]->data(this), &data_); - break; - case SIGMOID: - Map<op::Sigmoid<float>, float>(srclayers[0]->data(this), &data_); - break; - case TANH: - Map<op::Tanh<float>, float>(srclayers[0]->data(this), &data_); - break; - /* - case ActivationType_STANH: - Map<op::STanh<float>, float>(srclayers[0]->data(this), &data_); - break; - */ - default: - LOG(ERROR) << "Unknow activation type " << - layer_conf_.activation_conf().type(); - } -} -void -ActivationLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - Blob<float> * gsrc = srclayers[0]->mutable_grad(this); - switch (layer_conf_.activation_conf().type()) { - case RELU: - Map<op::ReluGrad<float>, float>(data_, gsrc); - Mult(*gsrc, grad_, gsrc); - break; - case SIGMOID: - Map<op::SigmoidGrad<float>, float>(data_, gsrc); - Mult(*gsrc, grad_, gsrc); - break; - case TANH: - Map<op::TanhGrad<float>, float>(data_, gsrc); - Mult(*gsrc, grad_, gsrc); - break; - /* - case ActivationType_STANH: - Map<op::STanhGrad<float>, float>(data_, gsrc); - Mult(*gsrc, grad_, gsrc); - break; - */ - default: - LOG(ERROR) << "Unknow activation type " << - layer_conf_.activation_conf().type(); - } -} - -} // namespace singa
