SINGA-82 Refactor input layers using data store abstraction add header guard for textfile_store.h, image_transform.h format code
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/dc7f1996 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/dc7f1996 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/dc7f1996 Branch: refs/heads/master Commit: dc7f1996df26687612f61945bbb58ccbe0db65f4 Parents: 4afc7b8 Author: wang sheng <[email protected]> Authored: Wed Oct 7 15:09:21 2015 +0800 Committer: wang sheng <[email protected]> Committed: Wed Oct 7 15:19:59 2015 +0800 ---------------------------------------------------------------------- examples/mnist/create_data.cc | 5 +++-- examples/rnnlm/rnnlm.cc | 5 +++-- include/io/kvfile.h | 6 +++--- include/io/kvfile_store.h | 9 +++++---- include/io/store.h | 10 +++++++--- include/io/textfile_store.h | 12 +++++++++--- include/neuralnet/input_layer.h | 2 +- include/singa.h | 2 +- include/utils/image_transform.h | 8 +++++++- include/utils/tokenizer.h | 25 +++++++++++++++---------- src/io/kvfile.cc | 7 ++++--- src/io/kvfile_store.cc | 10 +++++----- src/io/textfile_store.cc | 12 ++++++------ src/neuralnet/input_layer.cc | 8 ++++---- src/test/test_csv_record_layer.cc | 2 +- src/test/test_proto_record_layer.cc | 2 +- src/utils/image_transform.cc | 3 ++- 17 files changed, 77 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/mnist/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc index aad1f56..14b279f 100644 --- a/examples/mnist/create_data.cc +++ b/examples/mnist/create_data.cc @@ -105,9 +105,10 @@ void create_data(const char* image_filename, const char* label_filename, int main(int argc, char** argv) { if (argc != 4) { - std::cout<<"This program create a DataShard for a MNIST dataset\n" + std::cout << "This program create a DataShard for a MNIST dataset\n" "Usage:\n" - " create_shard.bin input_image_file input_label_file output_db_file\n" + " create_shard.bin input_image_file input_label_file" + " output_db_file\n" "The MNIST dataset could be downloaded at\n" " http://yann.lecun.com/exdb/mnist/\n"; "You should gunzip them after downloading."; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/rnnlm/rnnlm.cc ---------------------------------------------------------------------- diff --git a/examples/rnnlm/rnnlm.cc b/examples/rnnlm/rnnlm.cc index a6b65f7..641b465 100644 --- a/examples/rnnlm/rnnlm.cc +++ b/examples/rnnlm/rnnlm.cc @@ -18,12 +18,13 @@ * under the License. * *************************************************************/ +#include "./rnnlm.h" + #include <string> #include <algorithm> #include "mshadow/tensor.h" #include "mshadow/tensor_expr.h" #include "mshadow/cxxnet_op.h" -#include "./rnnlm.h" #include "./rnnlm.pb.h" namespace rnnlm { @@ -64,7 +65,7 @@ void DataLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) { window_ = 0; } -void SetInst(int k, WordRecord& word, Blob<float>* to) { +void SetInst(int k, const WordRecord& word, Blob<float>* to) { float* dptr = to->mutable_cpu_data() + k * 4; dptr[0] = static_cast<float>(word.word_index()); dptr[1] = static_cast<float>(word.class_index()); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile.h ---------------------------------------------------------------------- diff --git a/include/io/kvfile.h b/include/io/kvfile.h index d70f198..27dd35e 100644 --- a/include/io/kvfile.h +++ b/include/io/kvfile.h @@ -32,8 +32,8 @@ #include <google/protobuf/message.h> #endif -namespace singa { namespace io { - +namespace singa { +namespace io { /** * KVFile stores training/validation/test tuples. @@ -171,7 +171,7 @@ class KVFile { //!< bytes in buf_, used in reading int bufsize_ = 0; }; -} /* io */ +} // namespace io /** * @deprecated {ShardData is deprecated! Use KVFile}. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile_store.h ---------------------------------------------------------------------- diff --git a/include/io/kvfile_store.h b/include/io/kvfile_store.h index c3fd868..bcd70eb 100644 --- a/include/io/kvfile_store.h +++ b/include/io/kvfile_store.h @@ -19,7 +19,6 @@ * *************************************************************/ - #ifndef SINGA_IO_KVFILE_STORE_H_ #define SINGA_IO_KVFILE_STORE_H_ @@ -27,7 +26,8 @@ #include "io/store.h" #include "io/kvfile.h" -namespace singa { namespace io { +namespace singa { +namespace io { /** * Use the KVFile as the data storage. @@ -49,6 +49,7 @@ class KVFileStore : public Store { Mode mode_; }; -} /* io */ -} /* singa */ +} // namespace io +} // namespace singa + #endif // SINGA_IO_KVFILE_STORE_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/store.h ---------------------------------------------------------------------- diff --git a/include/io/store.h b/include/io/store.h index f3cc282..15afb6a 100644 --- a/include/io/store.h +++ b/include/io/store.h @@ -24,7 +24,9 @@ #include <string> -namespace singa { namespace io { +namespace singa { +namespace io { + using std::string; enum Mode { kCreate, kRead, kAppend }; @@ -96,6 +98,8 @@ Store* CreateStore(const string& backend); * @param[in] mode kRead or kCreate or kAppend */ Store* OpenStore(const string& backend, const string& path, Mode mode); -} // namespace io -} /* singa */ + +} // namespace io +} // namespace singa + #endif // SINGA_IO_STORE_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/textfile_store.h ---------------------------------------------------------------------- diff --git a/include/io/textfile_store.h b/include/io/textfile_store.h index 788dc20..5450f00 100644 --- a/include/io/textfile_store.h +++ b/include/io/textfile_store.h @@ -19,11 +19,15 @@ * *************************************************************/ +#ifndef SINGA_IO_TEXTFILE_STORE_H_ +#define SINGA_IO_TEXTFILE_STORE_H_ #include <fstream> +#include <string> #include "io/store.h" -namespace singa { namespace io { +namespace singa { +namespace io { /** * Use text file as the data storage, one line per tuple. * @@ -45,6 +49,8 @@ class TextFileStore : public Store { std::fstream* fs_ = nullptr; Mode mode_; }; -} /* io */ -} /* singa */ +} // namespace io +} // namespace singa + +#endif // SINGA_IO_TEXTFILE_STORE_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/neuralnet/input_layer.h ---------------------------------------------------------------------- diff --git a/include/neuralnet/input_layer.h b/include/neuralnet/input_layer.h index 65e42cb..2e8725e 100644 --- a/include/neuralnet/input_layer.h +++ b/include/neuralnet/input_layer.h @@ -24,9 +24,9 @@ #include <string> #include <vector> +#include "io/store.h" #include "neuralnet/layer.h" #include "utils/data_shard.h" -#include "io/store.h" /** * \file this file includes the declarations of input layers that inherit the * base InputLayer to load input features. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/singa.h ---------------------------------------------------------------------- diff --git a/include/singa.h b/include/singa.h index 63acb0a..ddfec0e 100644 --- a/include/singa.h +++ b/include/singa.h @@ -23,6 +23,7 @@ #define SINGA_SINGA_H_ #include "comm/socket.h" +#include "io/store.h" #include "neuralnet/neuralnet.h" #include "neuralnet/layer.h" #include "proto/job.pb.h" @@ -31,7 +32,6 @@ #include "utils/param.h" #include "utils/singleton.h" #include "utils/factory.h" -#include "io/store.h" #include "./driver.h" #endif // SINGA_SINGA_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/image_transform.h ---------------------------------------------------------------------- diff --git a/include/utils/image_transform.h b/include/utils/image_transform.h index 24fe75d..2867ad2 100644 --- a/include/utils/image_transform.h +++ b/include/utils/image_transform.h @@ -18,6 +18,10 @@ * under the License. * *************************************************************/ + +#ifndef SINGA_UTILS_IMAGE_TRANSFORM_H_ +#define SINGA_UTILS_IMAGE_TRANSFORM_H_ + #include <glog/logging.h> // TODO(wangwei) provide image transformation API, the implementation can be // done by opencv, manual transform, or mshadow. @@ -26,4 +30,6 @@ namespace singa { void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop, int w_crop, int h_offset, int w_offset, int channel, int height, int width, float scale, float* out); -} /* singa */ +} // namespace singa + +#endif // SINGA_UTILS_IMAGE_TRANSFORM_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/tokenizer.h ---------------------------------------------------------------------- diff --git a/include/utils/tokenizer.h b/include/utils/tokenizer.h index fc6ba8a..c66e0af 100644 --- a/include/utils/tokenizer.h +++ b/include/utils/tokenizer.h @@ -18,10 +18,13 @@ * under the License. * *************************************************************/ -#ifndef SINGA_UTILS_TOKENIER_H_ -#define SINGA_UTILS_TOKENIER_H_ -#include <string> + +#ifndef SINGA_UTILS_TOKENIZER_H_ +#define SINGA_UTILS_TOKENIZER_H_ + #include <glog/logging.h> +#include <string> + namespace singa { /** * Tokenize a string. @@ -48,12 +51,14 @@ class Tokenizer { out = buf_.substr(start, pos); return *this; } - bool Valid() { return start_ < buf_.length(); } - private: - unsigned start_; - std::string sep_; - const std::string& buf_; + + private: + unsigned start_; + std::string sep_; + const std::string& buf_; }; -} /* singa */ -#endif // SINGA_UTILS_TOKENIER_H_ + +} // namespace singa + +#endif // SINGA_UTILS_TOKENIZER_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile.cc ---------------------------------------------------------------------- diff --git a/src/io/kvfile.cc b/src/io/kvfile.cc index aa52150..465d3ee 100644 --- a/src/io/kvfile.cc +++ b/src/io/kvfile.cc @@ -22,9 +22,10 @@ #include "io/kvfile.h" #include <glog/logging.h> -#include <string> -namespace singa { namespace io { +namespace singa { +namespace io { + KVFile::KVFile(const std::string& path, Mode mode, int capacity) : path_(path), mode_(mode), capacity_(capacity) { buf_ = new char[capacity]; @@ -213,5 +214,5 @@ bool KVFile::PrepareNextField(int size) { return true; } -} /* io */ +} // namespace io } // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile_store.cc ---------------------------------------------------------------------- diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc index 11609bf..4d251f9 100644 --- a/src/io/kvfile_store.cc +++ b/src/io/kvfile_store.cc @@ -19,11 +19,12 @@ * *************************************************************/ - #include "io/kvfile_store.h" + #include <glog/logging.h> -namespace singa { namespace io { +namespace singa { +namespace io { bool KVFileStore::Open(const std::string& source, Mode mode) { CHECK(file_ == nullptr); @@ -66,6 +67,5 @@ void KVFileStore::Flush() { file_->Flush(); } -} /* io */ - -} /* singa */ +} // namespace io +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/textfile_store.cc ---------------------------------------------------------------------- diff --git a/src/io/textfile_store.cc b/src/io/textfile_store.cc index 77694a0..9371de7 100644 --- a/src/io/textfile_store.cc +++ b/src/io/textfile_store.cc @@ -23,13 +23,14 @@ #include "io/textfile_store.h" #include <glog/logging.h> -namespace singa { namespace io { +namespace singa { +namespace io { + bool TextFileStore::Open(const std::string& source, Mode mode) { if (mode == kRead) fs_ = new std::fstream(source, std::fstream::in); - else if (mode == kCreate) { + else if (mode == kCreate) fs_ = new std::fstream(source, std::fstream::out); - } mode_ = mode; return fs_->is_open(); } @@ -79,6 +80,5 @@ void TextFileStore::Flush() { fs_->flush(); } -} /* io */ - -} /* singa */ +} // namespace io +} // namespace singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/neuralnet/input_layer.cc ---------------------------------------------------------------------- diff --git a/src/neuralnet/input_layer.cc b/src/neuralnet/input_layer.cc index 1599def..d4bef70 100644 --- a/src/neuralnet/input_layer.cc +++ b/src/neuralnet/input_layer.cc @@ -104,10 +104,10 @@ void StoreInputLayer::ComputeFeature(int flag, store_->SeekToFirst(); CHECK(store_->Read(&key, &val)); } - random_skip_ --; + random_skip_--; } } - for (int k = 0; k < batchsize_; k++){ + for (int k = 0; k < batchsize_; k++) { if (!store_->Read(&key, &val)) { store_->SeekToFirst(); CHECK(store_->Read(&key, &val)); @@ -153,7 +153,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag, if (mean_.count()) { const float* mean = mean_.cpu_data(); - for (int k = 0; k < batchsize_; k++){ + for (int k = 0; k < batchsize_; k++) { float* dptr = data_.mutable_cpu_data() + k * mean_.count(); for (int i = 0; i < mean_.count(); i++) { dptr[i] -= mean[i]; @@ -162,7 +162,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag, } if (std_.count()) { const float* std = std_.cpu_data(); - for (int k = 0; k < batchsize_; k++){ + for (int k = 0; k < batchsize_; k++) { float* dptr = data_.mutable_cpu_data() + k * std_.count(); for (int i = 0; i < std_.count(); i++) { dptr[i] /= std[i]; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_csv_record_layer.cc ---------------------------------------------------------------------- diff --git a/src/test/test_csv_record_layer.cc b/src/test/test_csv_record_layer.cc index 4517698..2839db9 100644 --- a/src/test/test_csv_record_layer.cc +++ b/src/test/test_csv_record_layer.cc @@ -48,7 +48,7 @@ class CSVRecordLayerTest : public ::testing::Test { TEST_F(CSVRecordLayerTest, Setup) { singa::CSVRecordLayer layer; layer.Setup(csv_conf, std::vector<singa::Layer*>{}); - EXPECT_EQ(2, layer.aux_data().size()); + EXPECT_EQ(2, static_cast<int>(layer.aux_data().size())); EXPECT_EQ(6, layer.data(nullptr).count()); } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_proto_record_layer.cc ---------------------------------------------------------------------- diff --git a/src/test/test_proto_record_layer.cc b/src/test/test_proto_record_layer.cc index 0a39c13..03c7230 100644 --- a/src/test/test_proto_record_layer.cc +++ b/src/test/test_proto_record_layer.cc @@ -79,7 +79,7 @@ class ProtoRecordLayerTest : public ::testing::Test { TEST_F(ProtoRecordLayerTest, Setup) { singa::ProtoRecordLayer layer; layer.Setup(image_conf, std::vector<singa::Layer*>{}); - EXPECT_EQ(2, layer.aux_data().size()); + EXPECT_EQ(2, static_cast<int>(layer.aux_data().size())); EXPECT_EQ(6, layer.data(nullptr).count()); } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/utils/image_transform.cc ---------------------------------------------------------------------- diff --git a/src/utils/image_transform.cc b/src/utils/image_transform.cc index 4851334..1e18e6b 100644 --- a/src/utils/image_transform.cc +++ b/src/utils/image_transform.cc @@ -53,4 +53,5 @@ void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop, } } } -} /* singa */ + +} // namespace singa
