Repository: incubator-singa Updated Branches: refs/heads/master 9ff176c30 -> 714fd2cb3
SIGNA-97 Add HDFS Store Check with cpplint Remove 'make compile' option in Makefile.example Add check for unrecognized backend type Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/714fd2cb Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/714fd2cb Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/714fd2cb Branch: refs/heads/master Commit: 714fd2cb35484a682dec8c01b9f80c88339b767d Parents: befe5ca Author: WANG Sheng <[email protected]> Authored: Sat Jan 2 19:22:31 2016 +0800 Committer: WANG Sheng <[email protected]> Committed: Sat Jan 2 19:58:14 2016 +0800 ---------------------------------------------------------------------- examples/cifar10/Makefile.example | 8 +------- examples/cifar10/create_data.cc | 3 ++- examples/mnist/Makefile.example | 8 +------- examples/mnist/create_data.cc | 5 +++-- include/singa/io/kvfile_store.h | 2 +- include/singa/io/textfile_store.h | 2 +- src/io/kvfile_store.cc | 2 +- src/io/store.cc | 16 ++++++++++------ 8 files changed, 20 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/examples/cifar10/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example index 16dc052..72c326b 100644 --- a/examples/cifar10/Makefile.example +++ b/examples/cifar10/Makefile.example @@ -30,11 +30,6 @@ cifar-10-binary-bin: wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz tar xf cifar-10-binary.tar.gz -compile: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \ - -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ - -Wl,-rpath=../../.libs/ -o create_data.bin - create: $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \ -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ @@ -42,8 +37,7 @@ create: ./create_data.bin cifar-10-batches-bin . create_hdfs: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \ + $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \ -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ -Wl,-rpath=../../.libs/ -o create_data.bin ./create_data.bin cifar-10-batches-bin $(HDFS_DIR) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/examples/cifar10/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/cifar10/create_data.cc b/examples/cifar10/create_data.cc index 37c58f6..5564c38 100644 --- a/examples/cifar10/create_data.cc +++ b/examples/cifar10/create_data.cc @@ -69,7 +69,8 @@ void create_data(const string& input_folder, const string& output_folder) { for (int i = 0; i < kCIFARImageNBytes; i++) mean.add_data(0.f); - string store_backend = (output_folder.find("hdfs")!=-1) ? "hdfsfile" : "kvfile"; + string store_backend = output_folder.find("hdfs") !=-1 ? + "hdfsfile" : "kvfile"; auto store = singa::io::CreateStore(store_backend); CHECK(store->Open(output_folder + "/train_data.bin", singa::io::kCreate)); LOG(INFO) << "Preparing training data"; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/examples/mnist/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example index 8c1c838..7b516e7 100644 --- a/examples/mnist/Makefile.example +++ b/examples/mnist/Makefile.example @@ -26,7 +26,6 @@ libs :=singa glog protobuf HDFS_MNIST_TRAIN := hdfs://node0:9000/examples/mnist/train_data.bin HDFS_MNIST_TEST := hdfs://node0:9000/examples/mnist/test_data.bin - download: mnist mnist: @@ -37,11 +36,6 @@ mnist: gunzip train-images-idx3-ubyte.gz && gunzip train-labels-idx1-ubyte.gz gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz -compile: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 -I../../include \ - -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ - -o create_data.bin - create: $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \ -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ @@ -50,7 +44,7 @@ create: ./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte test_data.bin create_hdfs: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \ + $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 -I../../include \ -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ -o create_data.bin ./create_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte $(HDFS_MNIST_TRAIN) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/examples/mnist/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc index ff166b4..34c287f 100644 --- a/examples/mnist/create_data.cc +++ b/examples/mnist/create_data.cc @@ -79,8 +79,9 @@ void create_data(const char* image_filename, const char* label_filename, cols = swap_endian(cols); // read backend from the job.conf - string store_backend = (string(output).find("hdfs")!=-1) ? "hdfsfile" : "kvfile"; - auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate); + string store_backend = string(output).find("hdfs") != -1 ? + "hdfsfile" : "kvfile"; + auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate); char label; char* pixels = new char[rows * cols]; int count = 0; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/include/singa/io/kvfile_store.h ---------------------------------------------------------------------- diff --git a/include/singa/io/kvfile_store.h b/include/singa/io/kvfile_store.h index 73e4127..50b8f4f 100644 --- a/include/singa/io/kvfile_store.h +++ b/include/singa/io/kvfile_store.h @@ -41,7 +41,7 @@ class KVFileStore : public Store { void Close() override; bool Read(std::string* key, std::string* value) override; void SeekToFirst() override; - void Seek(int offset) override; + void Seek(int offset) override; bool Write(const std::string& key, const std::string& value) override; void Flush() override; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/include/singa/io/textfile_store.h ---------------------------------------------------------------------- diff --git a/include/singa/io/textfile_store.h b/include/singa/io/textfile_store.h index 2cc6571..83bcbfa 100644 --- a/include/singa/io/textfile_store.h +++ b/include/singa/io/textfile_store.h @@ -41,7 +41,7 @@ class TextFileStore : public Store { void Close() override; bool Read(std::string* key, std::string* value) override; void SeekToFirst() override; - void Seek(int offset) override; + void Seek(int offset) override; bool Write(const std::string& key, const std::string& value) override; void Flush() override; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/src/io/kvfile_store.cc ---------------------------------------------------------------------- diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc index e77f49b..a2a40cd 100644 --- a/src/io/kvfile_store.cc +++ b/src/io/kvfile_store.cc @@ -56,7 +56,7 @@ void KVFileStore::SeekToFirst() { file_->SeekToFirst(); } -void KVFileStore::Seek(int offset){ +void KVFileStore::Seek(int offset) { LOG(FATAL) << "Operation not supported."; } http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/714fd2cb/src/io/store.cc ---------------------------------------------------------------------- diff --git a/src/io/store.cc b/src/io/store.cc index f2a4404..4621772 100644 --- a/src/io/store.cc +++ b/src/io/store.cc @@ -20,13 +20,15 @@ *************************************************************/ #include "singa/io/store.h" +#include <glog/logging.h> #include "singa/io/kvfile_store.h" #include "singa/io/textfile_store.h" #ifdef USE_HDFS #include "singa/io/hdfs_store.h" #endif -namespace singa { namespace io { +namespace singa { +namespace io { Store* CreateStore(const std::string& backend) { Store *store = nullptr; @@ -38,21 +40,23 @@ Store* CreateStore(const std::string& backend) { #ifdef USE_LMDB if (backend == "lmdb") { - return new LMDBStore(); + store = new LMDBStore(); } #endif #ifdef USE_OPENCV if (backend == "imagefolder") { - store = new ImageFolderStore(); + store = new ImageFolderStore(); } #endif #ifdef USE_HDFS if (backend == "hdfsfile") { - store = new HDFSStore(); + store = new HDFSStore(); } #endif + + CHECK(store) << "Backend type (" << backend << ") not recognized"; return store; } @@ -61,6 +65,6 @@ Store* OpenStore(const string& backend, const string& path, Mode mode) { store->Open(path, mode); return store; } -} /* io */ -} /* singa */ +} // namespace io +} // namespace singa
