SINGA-97 Add HDFS Store Revert changes to the examples directory, assuming that users upload data to HDFS manually.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/374f11d6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/374f11d6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/374f11d6 Branch: refs/heads/master Commit: 374f11d62e83df6b681c36f9557b6825ab8c236a Parents: 8a07a29 Author: Anh Dinh <[email protected]> Authored: Mon Dec 28 15:51:40 2015 +0800 Committer: WANG Sheng <[email protected]> Committed: Sat Jan 2 19:58:14 2016 +0800 ---------------------------------------------------------------------- Makefile.am | 2 +- examples/cifar10/Makefile.example | 5 ----- examples/cifar10/create_data.cc | 30 ++++++++++++------------------ examples/mnist/Makefile.example | 5 ----- examples/mnist/create_data.cc | 13 +------------ include/singa/io/hdfsfile.h | 2 +- src/io/kvfile_store.cc | 4 +++- 7 files changed, 18 insertions(+), 43 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/Makefile.am ---------------------------------------------------------------------- diff --git a/Makefile.am b/Makefile.am index d78a150..cc308f7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,7 +33,7 @@ CFLAGS += $(DEBUG) CXXFLAGS += $(DEBUG) AC_CXXFLAGS = $(DEBUG) -INCLUDES = -I$(top_srcdir)/include -I/usr/local/include/hdfs +INCLUDES = -I$(top_srcdir)/include -I/usr/local/include PROTOS := $(top_srcdir)/src/proto/singa.proto \ $(top_srcdir)/src/proto/job.proto \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example index 775e165..dd65d7d 100644 --- a/examples/cifar10/Makefile.example +++ b/examples/cifar10/Makefile.example @@ -28,11 +28,6 @@ cifar-10-binary-bin: wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz tar xf cifar-10-binary.tar.gz -compile: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \ - -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ - -Wl,-rpath=../../.libs/ -o create_data.bin - create: $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \ -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/cifar10/create_data.cc b/examples/cifar10/create_data.cc index 05169d8..5873c0e 100644 --- a/examples/cifar10/create_data.cc +++ b/examples/cifar10/create_data.cc @@ -19,16 +19,16 @@ * *************************************************************/ - -/** - * Create training and test DataShard for CIFAR dataset. - * It is adapted from convert_cifar_data from Caffe. - * create_shard.bin <input> <output_folder> - * - * Read from JobConf object the option to use KVfile, HDFS or other (1st layer - * store_conf object). - * To load to HDFS, specify "hdfs://namenode/examples" as the output folder - */ +// +// This code creates training and test DataShard for CIFAR dataset. +// It is adapted from the convert_cifar_data from Caffe +// +// Usage: +// create_shard.bin input_folder output_folder +// +// The CIFAR dataset could be downloaded at +// http://www.cs.toronto.edu/~kriz/cifar.html +// #include <glog/logging.h> #include <fstream> @@ -38,8 +38,6 @@ #include "singa/io/store.h" #include "singa/proto/common.pb.h" -#include "singa/utils/common.h" -#include "singa/proto/job.pb.h" using std::string; @@ -47,7 +45,6 @@ const int kCIFARSize = 32; const int kCIFARImageNBytes = 3072; const int kCIFARBatchSize = 10000; const int kCIFARTrainBatches = 5; -const char JOB_CONFIG[] = "job.conf"; void read_image(std::ifstream* file, int* label, char* buffer) { char label_char; @@ -61,6 +58,7 @@ void create_data(const string& input_folder, const string& output_folder) { int label; char str_buffer[kCIFARImageNBytes]; string rec_buf; + singa::RecordProto image; image.add_shape(3); image.add_shape(kCIFARSize); @@ -71,11 +69,7 @@ void create_data(const string& input_folder, const string& output_folder) { for (int i = 0; i < kCIFARImageNBytes; i++) mean.add_data(0.f); - singa::JobProto job_proto; - singa::ReadProtoFromTextFile(JOB_CONFIG, &job_proto); - string store_backend = - job_proto.neuralnet().layer(0).store_conf().backend(); - auto store = singa::io::CreateStore(store_backend); + auto store = singa::io::CreateStore("kvfile"); CHECK(store->Open(output_folder + "/train_data.bin", singa::io::kCreate)); LOG(INFO) << "Preparing training data"; int count = 0; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example index ba2308b..733633d 100644 --- a/examples/mnist/Makefile.example +++ b/examples/mnist/Makefile.example @@ -33,11 +33,6 @@ mnist: gunzip train-images-idx3-ubyte.gz && gunzip train-labels-idx1-ubyte.gz gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz -compile: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 -I../../include \ - -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ - -o create_data.bin - create: $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \ -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc index 66a4905..5e51e97 100644 --- a/examples/mnist/create_data.cc +++ b/examples/mnist/create_data.cc @@ -38,19 +38,14 @@ #include "singa/io/store.h" #include "singa/utils/common.h" #include "singa/proto/common.pb.h" -#include "singa/proto/job.pb.h" using std::string; -const char JOB_CONFIG[] = "job.conf"; - uint32_t swap_endian(uint32_t val) { val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF); return (val << 16) | (val >> 16); } -// output is the full path, unlike create_data in CIFAR with only -// specifies the directory void create_data(const char* image_filename, const char* label_filename, const char* output) { // Open files @@ -81,13 +76,7 @@ void create_data(const char* image_filename, const char* label_filename, image_file.read(reinterpret_cast<char*>(&cols), 4); cols = swap_endian(cols); - // read backend from the job.conf - singa::JobProto job_proto; - singa::ReadProtoFromTextFile(JOB_CONFIG.c_str(), &job_proto); - string store_backend = - job_proto.neuralnet().layer(0).store_conf().backend(); - - auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate); + auto store = singa::io::OpenStore("kvfile", output, singa::io::kCreate); char label; char* pixels = new char[rows * cols]; int count = 0; http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/include/singa/io/hdfsfile.h ---------------------------------------------------------------------- diff --git a/include/singa/io/hdfsfile.h b/include/singa/io/hdfsfile.h index f92910e..cd3ded3 100644 --- a/include/singa/io/hdfsfile.h +++ b/include/singa/io/hdfsfile.h @@ -33,7 +33,7 @@ #include <google/protobuf/message.h> #endif -#include <hdfs.h> +#include <hdfs/hdfs.h> namespace singa { namespace io { http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/src/io/kvfile_store.cc ---------------------------------------------------------------------- diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc index 79e2a40..e77f49b 100644 --- a/src/io/kvfile_store.cc +++ b/src/io/kvfile_store.cc @@ -56,7 +56,9 @@ void KVFileStore::SeekToFirst() { file_->SeekToFirst(); } -void KVFileStore::Seek(int offset){} +void KVFileStore::Seek(int offset){ + LOG(FATAL) << "Operation not supported."; +} bool KVFileStore::Write(const std::string& key, const std::string& value) { CHECK_NE(mode_, kRead);
