SINGA-97 Add HDFS Store Minor change to Makefile.example to get MNIST HDFS uploader to work. Use HDFS_MNIST_TRAIN and HDFS_MNIST_TEST to specify the destination directories.
Fixed minor bug in mnist/create_data.cc Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/befe5ca5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/befe5ca5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/befe5ca5 Branch: refs/heads/master Commit: befe5ca51c5959fec0f04b4ce06c02a3b9e409e7 Parents: aada365 Author: Anh Dinh <[email protected]> Authored: Sat Jan 2 16:16:55 2016 +0800 Committer: WANG Sheng <[email protected]> Committed: Sat Jan 2 19:58:14 2016 +0800 ---------------------------------------------------------------------- examples/mnist/Makefile.example | 14 ++++++++------ examples/mnist/create_data.cc | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/befe5ca5/examples/mnist/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example index 48d2fd8..8c1c838 100644 --- a/examples/mnist/Makefile.example +++ b/examples/mnist/Makefile.example @@ -23,7 +23,9 @@ libs :=singa glog protobuf .PHONY: all download create -HDFS_DIR := hdfs://node0:9000/examples/cifar10 +HDFS_MNIST_TRAIN := hdfs://node0:9000/examples/mnist/train_data.bin +HDFS_MNIST_TEST := hdfs://node0:9000/examples/mnist/test_data.bin + download: mnist @@ -48,8 +50,8 @@ create: ./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte test_data.bin create_hdfs: - $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \ - -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \ - -Wl,-rpath=../../.libs/ -o create_data.bin - ./create_data.bin cifar-10-batches-bin $(HDFS_DIR) - + $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \ + -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ + -o create_data.bin + ./create_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte $(HDFS_MNIST_TRAIN) + ./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte $(HDFS_MNIST_TEST) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/befe5ca5/examples/mnist/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc index 59da860..ff166b4 100644 --- a/examples/mnist/create_data.cc +++ b/examples/mnist/create_data.cc @@ -79,8 +79,8 @@ void create_data(const char* image_filename, const char* label_filename, cols = swap_endian(cols); // read backend from the job.conf - string store_backend = (output_folder.find("hdfs")!=-1) ? "hdfsfile" : "kvfile"; - auto store = singa::io::CreateStore(store_backend); + string store_backend = (string(output).find("hdfs")!=-1) ? "hdfsfile" : "kvfile"; + auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate); char label; char* pixels = new char[rows * cols]; int count = 0;
