SINGA-97 Add HDFS Store
Modify compilation files. Now as a user, one can build SINGA with hdfs support
by running:
./configure --enable-hdfs --with-libhdfs=/PATH/TO/HDFS3
--with-libhdfs is optional as by default the path is /usr/local/.wq
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/9fbc8ee7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/9fbc8ee7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/9fbc8ee7
Branch: refs/heads/master
Commit: 9fbc8ee7aabbbdc2f76cdcccdf346e14d4544f1a
Parents: 374f11d
Author: xiezl <[email protected]>
Authored: Thu Dec 31 15:25:01 2015 +0800
Committer: WANG Sheng <[email protected]>
Committed: Sat Jan 2 19:58:14 2016 +0800
----------------------------------------------------------------------
Makefile.am | 36 ++++++++++++++++++++++++++----------
configure.ac | 50 +++++++++++++++++++++++++++++++++++++++++++-------
2 files changed, 69 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9fbc8ee7/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index cc308f7..a4315c8 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -29,11 +29,11 @@ MSHADOW_FLAGS = -DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1
-DMSHADOW_USE_MKL=0
DEFAULT_FLAGS = -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
$(MSHADOW_FLAGS) -funroll-loops -DTHREADED
-CFLAGS += $(DEBUG)
-CXXFLAGS += $(DEBUG)
-AC_CXXFLAGS = $(DEBUG)
+CFLAGS = $(DEBUG)
+CXXFLAGS = $(DEBUG)
+#AC_CXXFLAGS = $(DEBUG)
-INCLUDES = -I$(top_srcdir)/include -I/usr/local/include
+INCLUDES = -I$(top_srcdir)/include
PROTOS := $(top_srcdir)/src/proto/singa.proto \
$(top_srcdir)/src/proto/job.proto \
@@ -62,6 +62,11 @@ CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
PY_SRCS := tool/python/singa/driver_wrap.cxx \
src/driver.cc
+HDFS_SRCS := src/io/hdfsfile.cc \
+ src/io/hdfsfile_store.cc
+HDFS_HDRS := include/singa/io/hdfsfile.h \
+ include/singa/io/hdfsfile_store.h
+
SINGA_SRCS := src/driver.cc \
src/server.cc \
src/worker.cc \
@@ -102,8 +107,6 @@ SINGA_SRCS := src/driver.cc \
src/io/kvfile_store.cc \
src/io/textfile_store.cc \
src/io/store.cc \
- src/io/hdfsfile.cc \
- src/io/hdfsfile_store.cc \
src/utils/cluster.cc \
src/utils/cluster_rt.cc \
src/utils/graph.cc \
@@ -202,6 +205,11 @@ libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
libsinga_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
endif
+if DHDFS
+libsinga_la_SOURCES += $(HDFS_SRCS)
+libsinga_la_CXXFLAGS += $(HDFS_CFLAGS)
+libsinga_la_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
#bin_PROGRAMS = singa
singa_SOURCES = src/main.cc
@@ -213,8 +221,7 @@ singa_LDFLAGS = -lsinga \
-lopenblas \
-lzmq \
-lczmq \
- -lzookeeper_mt \
- -lhdfs3
+ -lzookeeper_mt
if LMDB
singa_LDFLAGS += -llmdb
endif
@@ -231,6 +238,11 @@ singa_CXXFLAGS += $(CUDNN_CFLAGS)
singa_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
endif
+if DHDFS
+singa_SOURCES += $(HDFS_SRCS)
+singa_CXXFLAGS += $(HDFS_CFLAGS)
+singa_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
#bin_PROGRAMS += singatool
singatool_SOURCES = src/utils/tool.cc #$(CUDA_SRCS) $(CUDA_HDRS) $(CUDNN_SRCS)
singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas
\
@@ -238,8 +250,7 @@ singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD
-Wno-unknown-pragmas \
singatool_LDFLAGS = -lsinga \
-lglog \
-lprotobuf \
- -lzookeeper_mt \
- -lhdfs3
+ -lzookeeper_mt
#if DCUDA
#singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS)
@@ -253,6 +264,11 @@ singatool_LDFLAGS = -lsinga \
#singatool_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
#endif
+if DHDFS
+singatool_SOURCES += $(HDFS_SRCS)
+singatool_CXXFLAGS += $(HDFS_CFLAGS)
+singatool_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+endif
#lib_LTLIBRARIES += libgtest.la
libgtest_la_SOURCES = $(GTEST_HDRS) $(GTEST_SRCS)
libgtest_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive
-I$(top_srcdir)/include
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9fbc8ee7/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index bde1d8e..82db55a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -55,7 +55,6 @@ if test "$cuda_prefix" == "yes"; then
fi
fi
-
if test x"$cudaval" = x"yes"; then
AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
if test -x "$cuda_prefix/bin/nvcc"; then
@@ -158,23 +157,60 @@ if test x"$enable_lmdb" = x"yes"; then
AC_DEFINE(LMDB, 1, [Enable Option layer])
fi
+PROGS=''
+LTLIBS=''
+AC_ARG_ENABLE(hdfs,
+ AS_HELP_STRING([--enable-hdfs],[enable hdfs support]),
+ [enable_hdfs=yes],[enable_hdfs=no])
+AM_CONDITIONAL(DHDFS, test "$enable_hdfs" = yes)
+
+AC_ARG_WITH([libhdfs],
+ [AS_HELP_STRING([--with-libhdfs=PATH], [prefix where libhdfs is
installed])],
+ [hdfs_prefix=$withval], [hdfs_prefix="/usr/local"])
+if test "$hdfs_prefix" == "yes"; then
+ if test "$withval" == "yes"; then
+ cudnn_prefix="/usr/local"
+ fi
+fi
+
+if test x"$enable_hdfs" != x"no"; then
+ HDFS_CFLAGS="-I$hdfs_prefix/include"
+ HDFS_LDFLAGS="-L$hdfs_prefix/lib"
+ HDFS_LIBS="-lhdfs3"
+ LIBS="$LIBS $HDFS_LIBS"
+ LDFLAGS="$LDFLAGS $HDFS_LDFLAGS"
+ DEBUG+=" -DUSE_HDFS"
+ AC_DEFINE(DHDFS,[1],[Defined if HDFS should be used])
+ AC_CHECK_LIB([hdfs3], [main], [], [
+ AC_MSG_ERROR([unable to find hdfs3 library])
+ ])
+else
+ HDFS_CFLAGS=""
+ HDFS_LDFLAGS=""
+ HDFS_LIBS=""
+fi
+
+AC_SUBST(HDFS_CFLAGS)
+AC_SUBST(HDFS_LDFLAGS)
+AC_SUBST(HDFS_LIBS)
+
AC_ARG_ENABLE(test,
AS_HELP_STRING([--enable-test],[enable singa test]),
[enable_test=yes],[enable_test=no])
AM_CONDITIONAL(SINGATEST, test "$enable_test" = yes)
if test x"$enable_test" != x"no"; then
- PROGS='singatest test '
- LTLIBS='libgtest.la '
+ PROGS+='singatest test '
+ LTLIBS+='libgtest.la '
else
- PROGS=''
- LTLIBS=''
+ PROGS+=''
+ LTLIBS+=''
fi
AC_ARG_ENABLE(debug,
AS_HELP_STRING([--enable-debug],[enable debug mode]),
[enable_debug=yes],[enable_debug=no])
-AM_CONDITIONAL(DEBUG, test "$enable_debug" = yes)
-if test x"$enable_debug" != x"no"; then
+AM_CONDITIONAL(SINGADEBUG, [test "$enable_debug" = yes])
+if test x"$enable_debug" == x"yes"; then
DEBUG+=' -g'
else
DEBUG+=' -O2'
