SINGA-120 - Implemented GRU and BPTT: 1) Updated Driver.cc to register GRU; 2) Updated job.proto to include configuration of GRU; 3) Updated configure.ac to fix some compliation errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/473c9858 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/473c9858 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/473c9858 Branch: refs/heads/master Commit: 473c9858aafd4cd15e6f6316deaeca1f1c0dfffc Parents: ddf4e79 Author: Ju Fan <[email protected]> Authored: Fri Jan 1 10:46:51 2016 +0800 Committer: Wei Wang <[email protected]> Committed: Wed Jan 6 01:53:05 2016 +0800 ---------------------------------------------------------------------- Makefile.am | 38 ++++++++++++++++++++------------------ Makefile.gpu | 6 +++--- configure.ac | 4 ++-- src/driver.cc | 1 + src/proto/job.proto | 27 +++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/473c9858/Makefile.am ---------------------------------------------------------------------- diff --git a/Makefile.am b/Makefile.am index 3c282e3..aa88348 100644 --- a/Makefile.am +++ b/Makefile.am @@ -19,7 +19,6 @@ #* #*************************************************************/ - ACLOCAL_AMFLAGS = -I config AUTOMAKE_OPTIONS = foreign subdir-objects @@ -33,7 +32,7 @@ CFLAGS = $(DEBUG) CXXFLAGS = $(DEBUG) #AC_CXXFLAGS = $(DEBUG) -INCLUDES = -I$(top_srcdir)/include +INCLUDES = -I$(top_srcdir)/include PROTOS := $(top_srcdir)/src/proto/singa.proto \ $(top_srcdir)/src/proto/job.proto \ @@ -50,7 +49,7 @@ PROTO_PYS := tool/python/pb2/singa_pb2.py \ CUDA_SRCS := src/utils/math_kernel.cu CUDA_OBJS := src/utils/math_kernel.o -CUDA_HDRS := include/singa/utils/math_kernel.h +CUDA_HDRS := include/singa/utils/math_kernel.h CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \ src/neuralnet/neuron_layer/cudnn_softmax.cc \ @@ -63,9 +62,9 @@ PY_SRCS := tool/python/singa/driver_wrap.cxx \ src/driver.cc HDFS_SRCS := src/io/hdfsfile.cc \ - src/io/hdfsfile_store.cc + src/io/hdfsfile_store.cc HDFS_HDRS := include/singa/io/hdfsfile.h \ - include/singa/io/hdfsfile_store.h + include/singa/io/hdfsfile_store.h SINGA_SRCS := src/driver.cc \ src/server.cc \ @@ -96,6 +95,7 @@ SINGA_SRCS := src/driver.cc \ src/neuralnet/neuron_layer/lrn.cc \ src/neuralnet/neuron_layer/pooling.cc \ src/neuralnet/neuron_layer/rbm.cc \ + src/neuralnet/neuron_layer/gru.cc \ src/neuralnet/neuron_layer/relu.cc \ src/neuralnet/neuron_layer/sigmoid.cc \ src/neuralnet/neuron_layer/softmax.cc \ @@ -170,7 +170,9 @@ TEST_SRCS := include/gtest/gtest_main.cc \ src/test/test_store.cc \ src/test/test_connection_layers.cc \ src/test/test_record_input_layer.cc \ - src/test/test_csv_input_layer.cc + src/test/test_csv_input_layer.cc \ + src/test/test_gru_layer.cc \ + src/test/test_unrolling.cc #EXTRA_PROGRAMS = $(PROGS) EXTRA_PROGRAMS = singatest test @@ -186,7 +188,7 @@ py_LTLIBRARIES = $(PY_PROGS) #lib_LTLIBRARIES = libsinga.la libsinga_la_SOURCES = $(PROTO_SRCS) $(SINGA_SRCS) libsinga_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include -libsinga_la_LDFLAGS = +libsinga_la_LDFLAGS = if LMDB libsinga_la_CXXFLAGS += -DUSE_LMDB endif @@ -194,7 +196,7 @@ if DCUDA libsinga_la_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS) libsinga_la_CXXFLAGS += $(CUDA_CFLAGS) libsinga_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS) -lsingagpu -libsinga_la_LIBADD = libsingagpu.so +libsinga_la_LIBADD = libsingagpu.so endif if DCUDNN @@ -215,11 +217,11 @@ singa_CXXFLAGS = $(DEFAULT_FLAGS) -MMD -I$(top_srcdir)/include singa_LDFLAGS = -lsinga \ -lglog \ -lprotobuf \ - -lrt \ + #-lrt \ -lopenblas \ -lzmq \ -lczmq \ - -lzookeeper_mt + -lzookeeper_mt if LMDB singa_LDFLAGS += -llmdb endif @@ -248,10 +250,10 @@ singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \ singatool_LDFLAGS = -lsinga \ -lglog \ -lprotobuf \ - -lzookeeper_mt + -lzookeeper_mt #if DCUDA -#singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS) +#singatool_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS) #singatool_CXXFLAGS += $(CUDA_CFLAGS) #singatool_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS) #endif @@ -283,7 +285,7 @@ singatest_LDADD = ./libgtest.la singatest_LDFLAGS = -lsinga \ -lglog \ -lprotobuf \ - -lrt \ + #-lrt \ -lopenblas \ -lzmq \ -lczmq \ @@ -293,7 +295,7 @@ if LMDB singatest_LDFLAGS += -llmdb endif -if DCUDA +if DCUDA singatest_SOURCES += $(CUDA_SRCS) $(CUDA_HDRS) singatest_CXXFLAGS += $(CUDA_CFLAGS) singatest_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS) @@ -310,13 +312,13 @@ _driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include _driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir) if DCUDA -_driver_la_CXXFLAGS += $(CUDA_CFLAGS) -_driver_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS) +_driver_la_CXXFLAGS += $(CUDA_CFLAGS) +_driver_la_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS) endif if DCUDNN _driver_la_CXXFLAGS += $(CUDNN_CFLAGS) -_driver_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS) +_driver_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS) endif clean-local: @@ -348,7 +350,7 @@ rat: $(NVCC) $(MSHADOW_FLAGS) --shared -Xcompiler -fPIC $(CUDA_CFLAGS) $(CUDA_LDFLAGS) $(CUDA_LIBS) -I$(top_srcdir)/include -std=c++11 -G -c -o $@ $< libsingagpu.so: $(CUDA_OBJS) - $(NVCC) -o libsingagpu.so -shared -Xcompiler -fPIC $(CUDA_OBJS) $(CUDA_CFLAGS) $(CUDA_LDFLAGS) $(CUDA_LIBS) -I$(top_srcdir)/include -std=c++11 -G + $(NVCC) -o libsingagpu.so -shared -Xcompiler -fPIC $(CUDA_OBJS) $(CUDA_CFLAGS) $(CUDA_LDFLAGS) $(CUDA_LIBS) -I$(top_srcdir)/include -std=c++11 -G install-pyLTLIBRARIES: $(py_LTLIBRARIES) touch tool/python/singa/__init__.py http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/473c9858/Makefile.gpu ---------------------------------------------------------------------- diff --git a/Makefile.gpu b/Makefile.gpu index c35a445..11d6827 100644 --- a/Makefile.gpu +++ b/Makefile.gpu @@ -20,7 +20,7 @@ ###################User Config Varaibles ############################# # third-party library installation folder -HOME_DIR := /media/hd1/home/wangwei/local +HOME_DIR := /home/wangwei/local # must config the cudnn folder if using cudnn CUDNN_DIR := $(HOME_DIR)/cudnn @@ -30,7 +30,7 @@ CUDA_DIR := /usr/local/cuda # Lib folder for system and external libs. You may need to change it. LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(CUDNN_DIR)/lib64 $(CUDA_DIR)/lib64 $(CUDA_DIR)/lib # Header folder for system and external libs. You may need to change it. -INCLUDE_DIRS := ./include $(HOME_DIR)/include $(CUDNN_DIR)/include $(CUDA_DIR)/include +INCLUDE_DIRS := ./include $(HOME_DIR)/include $(CUDNN_DIR)/include $(CUDA_DIR)/include # g++ location, should support c++11, tested with 4.8.1 CXX := g++ CUCXX := nvcc @@ -85,7 +85,7 @@ TEST_CUDA_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cu") TEST_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_CUDA_SRCS:.cu=.o))) -include $(TEST_CUDA_OBJS:%.o=%.P) -SINGA_CUDA_SRCS :=$(shell find src/ -name "*.cu") +SINGA_CUDA_SRCS := $(shell find src/ \( -path "src/test" \) -prune -o \( -name "*.cu" -type f \) -print ) SINGA_CUDA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_CUDA_SRCS:.cu=.o))) -include $(SINGA_CUDA_OBJS:%.o=%.P) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/473c9858/configure.ac ---------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index 82db55a..2377fa9 100644 --- a/configure.ac +++ b/configure.ac @@ -224,10 +224,10 @@ if test x"$enable_python" != x"no"; then AC_CHECK_PROG(PY_CHECK,python,yes) if test x"$PY_CHECK" != x"yes"; then AC_MSG_ERROR([Cannot find command "python". Please intall before make.]) - else + else PYLIBS="-lpython`python -V 2>&1 | awk '{print substr($2,1,3)}'`" fi -fi +fi AC_SUBST(PYLIBS) AC_ARG_WITH([python], http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/473c9858/src/driver.cc ---------------------------------------------------------------------- diff --git a/src/driver.cc b/src/driver.cc index c17ea82..5e0772b 100644 --- a/src/driver.cc +++ b/src/driver.cc @@ -112,6 +112,7 @@ void Driver::Init(int argc, char **argv) { RegisterLayer<SoftmaxLossLayer, int>(kSoftmaxLoss); RegisterLayer<STanhLayer, int>(kSTanh); RegisterLayer<SoftmaxLayer, int>(kSoftmax); + RegisterLayer<GRULayer, int>(kGRU); #ifdef USE_LMDB RegisterLayer<LMDBDataLayer, int>(kLMDBData); http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/473c9858/src/proto/job.proto ---------------------------------------------------------------------- diff --git a/src/proto/job.proto b/src/proto/job.proto index db55987..7cdc287 100644 --- a/src/proto/job.proto +++ b/src/proto/job.proto @@ -195,6 +195,11 @@ message LayerProto { // share data and grad blob with the single src layer, e.g., relu layer can // share blobs from conv layer. It is useful for saving memory space. optional bool share_src_blobs = 22 [default = false]; + + optional int32 unroll_len = 23 [default = 1]; + repeated UnrollConnType unroll_conn_type = 24; + repeated int32 shift = 25; + // overrides the partition dimension for neural net optional int32 partition_dim = 60 [default = -1]; // names of parameters shared from other layers @@ -222,6 +227,7 @@ message LayerProto { optional RBMProto rbm_conf = 209; optional ReLUProto relu_conf = 211; optional SoftmaxProto softmax_conf = 214; + optional GRUProto gru_conf = 215; // configuration for loss layers, id range [300, 400) optional SoftmaxLossProto softmaxloss_conf = 301; @@ -436,6 +442,15 @@ message RBMProto { optional bool gaussian = 3 [default = false]; // use gaussian sampling or not } +// Message that stores parameters used by GRULayer +message GRUProto { + // dimension of hidden state for the layer + required int32 dim_hidden = 1; + // use bias vector or not + optional bool bias_term = 2 [default = true]; +} + + // Message that stores parameters used by InnerProductLayer message InnerProductProto { // number of outputs for the layer @@ -594,6 +609,7 @@ enum LayerType { kSTanh = 212; kSigmoid = 213; kSoftmax = 214; + kGRU = 215; // cudnn v3 kCudnnConv = 250; kCudnnPool = 251; @@ -706,3 +722,14 @@ enum InitMethod { // For user defined init method kUserInit = 101; } + +enum UnrollConnType { + // i-th unrolled layer <- (i - shift)-th src unrolled layer + kUnrollOneToOne = 1; + // i-th unrolled layer <- all src unrolled layers + kUnrollOneToAll = 2; + // i-th unrolled layer <- last unrolled src layer + kUnrollFirstToLast = 3; + // customized connection type defined by src_conn + kUnrollCustomized = 4; +}
