http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rbm/autoencoder.conf ---------------------------------------------------------------------- diff --git a/examples/rbm/autoencoder.conf b/examples/rbm/autoencoder.conf deleted file mode 100644 index 223ad0d..0000000 --- a/examples/rbm/autoencoder.conf +++ /dev/null @@ -1,229 +0,0 @@ -name: "auto-encoder" -train_steps: 12200 -test_steps:100 -test_freq:1000 -disp_freq:100 -checkpoint_path: "examples/rbm/rbm4/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm3/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0" -train_one_batch{ - alg: kBP -} -updater{ - type: kAdaGrad - learning_rate{ - base_lr: 0.01 - type: kFixed - } -} - -neuralnet { - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/train_data.bin" - batchsize: 100 - std_value: 255 - shape: 784 - } - include: kTrain - } - - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/test_data.bin" - std_value: 255 - batchsize: 100 - shape: 784 - } - include: kTest - } - - - layer{ - name: "Inner1" - type: kInnerProduct - srclayers:"data" - innerproduct_conf{ - num_output: 1000 - } - param{ - name: "w1" - } - param{ - name: "b12" - } - } - - layer{ - name: "Sigmoid1" - type: kSigmoid - srclayers:"Inner1" - } - layer{ - name: "Inner2" - type: kInnerProduct - srclayers:"Sigmoid1" - innerproduct_conf{ - num_output: 500 - } - param{ - name: "w2" - } - param{ - name: "b22" - } - } - - layer{ - name: "Sigmoid2" - type: kSigmoid - srclayers:"Inner2" - } - - layer{ - name: "Inner3" - type: kInnerProduct - srclayers:"Sigmoid2" - innerproduct_conf{ - num_output: 250 - } - param{ - name: "w3" - } - param{ - name: "b32" - } - } - - layer{ - name: "Sigmoid3" - type: kSigmoid - srclayers:"Inner3" - } - - layer{ - name: "Inner4" - type: kInnerProduct - srclayers:"Sigmoid3" - innerproduct_conf{ - num_output: 30 - } - param{ - name: "w4" - } - param{ - name: "b42" - - } - } - - layer{ - name: "Inner5" - type: kInnerProduct - #srclayers:"Sigmoid4" - srclayers:"Inner4" - innerproduct_conf{ - num_output: 250 - transpose: true - } - param{ - name: "w5" - share_from: "w4" - } - param{ - name: "b41" - } - } - - layer{ - name: "Sigmoid5" - type: kSigmoid - srclayers:"Inner5" - } - layer{ - name: "Inner6" - type: kInnerProduct - srclayers:"Sigmoid5" - innerproduct_conf{ - num_output: 500 - transpose: true - } - param{ - name: "w6" - share_from: "w3" - } - param{ - name: "b31" - } - } - - layer{ - name: "Sigmoid6" - type: kSigmoid - srclayers:"Inner6" - } - layer{ - name: "Inner7" - type: kInnerProduct - srclayers:"Sigmoid6" - innerproduct_conf{ - num_output: 1000 - transpose: true - } - param{ - name: "w7" - share_from: "w2" - } - param{ - name: "b21" - } - - } - - layer{ - name: "Sigmoid7" - type: kSigmoid - srclayers:"Inner7" - } - layer{ - name: "Inner8" - type: kInnerProduct - srclayers:"Sigmoid7" - innerproduct_conf{ - num_output: 784 - transpose: true - } - param{ - name: "w8" - share_from: "w1" - } - param{ - name: "b11" - } - } - - layer{ - name: "Sigmoid8" - type: kSigmoid - srclayers:"Inner8" - } - - layer{ - name: "loss" - type:kEuclideanLoss - srclayers:"Sigmoid8" - srclayers:"data" - } -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - workspace: "examples/rbm/autoencoder/" -}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rbm/rbm1.conf ---------------------------------------------------------------------- diff --git a/examples/rbm/rbm1.conf b/examples/rbm/rbm1.conf deleted file mode 100644 index 696a8cb..0000000 --- a/examples/rbm/rbm1.conf +++ /dev/null @@ -1,101 +0,0 @@ -name: "rbm1" -train_steps: 6000 -test_steps:100 -test_freq:500 -disp_freq: 100 -train_one_batch{ - alg: kCD -} -updater{ - type: kSGD - momentum: 0.8 - weight_decay: 0.0002 - learning_rate{ - base_lr: 0.1 - type: kFixed - } -} - -neuralnet { - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/train_data.bin" - batchsize: 100 - std_value: 255 - shape: 784 - } - include: kTrain - } - - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/test_data.bin" - std_value: 255 - batchsize: 100 - shape: 784 - } - include: kTest - } - -layer{ - name: "RBMVis" - type: kRBMVis - srclayers:"data" - srclayers:"RBMHid" - rbm_conf{ - hdim: 1000 - } - param{ - name: "w1" - init{ - type: kGaussian - mean: 0.0 - std: 0.1 - } - } - - param{ - name: "b11" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} - -layer{ - name: "RBMHid" - type: kRBMHid - srclayers:"RBMVis" - rbm_conf{ - hdim: 1000 - } - param{ - name: "w1_" - share_from: "w1" - } - - param{ - name: "b12" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - nservers_per_group: 1 - nworkers_per_group: 1 - workspace: "examples/rbm/rbm1/" -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rbm/rbm2.conf ---------------------------------------------------------------------- diff --git a/examples/rbm/rbm2.conf b/examples/rbm/rbm2.conf deleted file mode 100644 index ddb9681..0000000 --- a/examples/rbm/rbm2.conf +++ /dev/null @@ -1,122 +0,0 @@ -name: "rbm2" -train_steps: 6000 -test_steps:100 -test_freq:500 -disp_freq: 100 -train_one_batch{ - alg: kCD -} -checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0" -updater{ - type: kSGD - momentum: 0.8 - weight_decay: 0.0002 - learning_rate{ - base_lr: 0.1 - type: kFixed - } -} - -neuralnet { - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/train_data.bin" - batchsize: 100 - std_value: 255 - shape: 784 - } - include: kTrain - } - - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/test_data.bin" - std_value: 255 - batchsize: 100 - shape: 784 - } - include: kTest - } - -layer{ - name: "Inner1" - type: kInnerProduct - srclayers:"data" - innerproduct_conf{ - num_output: 1000 - } - param{ - name: "w1" - } - param{ - name: "b12" - } -} - -layer{ - name: "Sigmoid1" - type: kSigmoid - srclayers:"Inner1" -} - -layer{ - name: "RBMVis" - type: kRBMVis - srclayers:"Sigmoid1" - srclayers:"RBMHid" - rbm_conf{ - hdim: 500 - } - param{ - name: "w2" - init{ - type: kGaussian - mean: 0.0 - std: 0.1 - } - } - - param{ - name: "b21" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} - -layer{ - name: "RBMHid" - type: kRBMHid - srclayers:"RBMVis" - rbm_conf{ - hdim: 500 - } - param{ - name: "w2_" - share_from: "w2" - } - param{ - name: "b22" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - nservers_per_group: 1 - nworkers_per_group: 1 - workspace: "examples/rbm/rbm2/" -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rbm/rbm3.conf ---------------------------------------------------------------------- diff --git a/examples/rbm/rbm3.conf b/examples/rbm/rbm3.conf deleted file mode 100644 index 44eae77..0000000 --- a/examples/rbm/rbm3.conf +++ /dev/null @@ -1,147 +0,0 @@ -name: "rbm3" -train_steps: 6000 -test_steps:100 -test_freq:500 -disp_freq: 100 -train_one_batch{ - alg: kCD -} -checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0" - -updater{ - type: kSGD - momentum: 0.8 - weight_decay: 0.0002 - learning_rate{ - base_lr: 0.1 - type: kFixed - } -} - - -neuralnet { - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/train_data.bin" - batchsize: 100 - std_value: 255 - shape: 784 - } - include: kTrain - } - - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/test_data.bin" - std_value: 255 - batchsize: 100 - shape: 784 - } - include: kTest - } - - -layer{ - name: "Inner1" - type: kInnerProduct - srclayers:"data" - innerproduct_conf{ - num_output: 1000 - } - param{ - name: "w1" - } - param{ - name: "b12" - } - } - - layer{ - name: "Sigmoid1" - type: kSigmoid - srclayers:"Inner1" - } - -layer{ - name: "Inner2" - type: kInnerProduct - srclayers:"Sigmoid1" - innerproduct_conf{ - num_output: 500 - } - param{ - name: "w2" - } - param{ - name: "b22" - } - } - - layer{ - name: "Sigmoid2" - type: kSigmoid - srclayers:"Inner2" - } -layer{ - name: "RBMVis" - type: kRBMVis - srclayers:"Sigmoid2" - srclayers:"RBMHid" - rbm_conf{ - hdim: 250 - } - param{ - name: "w3" - init{ - type: kGaussian - mean: 0.0 - std: 0.1 - } - } - - param{ - name: "b31" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} - -layer{ - name: "RBMHid" - type: kRBMHid - srclayers:"RBMVis" - rbm_conf{ - hdim: 250 - } - param{ - name: "w3_" - share_from: "w3" - } - - param{ - name: "b32" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - nservers_per_group: 1 - nworkers_per_group: 1 - workspace: "examples/rbm/rbm3/" -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rbm/rbm4.conf ---------------------------------------------------------------------- diff --git a/examples/rbm/rbm4.conf b/examples/rbm/rbm4.conf deleted file mode 100644 index bb023c4..0000000 --- a/examples/rbm/rbm4.conf +++ /dev/null @@ -1,167 +0,0 @@ -name: "rbm4" -train_steps: 6000 -test_steps: 100 -test_freq: 500 -disp_freq: 100 -train_one_batch{ - alg: kCD -} -checkpoint_path: "examples/rbm/rbm3/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm2/checkpoint/step6000-worker0" -checkpoint_path: "examples/rbm/rbm1/checkpoint/step6000-worker0" -updater{ - type: kSGD - momentum: 0.8 - weight_decay: 0.0002 - learning_rate{ - base_lr: 0.001 - type: kFixed - } -} - -neuralnet { - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/train_data.bin" - batchsize: 100 - std_value: 255 - shape: 784 - } - include: kTrain - } - - layer { - name: "data" - type: kRecordInput - store_conf { - backend: "kvfile" - path: "examples/mnist/test_data.bin" - std_value: 255 - batchsize: 100 - shape: 784 - } - include: kTest - } - - - layer{ - name: "Inner1" - type: kInnerProduct - srclayers:"data" - innerproduct_conf{ - num_output: 1000 - } - param{ - name: "w1" - } - param{ - name: "b12" - } - } - - layer{ - name: "Sigmoid1" - type: kSigmoid - srclayers:"Inner1" - } - -layer{ - name: "Inner2" - type: kInnerProduct - srclayers:"Sigmoid1" - innerproduct_conf{ - num_output: 500 - } - param{ - name: "w2" - } - param{ - name: "b22" - } - } - - layer{ - name: "Sigmoid2" - type: kSigmoid - srclayers:"Inner2" - } - -layer{ - name: "Inner3" - type: kInnerProduct - srclayers:"Sigmoid2" - innerproduct_conf{ - num_output: 250 - } - param{ - name: "w3" - } - param{ - name: "b32" - } - } - - layer{ - name: "Sigmoid3" - type: kSigmoid - srclayers:"Inner3" - } - -layer{ - name: "RBMVis" - type: kRBMVis - srclayers:"Sigmoid3" - srclayers:"RBMHid" - rbm_conf{ - hdim: 30 - } - param{ - name: "w4" - init{ - type: kGaussian - mean: 0.0 - std: 0.1 - } - } - param{ - name: "b41" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} - -layer{ - name: "RBMHid" - type: kRBMHid - srclayers:"RBMVis" - rbm_conf{ - hdim: 30 - gaussian: true - } - param{ - name: "w4_" - share_from: "w4" - } - param{ - name: "b42" - wd_scale: 0 - init{ - type: kConstant - value: 0.0 - } - } -} -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - nservers_per_group: 1 - nworkers_per_group: 1 - workspace: "examples/rbm/rbm4/" -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/Makefile.example ---------------------------------------------------------------------- diff --git a/examples/rnnlm/Makefile.example b/examples/rnnlm/Makefile.example deleted file mode 100644 index 13c5e42..0000000 --- a/examples/rnnlm/Makefile.example +++ /dev/null @@ -1,52 +0,0 @@ -#/************************************************************ -#* -#* Licensed to the Apache Software Foundation (ASF) under one -#* or more contributor license agreements. See the NOTICE file -#* distributed with this work for additional information -#* regarding copyright ownership. The ASF licenses this file -#* to you under the Apache License, Version 2.0 (the -#* "License"); you may not use this file except in compliance -#* with the License. You may obtain a copy of the License at -#* -#* http://www.apache.org/licenses/LICENSE-2.0 -#* -#* Unless required by applicable law or agreed to in writing, -#* software distributed under the License is distributed on an -#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -#* KIND, either express or implied. See the License for the -#* specific language governing permissions and limitations -#* under the License. -#* -#*************************************************************/ - -MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 - -libs :=singa glog protobuf -filename = rnnlm-0.4b.tgz -# note: filelink for rnnlm-0.4b may change -filelink = https://f25ea9ccb7d3346ce6891573d543960492b92c30.googledrive.com/host/0ByxdPXuxLPS5RFM5dVNvWVhTd0U -dirname = $(patsubst %.tgz,%, $(filename)) -numclass = 100 -dirshards = train_shard valid_shard test_shard - - - -download: - wget $(filelink)/$(filename) - tar zxf $(filename) - rm $(filename) - -create: - protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. rnnlm.proto - $(CXX) create_data.cc rnnlm.pb.cc -std=c++11 -lsinga -lprotobuf -lzookeeper_mt -lglog -I../../include -I../../include/singa/proto \ - -L../../.libs/ -L/usr/local/lib -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \ - -o create_data.bin - for d in $(dirshards); do mkdir -p $${d}; done - ./create_data.bin -train $(dirname)/train -test $(dirname)/test -valid $(dirname)/valid -class_size $(numclass) - - -rnnlm: - protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. rnnlm.proto - $(CXX) main.cc rnnlm.cc rnnlm.pb.cc $(MSHADOW_FLAGS) -msse3 -std=c++11 -lsinga -lglog -lprotobuf -lopenblas -I../../include -I../../include/singa/proto \ - -L../../.libs/ -L/usr/local -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/\ - -o rnnlm.bin http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/README.md ---------------------------------------------------------------------- diff --git a/examples/rnnlm/README.md b/examples/rnnlm/README.md deleted file mode 100644 index 9e83686..0000000 --- a/examples/rnnlm/README.md +++ /dev/null @@ -1,52 +0,0 @@ -This example trains the [RNN model](http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf) proposed by Tomas Mikolov for [language modeling](https://en.wikipedia.org/wiki/Language_model) over text dataset contains 71350 words, provided at [RNNLM Toolkit](https://f25ea9ccb7d3346ce6891573d543960492b92c30.googledrive.com/host/0ByxdPXuxLPS5RFM5dVNvWVhTd0U). -The training objective (loss) is to minimize the [perplexity per word](https://en.wikipedia.org/wiki/Perplexity), which is equivalent to maximize the probability of predicting the next word given the current word in a sentence. -The purpose of this example is to show users how to implement and use their own layers for RNN in SINGA. -The example RNN model consists of six layers, namely RnnDataLayer, WordLayer, RnnLabelLayer, EmbeddingLayer, HiddenLayer, and OutputLayer. - -## File description - -The files in this folder include: - -* rnnlm.proto, definition of the configuration protocol of the layers. -* rnnlm.h, declaration of the layers. -* rnnlm.cc, definition of the layers. -* main.cc, main function that register the layers. -* Makefile.exmaple, Makefile for compiling all source code in this folder. -* job.conf, the job configuration for training the RNN language model. - - -## Data preparation - -To use the RNNLM dataset, we can download it and create DataShard by typing - - # in rnnlm/ folder - cp Makefile.example Makefile - make download - make create - -## Compilation - -The *Makefile.example* contains instructions for compiling the source code. - - # in rnnlm/ folder - cp Makefile.example Makefile - make rnnlm - -It will generate an executable file *rnnlm.bin*. - -## Running - -Make sure that there is one example job configuration file, named *job.conf*. - -Before running SINGA, we need to export the `LD_LIBRARY_PATH` to -include the libsinga.so by the following script. - - # at the root folder of SINGA - export LD_LIBRARY_PATH=.libs:$LD_LIBRARY_PATH - -Then, we can run SINGA as follows. - - # at the root folder of SINGA - ./bin/singa-run.sh -exec examples/rnnlm/rnnlm.bin -conf examples/rnnlm/job.conf - -You will see the values of loss and ppl at each training step. http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/create_data.cc ---------------------------------------------------------------------- diff --git a/examples/rnnlm/create_data.cc b/examples/rnnlm/create_data.cc deleted file mode 100644 index d1edbdb..0000000 --- a/examples/rnnlm/create_data.cc +++ /dev/null @@ -1,444 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ -/* - * This file include code from rnnlmlib-0.4 under BSD new license. - * Copyright (c) 2010-2012 Tomas Mikolov - * Copyright (c) 2013 Cantab Research Ltd - * All rights reserved. - */ - - -// -// This code creates DataShard for RNNLM dataset. -// The RNNLM dataset could be downloaded at -// http://www.rnnlm.org/ -// -// Usage: -// create_shard.bin -train [train_file] -valid [valid_file] -// -test [test_file] -class_size [# of classes] - -#include <cstring> -#include <cstdlib> -#include <cstdio> -#include <cmath> -#include <algorithm> -#include <fstream> - -#include "singa/io/store.h" -#include "singa/utils/common.h" -#include "singa/proto/common.pb.h" -#include "./rnnlm.pb.h" - -#define MAX_STRING 100 -#define BUFFER_LEN 32 -#define NL_STRING "</s>" - -using std::string; -using std::max; -using std::min; - -struct vocab_word { - int cn; - char word[MAX_STRING]; - int class_index; -}; - -struct vocab_word *vocab; -int vocab_max_size; -int vocab_size; -int *vocab_hash; -int vocab_hash_size; -int debug_mode; -int old_classes; -int *class_start; -int *class_end; -int class_size; - -char train_file[MAX_STRING]; -char valid_file[MAX_STRING]; -char test_file[MAX_STRING]; - -int valid_mode; -int test_mode; - -unsigned int getWordHash(char *word) { - unsigned int hash, a; - - hash = 0; - for (a = 0; a < strlen(word); a++) hash = hash * 237 + word[a]; - hash = hash % vocab_hash_size; - - return hash; -} - -int searchVocab(char *word) { - int a; - unsigned int hash; - - hash = getWordHash(word); - - if (vocab_hash[hash] == -1) return -1; - if (!strcmp(word, vocab[vocab_hash[hash]].word)) return vocab_hash[hash]; - - for (a = 0; a < vocab_size; a++) { // search in vocabulary - if (!strcmp(word, vocab[a].word)) { - vocab_hash[hash] = a; - return a; - } - } - - return -1; // return OOV if not found -} - -int addWordToVocab(char *word) { - unsigned int hash; - - snprintf(vocab[vocab_size].word, strlen(word)+1, "%s", word); - vocab[vocab_size].cn = 0; - vocab_size++; - - if (vocab_size + 2 >= vocab_max_size) { // reallocate memory if needed - vocab_max_size += 100; - vocab = (struct vocab_word *) realloc( - vocab, - vocab_max_size * sizeof(struct vocab_word)); - } - - hash = getWordHash(word); - vocab_hash[hash] = vocab_size - 1; - - return vocab_size - 1; -} - -void readWord(char *word, FILE *fin) { - int a = 0, ch; - - while (!feof(fin)) { - ch = fgetc(fin); - - if (ch == 13) continue; - - if ((ch == ' ') || (ch == '\t') || (ch == '\n')) { - if (a > 0) { - if (ch == '\n') ungetc(ch, fin); - break; - } - - if (ch == '\n') { - snprintf(word, strlen(NL_STRING) + 1, - "%s", const_cast<char *>(NL_STRING)); - return; - } else { - continue; - } - } - - word[a] = static_cast<char>(ch); - a++; - - if (a >= MAX_STRING) { - // printf("Too long word found!\n"); //truncate too long words - a--; - } - } - word[a] = 0; -} - -void sortVocab() { - int a, b, max; - vocab_word swap; - - for (a = 1; a < vocab_size; a++) { - max = a; - for (b = a + 1; b < vocab_size; b++) - if (vocab[max].cn < vocab[b].cn) max = b; - - swap = vocab[max]; - vocab[max] = vocab[a]; - vocab[a] = swap; - } -} - -int learnVocabFromTrainFile() { - char word[MAX_STRING]; - FILE *fin; - int a, i, train_wcn; - - for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1; - - fin = fopen(train_file, "rb"); - - vocab_size = 0; - - addWordToVocab(const_cast<char *>(NL_STRING)); - - train_wcn = 0; - while (1) { - readWord(word, fin); - if (feof(fin)) break; - - train_wcn++; - - i = searchVocab(word); - if (i == -1) { - a = addWordToVocab(word); - vocab[a].cn = 1; - } else { - vocab[i].cn++; - } - } - - sortVocab(); - - if (debug_mode > 0) { - printf("Vocab size: %d\n", vocab_size); - printf("Words in train file: %d\n", train_wcn); - } - - fclose(fin); - return 0; -} - -int splitClasses() { - double df, dd; - int i, a, b; - - df = 0; - dd = 0; - a = 0; - b = 0; - - class_start = reinterpret_cast<int *>(calloc(class_size, sizeof(int))); - memset(class_start, 0x7f, sizeof(int) * class_size); - class_end = reinterpret_cast<int *>(calloc(class_size, sizeof(int))); - memset(class_end, 0, sizeof(int) * class_size); - - if (old_classes) { // old classes - for (i = 0; i < vocab_size; i++) - b += vocab[i].cn; - for (i = 0; i < vocab_size; i++) { - df += vocab[i].cn / static_cast<double>(b); - if (df > 1) df = 1; - if (df > (a + 1) / static_cast<double>(class_size)) { - vocab[i].class_index = a; - if (a < class_size - 1) a++; - } else { - vocab[i].class_index = a; - } - } - } else { // new classes - for (i = 0; i < vocab_size; i++) - b += vocab[i].cn; - for (i = 0; i < vocab_size; i++) - dd += sqrt(vocab[i].cn / static_cast<double>(b)); - for (i = 0; i < vocab_size; i++) { - df += sqrt(vocab[i].cn / static_cast<double>(b)) / dd; - if (df > 1) df = 1; - if (df > (a + 1) / static_cast<double>(class_size)) { - vocab[i].class_index = a; - if (a < class_size - 1) a++; - } else { - vocab[i].class_index = a; - } - } - } - - // after dividing classes, update class start and class end information - for (i = 0; i < vocab_size; i++) { - a = vocab[i].class_index; - class_start[a] = min(i, class_start[a]); - class_end[a] = max(i + 1, class_end[a]); - } - return 0; -} - -int init_class() { - // debug_mode = 1; - debug_mode = 0; - vocab_max_size = 100; // largest length value for each word - vocab_size = 0; - vocab = (struct vocab_word *) calloc(vocab_max_size, - sizeof(struct vocab_word)); - vocab_hash_size = 100000000; - vocab_hash = reinterpret_cast<int *>(calloc(vocab_hash_size, sizeof(int))); - old_classes = 1; - - // read vocab - learnVocabFromTrainFile(); - - // split classes - splitClasses(); - - return 0; -} - -int create_data(const char *input_file, const char *output) { - auto* store = singa::io::OpenStore("kvfile", output, singa::io::kCreate); - WordRecord wordRecord; - - FILE *fin; - int a, i; - fin = fopen(input_file, "rb"); - - int wcnt = 0; - char key[BUFFER_LEN]; - char wordstr[MAX_STRING]; - string value; - while (1) { - readWord(wordstr, fin); - if (feof(fin)) break; - i = searchVocab(wordstr); - if (i == -1) { - if (debug_mode) printf("unknown word [%s] detected!", wordstr); - } else { - wordRecord.set_word(string(wordstr)); - wordRecord.set_word_index(i); - int class_idx = vocab[i].class_index; - wordRecord.set_class_index(class_idx); - wordRecord.set_class_start(class_start[class_idx]); - wordRecord.set_class_end(class_end[class_idx]); - int length = snprintf(key, BUFFER_LEN, "%05d", wcnt++); - wordRecord.SerializeToString(&value); - store->Write(string(key, length), value); - } - } - - fclose(fin); - store->Flush(); - delete store; - return 0; -} - -int argPos(char *str, int argc, char **argv) { - int a; - - for (a = 1; a < argc; a++) - if (!strcmp(str, argv[a])) - return a; - - return -1; -} - -int main(int argc, char **argv) { - int i; - FILE *f; - - // set debug mode - i = argPos(const_cast<char *>("-debug"), argc, argv); - if (i > 0) { - debug_mode = 1; - if (debug_mode > 0) - printf("debug mode: %d\n", debug_mode); - } - - // search for train file - i = argPos(const_cast<char *>("-train"), argc, argv); - if (i > 0) { - if (i + 1 == argc) { - printf("ERROR: training data file not specified!\n"); - return 0; - } - - snprintf(train_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]); - - if (debug_mode > 0) - printf("train file: %s\n", train_file); - - f = fopen(train_file, "rb"); - if (f == NULL) { - printf("ERROR: training data file not found!\n"); - return 0; - } - fclose(f); - } else { - printf("ERROR: training data must be set.\n"); - } - - // search for valid file - i = argPos(const_cast<char *>("-valid"), argc, argv); - if (i > 0) { - if (i + 1 == argc) { - printf("ERROR: validating data file not specified!\n"); - return 0; - } - - snprintf(valid_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]); - - if (debug_mode > 0) - printf("valid file: %s\n", valid_file); - - f = fopen(valid_file, "rb"); - if (f == NULL) { - printf("ERROR: validating data file not found!\n"); - return 0; - } - fclose(f); - valid_mode = 1; - } - - // search for test file - i = argPos(const_cast<char *>("-test"), argc, argv); - if (i > 0) { - if (i + 1 == argc) { - printf("ERROR: testing data file not specified!\n"); - return 0; - } - - snprintf(test_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]); - - if (debug_mode > 0) - printf("test file: %s\n", test_file); - - f = fopen(test_file, "rb"); - if (f == NULL) { - printf("ERROR: testing data file not found!\n"); - return 0; - } - fclose(f); - test_mode = 1; - } - - // search for class size - i = argPos(const_cast<char *>("-class_size"), argc, argv); - if (i > 0) { - if (i + 1 == argc) { - printf("ERROR: class size not specified!\n"); - return 0; - } - - class_size = atoi(argv[i + 1]); - - if (debug_mode > 0) - printf("class size: %d\n", class_size); - } - if (class_size <= 0) { - printf("ERROR: no or invalid class size received!\n"); - return 0; - } - - init_class(); - - create_data(train_file, "train_data.bin"); - if (valid_mode) create_data(valid_file, "valid_data.bin"); - if (test_mode) create_data(test_file, "test_data.bin"); - - return 0; -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/job.conf ---------------------------------------------------------------------- diff --git a/examples/rnnlm/job.conf b/examples/rnnlm/job.conf deleted file mode 100644 index aca1166..0000000 --- a/examples/rnnlm/job.conf +++ /dev/null @@ -1,120 +0,0 @@ -name: "rnnlm" -#To scan the training file (81350) 10 times -train_steps:81350 -#To scan the validation file (6828) once -validate_steps:683 -validate_freq:8135 -#disp_freq is specific to training -disp_freq:8135 -train_one_batch { -alg: kBP -} -updater{ - type: kSGD - learning_rate { - type: kFixedStep - fixedstep_conf:{ - step:0 - step:48810 - step:56945 - step:65080 - step:73215 - step_lr:0.1 - step_lr:0.05 - step_lr:0.025 - step_lr:0.0125 - step_lr:0.00625 - } - } -} - -neuralnet { -layer { - name: "data" - user_type: "kData" - [data_conf] { - backend: "kvfile" - path: "examples/rnnlm/train_data.bin" - max_window: 10 - } - include: kTrain -} - -layer { - name: "data" - user_type: "kData" - [data_conf] { - path: "examples/rnnlm/valid_data.bin" - max_window: 10 - } - include: kVal -} - -layer{ - name: "embedding" - user_type: "kEmbedding" - srclayers: "data" - [embedding_conf] { - word_dim: 15 - vocab_size: 3720 - } - param { - name: "w1" - init { - type: kUniform - low:-0.3 - high:0.3 - } - } -} - -layer{ - name: "hidden" - user_type: "kHidden" - srclayers:"embedding" - param{ - name: "w2" - init { - type: kUniform - low:-0.3 - high:0.3 - } - } -} -layer{ - name: "loss" - user_type: "kLoss" - srclayers:"hidden" - srclayers:"data" - [loss_conf] { - nclass:100 - vocab_size: 3720 - } - param{ - name: "w3" - init { - type: kUniform - low:-0.3 - high:0.3 - } - } - param{ - name: "w4" - init { - type: kUniform - low:-0.3 - high:0.3 - } - } -} - -} -cluster { - nworker_groups: 1 - nserver_groups: 1 - nservers_per_group: 1 - nworkers_per_group: 1 - nservers_per_procs: 1 - nworkers_per_procs: 1 - workspace: "examples/rnnlm/" -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/main.cc ---------------------------------------------------------------------- diff --git a/examples/rnnlm/main.cc b/examples/rnnlm/main.cc deleted file mode 100644 index 9124383..0000000 --- a/examples/rnnlm/main.cc +++ /dev/null @@ -1,49 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#include <glog/logging.h> -#include <string> -#include "singa/singa.h" -#include "rnnlm.h" -#include "rnnlm.pb.h" - -int main(int argc, char **argv) { - // initialize glog before creating the driver - google::InitGoogleLogging(argv[0]); - - singa::Driver driver; - driver.Init(argc, argv); - - // if -resume in argument list, set resume to true; otherwise false - int resume_pos = singa::ArgPos(argc, argv, "-resume"); - bool resume = (resume_pos != -1); - - // register all layers for rnnlm - driver.RegisterLayer<rnnlm::EmbeddingLayer, std::string>("kEmbedding"); - driver.RegisterLayer<rnnlm::HiddenLayer, std::string>("kHidden"); - driver.RegisterLayer<rnnlm::LossLayer, std::string>("kLoss"); - driver.RegisterLayer<rnnlm::DataLayer, std::string>("kData"); - - singa::JobProto jobConf = driver.job_conf(); - - driver.Train(resume, jobConf); - return 0; -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/rnnlm.cc ---------------------------------------------------------------------- diff --git a/examples/rnnlm/rnnlm.cc b/examples/rnnlm/rnnlm.cc deleted file mode 100644 index 641b465..0000000 --- a/examples/rnnlm/rnnlm.cc +++ /dev/null @@ -1,335 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ -#include "./rnnlm.h" - -#include <string> -#include <algorithm> -#include "mshadow/tensor.h" -#include "mshadow/tensor_expr.h" -#include "mshadow/cxxnet_op.h" -#include "./rnnlm.pb.h" - -namespace rnnlm { -using std::vector; -using std::string; - -using namespace mshadow; -using mshadow::cpu; -using mshadow::Shape; -using mshadow::Shape1; -using mshadow::Shape2; -using mshadow::Tensor; - -inline Tensor<cpu, 2> RTensor2(Blob<float>* blob) { - const vector<int>& shape = blob->shape(); - Tensor<cpu, 2> tensor(blob->mutable_cpu_data(), - Shape2(shape[0], blob->count() / shape[0])); - return tensor; -} - -inline Tensor<cpu, 1> RTensor1(Blob<float>* blob) { - Tensor<cpu, 1> tensor(blob->mutable_cpu_data(), Shape1(blob->count())); - return tensor; -} - - -/*******DataLayer**************/ -DataLayer::~DataLayer() { - if (store_ != nullptr) - delete store_; -} - -void DataLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) { - RNNLayer::Setup(conf, srclayers); - string key; - max_window_ = conf.GetExtension(data_conf).max_window(); - data_.Reshape(vector<int>{max_window_ + 1, 4}); - window_ = 0; -} - -void SetInst(int k, const WordRecord& word, Blob<float>* to) { - float* dptr = to->mutable_cpu_data() + k * 4; - dptr[0] = static_cast<float>(word.word_index()); - dptr[1] = static_cast<float>(word.class_index()); - dptr[2] = static_cast<float>(word.class_start()); - dptr[3] = static_cast<float>(word.class_end()); -} - -void ShiftInst(int from, int to, Blob<float>* data) { - const float* f = data->cpu_data() + from * 4; - float* t = data->mutable_cpu_data() + to * 4; - // hard code the feature dim to be 4; - t[0] = f[0]; t[1] = f[1]; t[2] = f[2]; t[3] = f[3]; -} - -void DataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - string key, value; - WordRecord word; - if (store_ == nullptr) { - store_ = singa::io::OpenStore( - layer_conf_.GetExtension(data_conf).backend(), - layer_conf_.GetExtension(data_conf).path(), - singa::io::kRead); - store_->Read(&key, &value); - word.ParseFromString(value); - SetInst(0, word, &data_); - } - ShiftInst(window_, 0, &data_); - window_ = max_window_; - for (int i = 1; i <= max_window_; i++) { - if (!store_->Read(&key, &value)) { - store_->SeekToFirst(); - CHECK(store_->Read(&key, &value)); - } - word.ParseFromString(value); - SetInst(i, word, &data_); - if (word.word_index() == 0) { - window_ = i; - break; - } - } -} - -/*******LabelLayer************** -void LabelLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - RNNLayer::Setup(conf, srclayers); - CHECK_EQ(srclayers.size(), 1); - int max_window = dynamic_cast<DataLayer*>(srclayers[0])->max_window(); - data_.Reshape(vector<int>{max_window, 4}); -} - -void LabelLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - const auto& records = dynamic_cast<DataLayer*>(srclayers[0])->records(); - float *label = data_.mutable_cpu_data(); - window_ = dynamic_cast<RNNLayer*>(srclayers[0])->window(); - for (int i = 0; i < window_; i++) { - WordRecord wordrecord = records[i + 1].GetExtension(word); - label[4 * i + 0] = wordrecord.class_start(); - label[4 * i + 1] = wordrecord.class_end(); - label[4 * i + 2] = wordrecord.word_index(); - label[4 * i + 3] = wordrecord.class_index(); - } -} -*/ - -/*******EmbeddingLayer**************/ -EmbeddingLayer::~EmbeddingLayer() { - delete embed_; -} - -void EmbeddingLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - RNNLayer::Setup(conf, srclayers); - CHECK_EQ(srclayers.size(), 1); - int max_window = srclayers[0]->data(this).shape()[0]; - word_dim_ = conf.GetExtension(embedding_conf).word_dim(); - data_.Reshape(vector<int>{max_window, word_dim_}); - grad_.ReshapeLike(data_); - vocab_size_ = conf.GetExtension(embedding_conf).vocab_size(); - embed_ = Param::Create(conf.param(0)); - embed_->Setup(vector<int>{vocab_size_, word_dim_}); -} - -void EmbeddingLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - auto datalayer = dynamic_cast<DataLayer*>(srclayers[0]); - window_ = datalayer->window(); - auto words = RTensor2(&data_); - auto embed = RTensor2(embed_->mutable_data()); - - const float* idxptr = datalayer->data(this).cpu_data(); - for (int t = 0; t < window_; t++) { - int idx = static_cast<int>(idxptr[t * 4]); - CHECK_GE(idx, 0); - CHECK_LT(idx, vocab_size_); - Copy(words[t], embed[idx]); - } -} - -void EmbeddingLayer::ComputeGradient(int flag, - const vector<Layer*>& srclayers) { - auto grad = RTensor2(&grad_); - auto gembed = RTensor2(embed_->mutable_grad()); - auto datalayer = dynamic_cast<DataLayer*>(srclayers[0]); - gembed = 0; - const float* idxptr = datalayer->data(this).cpu_data(); - for (int t = 0; t < window_; t++) { - int idx = static_cast<int>(idxptr[t * 4]); - Copy(gembed[idx], grad[t]); - } -} -/***********HiddenLayer**********/ -HiddenLayer::~HiddenLayer() { - delete weight_; -} - -void HiddenLayer::Setup(const LayerProto& conf, - const vector<Layer*>& srclayers) { - RNNLayer::Setup(conf, srclayers); - CHECK_EQ(srclayers.size(), 1); - const auto& innerproductData = srclayers[0]->data(this); - data_.ReshapeLike(srclayers[0]->data(this)); - grad_.ReshapeLike(srclayers[0]->grad(this)); - int word_dim = data_.shape()[1]; - weight_ = Param::Create(conf.param(0)); - weight_->Setup(std::vector<int>{word_dim, word_dim}); -} - -// hid[t] = sigmoid(hid[t-1] * W + src[t]) -void HiddenLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - window_ = dynamic_cast<RNNLayer*>(srclayers[0])->window(); - auto data = RTensor2(&data_); - auto src = RTensor2(srclayers[0]->mutable_data(this)); - auto weight = RTensor2(weight_->mutable_data()); - for (int t = 0; t < window_; t++) { // Skip the 1st component - if (t == 0) { - data[t] = expr::F<op::sigmoid>(src[t]); - } else { - data[t] = dot(data[t - 1], weight); - data[t] += src[t]; - data[t] = expr::F<op::sigmoid>(data[t]); - } - } -} - -void HiddenLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - auto data = RTensor2(&data_); - auto grad = RTensor2(&grad_); - auto weight = RTensor2(weight_->mutable_data()); - auto gweight = RTensor2(weight_->mutable_grad()); - auto gsrc = RTensor2(srclayers[0]->mutable_grad(this)); - gweight = 0; - TensorContainer<cpu, 1> tmp(Shape1(data_.shape()[1])); - // Check!! - for (int t = window_ - 1; t >= 0; t--) { - if (t < window_ - 1) { - tmp = dot(grad[t + 1], weight.T()); - grad[t] += tmp; - } - grad[t] = expr::F<op::sigmoid_grad>(data[t])* grad[t]; - } - gweight = dot(data.Slice(0, window_-1).T(), grad.Slice(1, window_)); - Copy(gsrc, grad); -} - -/*********** Implementation for LossLayer **********/ -LossLayer::~LossLayer() { - delete word_weight_; - delete class_weight_; -} - -void LossLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) { - RNNLayer::Setup(conf, srclayers); - CHECK_EQ(srclayers.size(), 2); - const auto& src = srclayers[0]->data(this); - int max_window = src.shape()[0]; - int vdim = src.count() / max_window; // Dimension of input - int vocab_size = conf.GetExtension(loss_conf).vocab_size(); - int nclass = conf.GetExtension(loss_conf).nclass(); - word_weight_ = Param::Create(conf.param(0)); - word_weight_->Setup(vector<int>{vocab_size, vdim}); - class_weight_ = Param::Create(conf.param(1)); - class_weight_->Setup(vector<int>{nclass, vdim}); - - pword_.resize(max_window); - pclass_.Reshape(vector<int>{max_window, nclass}); -} - -void LossLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) { - window_ = dynamic_cast<RNNLayer*>(srclayers[0])->window(); - auto pclass = RTensor2(&pclass_); - auto src = RTensor2(srclayers[0]->mutable_data(this)); - auto word_weight = RTensor2(word_weight_->mutable_data()); - auto class_weight = RTensor2(class_weight_->mutable_data()); - const float * label = srclayers[1]->data(this).cpu_data(); - - float loss = 0.f, ppl = 0.f; - for (int t = 0; t < window_; t++) { - // label is the next word - int start = static_cast<int>(label[(t + 1) * 4 + 2]); - int end = static_cast<int>(label[(t + 1) * 4 + 3]); - - auto wordWeight = word_weight.Slice(start, end); - CHECK_GT(end, start); - pword_[t].Reshape(std::vector<int>{end-start}); - auto pword = RTensor1(&pword_[t]); - pword = dot(src[t], wordWeight.T()); - Softmax(pword, pword); - - pclass[t] = dot(src[t], class_weight.T()); - Softmax(pclass[t], pclass[t]); - - int wid = static_cast<int>(label[(t + 1) * 4 + 0]); - int cid = static_cast<int>(label[(t + 1) * 4 + 1]); - CHECK_GT(end, wid); - CHECK_GE(wid, start); - loss_ += -log(std::max(pword[wid - start] * pclass[t][cid], FLT_MIN)); - ppl_ += log10(std::max(pword[wid - start] * pclass[t][cid], FLT_MIN)); - } - num_ += window_; -} - -void LossLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) { - auto pclass = RTensor2(&pclass_); - auto src = RTensor2(srclayers[0]->mutable_data(this)); - auto gsrc = RTensor2(srclayers[0]->mutable_grad(this)); - auto word_weight = RTensor2(word_weight_->mutable_data()); - auto gword_weight = RTensor2(word_weight_->mutable_grad()); - auto class_weight = RTensor2(class_weight_->mutable_data()); - auto gclass_weight = RTensor2(class_weight_->mutable_grad()); - const float * label = srclayers[1]->data(this).cpu_data(); - gclass_weight = 0; - gword_weight = 0; - for (int t = 0; t < window_; t++) { - int start = static_cast<int>(label[(t + 1) * 4 + 2]); - int end = static_cast<int>(label[(t + 1) * 4 + 3]); - int wid = static_cast<int>(label[(t + 1) * 4 + 0]); - int cid = static_cast<int>(label[(t + 1) * 4 + 1]); - auto pword = RTensor1(&pword_[t]); - CHECK_GT(end, wid); - CHECK_GE(wid, start); - - // gL/gclass_act - pclass[t][cid] -= 1.0; - // gL/gword_act - pword[wid - start] -= 1.0; - - // gL/gword_weight - gword_weight.Slice(start, end) += dot(pword.FlatTo2D().T(), - src[t].FlatTo2D()); - // gL/gclass_weight - gclass_weight += dot(pclass[t].FlatTo2D().T(), - src[t].FlatTo2D()); - - gsrc[t] = dot(pword, word_weight.Slice(start, end)); - gsrc[t] += dot(pclass[t], class_weight); - } -} - -const std::string LossLayer::ToString(bool debug, int flag) { - float loss = loss_ / num_; - float ppl = exp10(- ppl_ / num_); - loss_ = 0; - num_ = 0; - ppl_ = 0; - return "loss = " + std::to_string(loss) + ", ppl = " + std::to_string(ppl); -} -} // end of namespace rnnlm http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/rnnlm.h ---------------------------------------------------------------------- diff --git a/examples/rnnlm/rnnlm.h b/examples/rnnlm/rnnlm.h deleted file mode 100644 index 0e415e3..0000000 --- a/examples/rnnlm/rnnlm.h +++ /dev/null @@ -1,158 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -#ifndef EXAMPLES_RNNLM_RNNLM_H_ -#define EXAMPLES_RNNLM_RNNLM_H_ - -#include <string> -#include <vector> -#include "singa/singa.h" -#include "./rnnlm.pb.h" - -namespace rnnlm { -using std::vector; -using singa::LayerProto; -using singa::Layer; -using singa::Param; -using singa::Blob; -using singa::Metric; -/** - * Base RNN layer. May make it a base layer of SINGA. - */ -class RNNLayer : virtual public singa::Layer { - public: - /** - * The recurrent layers may be unrolled different times for different - * iterations, depending on the applications. For example, the ending word - * of a sentence may stop the unrolling; unrolling also stops when the max - * window size is reached. Every layer must reset window_ in its - * ComputeFeature function. - * - * @return the effective BPTT length, which is <= max_window. - */ - inline int window() { return window_; } - - protected: - //!< effect window size for BPTT - int window_; -}; - -/** - * Input layer that get read records from data shard - */ -class DataLayer : public RNNLayer, public singa::InputLayer { - public: - ~DataLayer(); - void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - int max_window() const { - return max_window_; - } - - private: - int max_window_; - singa::io::Store* store_ = nullptr; -}; - - -/** - * LabelLayer that read records_[1] to records_[window_] from DataLayer to - * offer label information -class LabelLayer : public RNNLayer { - public: - void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {} -}; - */ - - -/** - * Word embedding layer that get one row from the embedding matrix for each - * word based on the word index - */ -class EmbeddingLayer : public RNNLayer { - public: - ~EmbeddingLayer(); - void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; - const std::vector<Param*> GetParams() const override { - std::vector<Param*> params{embed_}; - return params; - } - - - private: - int word_dim_; - int vocab_size_; - //!< word embedding matrix of size vocab_size_ x word_dim_ - Param* embed_; -}; - - -/** - * hid[t] = sigmoid(hid[t-1] * W + src[t]) - */ -class HiddenLayer : public RNNLayer { - public: - ~HiddenLayer(); - void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; - - const std::vector<Param*> GetParams() const override { - std::vector<Param*> params{weight_}; - return params; - } - - - private: - Param* weight_; -}; - -/** - * p(word at t+1 is from class c) = softmax(src[t]*Wc)[c] - * p(w|c) = softmax(src[t]*Ww[Start(c):End(c)]) - * p(word at t+1 is w)=p(word at t+1 is from class c)*p(w|c) - */ -class LossLayer : public RNNLayer { - public: - ~LossLayer(); - void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; - void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; - void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; - - const std::string ToString(bool debug, int flag) override; - const std::vector<Param*> GetParams() const override { - std::vector<Param*> params{word_weight_, class_weight_}; - return params; - } - - private: - std::vector<Blob<float>> pword_; - Blob<float> pclass_; - Param* word_weight_, *class_weight_; - float loss_, ppl_; - int num_; -}; -} // namespace rnnlm -#endif // EXAMPLES_RNNLM_RNNLM_H_ http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/examples/rnnlm/rnnlm.proto ---------------------------------------------------------------------- diff --git a/examples/rnnlm/rnnlm.proto b/examples/rnnlm/rnnlm.proto deleted file mode 100644 index 4a4dcbc..0000000 --- a/examples/rnnlm/rnnlm.proto +++ /dev/null @@ -1,53 +0,0 @@ -/************************************************************ -* -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, -* software distributed under the License is distributed on an -* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -* KIND, either express or implied. See the License for the -* specific language governing permissions and limitations -* under the License. -* -*************************************************************/ - -import "job.proto"; -import "common.proto"; - -message EmbeddingProto { - optional int32 word_dim = 1; - optional int32 vocab_size = 2; -} - -message LossProto { - optional int32 nclass = 1; - optional int32 vocab_size = 2; -} - -message DataProto { - required string path = 1; - optional int32 max_window = 2; - optional string backend = 3 [default = "kvfile"]; -} - -extend singa.LayerProto { - optional EmbeddingProto embedding_conf = 1001; - optional LossProto loss_conf = 1002; - optional DataProto data_conf = 1003; -} - -message WordRecord { - optional string word = 1; - optional int32 word_index = 2; - optional int32 class_index = 3; - optional int32 class_start = 4; - optional int32 class_end = 5; -}
