SINGA-81 Add Python Helper - Revise src/driver.cc to skip the case of no conf file - Change import path for datasets in example python codes - Format python codes and Add license message - Update README.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/9ff176c3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/9ff176c3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/9ff176c3 Branch: refs/heads/master Commit: 9ff176c304efddb9169d20f7c213be9c222edf17 Parents: 2a6645c Author: chonho <[email protected]> Authored: Thu Dec 31 15:10:26 2015 +0800 Committer: Wei Wang <[email protected]> Committed: Fri Jan 1 20:13:25 2016 +0800 ---------------------------------------------------------------------- src/driver.cc | 4 +- tool/python/README.md | 138 +-- tool/python/examples/cifar10_cnn.py | 2 +- tool/python/examples/cifar10_cnn_cudnn.py | 2 +- tool/python/examples/cifar10_cnn_parameter.py | 2 +- tool/python/examples/cluster.conf | 6 - tool/python/examples/mnist_ae.py | 3 +- tool/python/examples/mnist_mlp.py | 2 +- tool/python/examples/mnist_mlp_cudnn.py | 34 - tool/python/examples/mnist_mlp_test.py | 2 +- tool/python/examples/mnist_rbm1.py | 2 +- tool/python/examples/mnist_rbm1_parameter.py | 26 - tool/python/examples/mnist_rbm2.py | 2 +- tool/python/examples/mnist_rbm2_parameter.py | 28 - tool/python/examples/mnist_rbm3.py | 2 +- tool/python/examples/mnist_rbm3_parameter.py | 28 - tool/python/examples/mnist_rbm4.py | 2 +- tool/python/examples/mnist_rbm4_parameter.py | 27 - tool/python/singa.py | 32 +- tool/python/singa/initializations.py | 97 +- tool/python/singa/layer.py | 672 +++++++----- tool/python/singa/model.py | 1072 +++++++++++--------- tool/python/singa/parameter.py | 225 ++-- tool/python/singa/utils/message.py | 112 +- tool/python/singa/utils/utility.py | 124 ++- 25 files changed, 1393 insertions(+), 1253 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/src/driver.cc ---------------------------------------------------------------------- diff --git a/src/driver.cc b/src/driver.cc index 67b5c28..c17ea82 100644 --- a/src/driver.cc +++ b/src/driver.cc @@ -60,8 +60,8 @@ void Driver::Init(int argc, char **argv) { SetupLog(singa_conf_.log_dir(), "driver"); // job conf passed by users as "-conf <path>" arg_pos = ArgPos(argc, argv, "-conf"); - CHECK_NE(arg_pos, -1); - ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_); + if (arg_pos != -1) + ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_); // register layers http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/README.md ---------------------------------------------------------------------- diff --git a/tool/python/README.md b/tool/python/README.md index 1133fa4..02e7fd1 100644 --- a/tool/python/README.md +++ b/tool/python/README.md @@ -10,29 +10,29 @@ |-- utils |-- utility.py |-- message.py + |-- examples + |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, mnist_ae.py, etc. |-- datasets |-- cifar10.py |-- mnist.py - |-- rbm.py - |-- ae.py - |-- examples - |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, rnnlm_usermodel.py, etc. ### How to Run ``` -bin/singa-run.sh -exe user_main.py -conf cluster.conf +bin/singa-run.sh -exec user_main.py ``` -The python code, e.g., `user_main.py`, would create the JobProto object and pass it to Driver::Train. -Currently, ./bin/singa-run.sh needs to get the cluster topology, hence we still need to pass a `cluster.conf` to it. -The cluster.conf has the configuration for a JobProto with all other fields empty except the cluster field. +The python code, e.g., user_main.py, would create the JobProto object and pass it to Driver::Train. -Note that `workspace` field in ClusterProto can be set in either (i) cluster.conf or (ii) python code. - -#### Examples +For example, ``` cd SINGA_ROOT -bin/singa-run.sh -exe tool/python/examples/cifar10_cnn.py -conf tool/python/examples/cluster.conf +bin/singa-run.sh -exec tool/python/examples/cifar10_cnn.py +``` + +Note that, in order to use the Python Helper feature, users need to add the following option +``` +./configure --enable-python --with-python=PYTHON_DIR ``` +where PYTHON_DIR has Python.h ### Layer class (inherited) @@ -47,15 +47,6 @@ bin/singa-run.sh -exe tool/python/examples/cifar10_cnn.py -conf tool/python/exam * RBM * Autoencoder -#### for user defined layers (IN PROGRESS) - -The following classes are designed to construct user-defined layers for RNNLM example. - -* Embedding -* RNNLM -* UserLossRNNLM - - ### Model class * Model class has `jobconf` (JobProto) and `layers` (layer list) @@ -92,60 +83,57 @@ fit() and evaluate() return train/test results, a dictionary containing * 'ppl' for ppl * 'se' for squred error -<<<<<<< HEAD -======= +#### To run Singa on GPU + +Users need to set a list of gpu ids to `device` field in fit() or evaluate(). + +For example, +``` +gpu_id = [0] +m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id) +``` + + ### Parameter class Users need to set parameter and initial values. For example, -* Parameter - * lr = (float) // learning rate - * wd = (float) // weight decay - -* Parameter initialization - * init = (string) // one of the types, 'uniform', 'constant', 'gaussian' - * for uniform [default] - * high = (float) - * low = (float) - * for constant - * value = (float) - * for gaussian - * mean = (float) - * std = (float) +* Parameter (fields in Param proto) + * lr = (float) // learning rate multiplier, used to scale the learning rate when updating parameters. + * wd = (float) // weight decay multiplier, used to scale the weight decay when updating parameters. + +* Parameter initialization (fields in ParamGen proto) + * init = (string) // one of the types, 'uniform', 'constant', 'gaussian' + * high = (float) // for 'uniform' + * low = (float) // for 'uniform' + * value = (float) // for 'constant' + * mean = (float) // for 'gaussian' + * std = (float) // for 'gaussian' * Weight (`w_param`) is 'gaussian' with mean=0, std=0.01 at default * Bias (`b_param`) is 'constant' with value=0 at default * How to update the parameter fields - * for updating Weight, put `w_` in front of field name - * for updating Bias, put `b_` in front of field name + * for updating Weight, put `w_` in front of field name + * for updating Bias, put `b_` in front of field name Several ways to set Parameter values ``` -m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...) +parw = Parameter(lr=2, wd=10, init='gaussian', std=0.1) +parb = Parameter(lr=1, wd=0, init='constant', value=0) +m.add(Convolution2D(10, w_param=parw, b_param=parb, ...) ``` ``` -parw = Parameter(lr=2, wd=10, init='constant', value=0) -m.add(Dense(10, w_param=parw, ...) +m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...) ``` ``` -parw = Parameter(init='constant', value=0) -m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...) +parw = Parameter(init='constant', mean=0) +m.add(Dense(10, w_param=parw, w_lr=1, w_wd=1, b_value=1, ...) ``` -#### To run Singa on GPU - -Users need to set a list of gpu ids to `device` field in fit() or evaluate(). -For example, -``` -gpu_id = [0] -m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id) -``` - ->>>>>>> cb1ffb4... SINGA-81 Add Python Helper #### Other classes * Store @@ -311,48 +299,6 @@ store = Store(path='test.bin', batch_size=100, ...) // parameter values a m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added ``` -<<<<<<< HEAD -### Parameter class - -Users need to set parameter and initial values. For example, - -* Parameter - * lr = (float) // learning rate - * wd = (float) // weight decay - -* Parameter initialization - * init = (string) // one of the types, 'uniform', 'constant', 'gaussian' - * for uniform [default] - * high = (float) - * low = (float) - * for constant - * value = (float) - * for gaussian - * mean = (float) - * std = (float) - -* Weight (w_param) is gaussian with mean=0, std=0.01 at default - -* Bias (b_param) is constant with value=0 at default - -* How to update the parameter fields - * for updating Weight, put 'w_' in front of field name - * for updating Bias, put 'b_' in front of field name - -Several ways to set Parameter values -``` -m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...) -``` -``` -parw = Parameter(lr=2, wd=10, init='constant', value=0) -m.add(Dense(10, w_param=parw, ...) -``` -``` -parw = Parameter(init='constant', value=0) -m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...) -``` -======= ->>>>>>> cb1ffb4... SINGA-81 Add Python Helper ### Cases to run singa http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/cifar10_cnn.py b/tool/python/examples/cifar10_cnn.py index 859a9a4..9e8e588 100755 --- a/tool/python/examples/cifar10_cnn.py +++ b/tool/python/examples/cifar10_cnn.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import cifar10 +from examples.datasets import cifar10 X_train, X_test, workspace = cifar10.load_data() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn_cudnn.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py index d4f4b7c..4269f37 100755 --- a/tool/python/examples/cifar10_cnn_cudnn.py +++ b/tool/python/examples/cifar10_cnn_cudnn.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import cifar10 +from examples.datasets import cifar10 X_train, X_test, workspace = cifar10.load_data() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn_parameter.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/cifar10_cnn_parameter.py b/tool/python/examples/cifar10_cnn_parameter.py index 4144fa5..39bdb43 100755 --- a/tool/python/examples/cifar10_cnn_parameter.py +++ b/tool/python/examples/cifar10_cnn_parameter.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import cifar10 +from examples.datasets import cifar10 X_train, X_test, workspace = cifar10.load_data() http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cluster.conf ---------------------------------------------------------------------- diff --git a/tool/python/examples/cluster.conf b/tool/python/examples/cluster.conf deleted file mode 100644 index 16623d8..0000000 --- a/tool/python/examples/cluster.conf +++ /dev/null @@ -1,6 +0,0 @@ -cluster { - nworker_groups: 1 - nserver_groups: 1 - nworkers_per_group: 1 - nworkers_per_procs: 1 -} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_ae.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_ae.py b/tool/python/examples/mnist_ae.py index 0b7e590..4528d9c 100755 --- a/tool/python/examples/mnist_ae.py +++ b/tool/python/examples/mnist_ae.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist # Sample parameter values for Autoencoder example rbmid = 4 @@ -18,7 +18,6 @@ m = Sequential('autoencoder', sys.argv) hid_dim = [1000, 500, 250, 30] m.add(Autoencoder(hid_dim, out_dim=784, activation='sigmoid', param_share=True)) - agd = AdaGrad(lr=0.01) topo = Cluster(workspace) m.compile(loss='mean_squared_error', optimizer=agd, cluster=topo) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_mlp.py b/tool/python/examples/mnist_mlp.py index da5ccce..1bd5bc5 100755 --- a/tool/python/examples/mnist_mlp.py +++ b/tool/python/examples/mnist_mlp.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist # Sample parameter values for Mnist MLP example pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000, http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp_cudnn.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_mlp_cudnn.py b/tool/python/examples/mnist_mlp_cudnn.py deleted file mode 100755 index d418950..0000000 --- a/tool/python/examples/mnist_mlp_cudnn.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -import sys, os -sys.path.append(os.path.join(os.path.dirname(__file__),'..')) -from singa.model import * -from singa.datasets import mnist - -# Sample parameter values for Mnist MLP example -pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000, - 'std_value' : 127.5, 'mean_value' : 127.5} -X_train, X_test, workspace = mnist.load_data(**pvalues) - -m = Sequential('mlp', argv=sys.argv) - -''' Weight and Bias are initialized by - uniform distribution with scale=0.05 at default -''' -m.add(Dense(2500, init='uniform', activation='tanh')) -m.add(Dense(2000, init='uniform', activation='tanh')) -m.add(Dense(1500, init='uniform', activation='tanh')) -m.add(Dense(1000, init='uniform', activation='tanh')) -m.add(Dense(500, init='uniform', activation='tanh')) -m.add(Dense(10, init='uniform', activation='softmax')) - -sgd = SGD(lr=0.001, lr_type='step') -topo = Cluster(workspace) -m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) - -gpu_id = [0] -m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id) -result = m.evaluate(X_test, batch_size=100, test_steps=10) - -#e.g., display result -#for k, v in sorted(result.items(), key=lambda x: x[0]): -# print k, v http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp_test.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_mlp_test.py b/tool/python/examples/mnist_mlp_test.py index 67cf3b3..de5b9c0 100755 --- a/tool/python/examples/mnist_mlp_test.py +++ b/tool/python/examples/mnist_mlp_test.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist # Sample parameter values for Mnist MLP example pvalues = {'batchsize' : 64, 'shape' : 784, http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm1.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm1.py b/tool/python/examples/mnist_rbm1.py index 765be8b..6cf249c 100755 --- a/tool/python/examples/mnist_rbm1.py +++ b/tool/python/examples/mnist_rbm1.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist rbmid = 1 pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm1_parameter.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm1_parameter.py b/tool/python/examples/mnist_rbm1_parameter.py deleted file mode 100755 index 54fe421..0000000 --- a/tool/python/examples/mnist_rbm1_parameter.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -import sys, os -sys.path.append(os.path.join(os.path.dirname(__file__),'..')) -from singa.model import * -from singa.datasets import mnist - -rbmid = 1 -pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} -X_train, X_test, workspace = mnist.load_data( - workspace = 'examples/rbm/rbm1', - nb_rbm = rbmid, - checkpoint_steps = 6000, - **pvalues) - -m = Energy('rbm'+str(rbmid), sys.argv) - -parw = Parameter(init='gaussian', mean=0, std=0.1) -parb = Parameter(wd=0, init='constant', value=0) -m.add(RBM(1000, w_param=parw, b_param=parb)) - -sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8) -topo = Cluster(workspace) -m.compile(optimizer=sgd, cluster=topo) -m.fit(X_train, alg='cd', nb_epoch=6000) -#result = m.evaluate(X_test, test_steps=100, test_freq=500) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm2.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm2.py b/tool/python/examples/mnist_rbm2.py index f4d187f..b5db36e 100755 --- a/tool/python/examples/mnist_rbm2.py +++ b/tool/python/examples/mnist_rbm2.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist rbmid = 2 pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm2_parameter.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm2_parameter.py b/tool/python/examples/mnist_rbm2_parameter.py deleted file mode 100755 index 9837836..0000000 --- a/tool/python/examples/mnist_rbm2_parameter.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -import sys, os -sys.path.append(os.path.join(os.path.dirname(__file__),'..')) -from singa.model import * -from singa.datasets import mnist - -rbmid = 2 -pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} -X_train, X_test, workspace = mnist.load_data( - workspace = 'examples/rbm/rbm2', - nb_rbm = rbmid, - checkpoint_steps = 6000, - **pvalues) - -m = Energy('rbm'+str(rbmid), sys.argv) - -out_dim = [1000, 500] -parw = Parameter(init='gaussian', mean=0, std=0.1) -parb = Parameter(wd=0, init='constant', value=0) -m.add(RBM(out_dim, w_param=parw, b_param=parb)) - - -sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8) -topo = Cluster(workspace) -m.compile(optimizer=sgd, cluster=topo) -m.fit(X_train, alg='cd', nb_epoch=6000) -#result = m.evaluate(X_test, test_steps=100, test_freq=500) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm3.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm3.py b/tool/python/examples/mnist_rbm3.py index 48bbe38..be1b789 100755 --- a/tool/python/examples/mnist_rbm3.py +++ b/tool/python/examples/mnist_rbm3.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist rbmid = 3 pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm3_parameter.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm3_parameter.py b/tool/python/examples/mnist_rbm3_parameter.py deleted file mode 100755 index 6c9a378..0000000 --- a/tool/python/examples/mnist_rbm3_parameter.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -import sys, os -sys.path.append(os.path.join(os.path.dirname(__file__),'..')) -from singa.model import * -from singa.datasets import mnist - -rbmid = 3 -pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} -X_train, X_test, workspace = mnist.load_data( - workspace = 'examples/rbm/rbm3', - nb_rbm = rbmid, - checkpoint_steps = 6000, - **pvalues) - -m = Energy('rbm'+str(rbmid), sys.argv) - -out_dim = [1000, 500, 250] -parw = Parameter(init='gaussian', mean=0, std=0.1) -parb = Parameter(wd=0, init='constant', value=0) -m.add(RBM(out_dim, w_param=parw, b_param=parb)) - - -sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8) -topo = Cluster(workspace) -m.compile(optimizer=sgd, cluster=topo) -m.fit(X_train, alg='cd', nb_epoch=6000) -#result = m.evaluate(X_test, test_steps=100, test_freq=500) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm4.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm4.py b/tool/python/examples/mnist_rbm4.py index f12b739..a4ee995 100755 --- a/tool/python/examples/mnist_rbm4.py +++ b/tool/python/examples/mnist_rbm4.py @@ -2,7 +2,7 @@ import sys, os sys.path.append(os.path.join(os.path.dirname(__file__),'..')) from singa.model import * -from singa.datasets import mnist +from examples.datasets import mnist rbmid = 4 pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm4_parameter.py ---------------------------------------------------------------------- diff --git a/tool/python/examples/mnist_rbm4_parameter.py b/tool/python/examples/mnist_rbm4_parameter.py deleted file mode 100755 index 2a7be1d..0000000 --- a/tool/python/examples/mnist_rbm4_parameter.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python -import sys, os -sys.path.append(os.path.join(os.path.dirname(__file__),'..')) -from singa.model import * -from singa.datasets import mnist - -rbmid = 4 -pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255} -X_train, X_test, workspace = mnist.load_data( - workspace = 'examples/rbm/rbm'+rbmid, - nb_rbm = rbmid, - checkpoint_steps = 6000, - **pvalues) - -m = Energy('rbm'+str(rbmid), sys.argv) - -out_dim = [1000, 500, 250, 30] -parw = Parameter(init='gaussian', mean=0, std=0.1) -parb = Parameter(wd=0, init='constant', value=0) -m.add(RBM(out_dim, w_param=parw, b_param=parb)) - -sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8) -topo = Cluster(workspace) -m.compile(optimizer=sgd, cluster=topo) -m.fit(X_train, alg='cd', nb_epoch=6000) -#result = m.evaluate(X_test, test_steps=100, test_freq=500) - http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa.py ---------------------------------------------------------------------- diff --git a/tool/python/singa.py b/tool/python/singa.py index 986a6b8..e44e94d 100755 --- a/tool/python/singa.py +++ b/tool/python/singa.py @@ -21,8 +21,6 @@ #* #*************************************************************/ - - import os import sys import string @@ -31,18 +29,18 @@ import singa.driver as driver from google.protobuf.text_format import Merge if __name__ == '__main__': - """Invoke the training program using this python script. - ./bin/singa-run.sh -exec tool/python/singa.py -conf examples/cifar10/job.conf - """" - - i = sys.argv.index("-conf") - s = open(sys.argv[i+1], 'r').read() - s = str(s) - j = job_pb2.JobProto() - Merge(s,j) - b = j.SerializeToString() - d = driver.Driver() - d.InitLog(sys.argv[0]) - d.Init(sys.argv) - d.Train(False,b) - #d.Test(b) + """Invoke the training program using this python script. + ./bin/singa-run.sh -exec tool/python/singa.py -conf examples/cifar10/job.conf + """ + + i = sys.argv.index('-conf') + s = open(sys.argv[i+1], 'r').read() + s = str(s) + j = job_pb2.JobProto() + Merge(s, j) + b = j.SerializeToString() + d = driver.Driver() + d.InitLog(sys.argv[0]) + d.Init(sys.argv) + d.Train(False, b) + #d.Test(b) http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/initializations.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/initializations.py b/tool/python/singa/initializations.py index f3037cd..f016f1f 100644 --- a/tool/python/singa/initializations.py +++ b/tool/python/singa/initializations.py @@ -1,34 +1,67 @@ #!/usr/bin/env python -def get(identifier, **kwargs): - - field = {} - - if identifier == 'none': - return - - if identifier == 'uniform': - scale = kwargs['scale'] if 'scale' in kwargs else 0.05 - names = ['low', 'high'] - values = [-scale, scale] - - elif identifier == 'constant': - names = ['value'] - values = [0] - - elif identifier == 'gaussian': - names = ['mean', 'std'] - values = [0, 0.01] - - elif identifier == 'conv2d': - names = ['stride', 'pad'] - values = [1, 0] - - elif identifier == 'lrn2d': - names = ['alpha', 'beta', 'knorm'] - values = [1, 0.75, 1] - - for i in range(len(names)): - field[names[i]] = kwargs[names[i]] if names[i] in kwargs else values[i] - - return field +#/************************************************************ +#* +#* Licensed to the Apache Software Foundation (ASF) under one +#* or more contributor license agreements. See the NOTICE file +#* distributed with this work for additional information +#* regarding copyright ownership. The ASF licenses this file +#* to you under the Apache License, Version 2.0 (the +#* "License"); you may not use this file except in compliance +#* with the License. You may obtain a copy of the License at +#* +#* http://www.apache.org/licenses/LICENSE-2.0 +#* +#* Unless required by applicable law or agreed to in writing, +#* software distributed under the License is distributed on an +#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +#* KIND, either express or implied. See the License for the +#* specific language governing permissions and limitations +#* under the License. +#* +#*************************************************************/ + +''' +This module pre-defines initial value for fields +''' + +def get_init_values(identifier, **kwargs): + ''' + This method returns field, a set of key-value pairs, that + key is specified by identifier and values are initialized. + ''' + + field = {} + + if identifier == 'none': + return + + if identifier == 'uniform': + scale = kwargs['scale'] if 'scale' in kwargs else 0.05 + names = ['low', 'high'] + values = [-scale, scale] + + elif identifier == 'constant': + names = ['value'] + values = [0] + + elif identifier == 'gaussian': + names = ['mean', 'std'] + values = [0, 0.01] + + elif identifier == 'conv2d': + names = ['stride', 'pad'] + values = [1, 0] + + elif identifier == 'lrn2d': + names = ['alpha', 'beta', 'knorm'] + values = [1, 0.75, 1] + + elif identifier == 'dropout': + names = ['ratio'] + values = [0.5] + + for i in range(len(names)): + field[names[i]] = kwargs[names[i]] if names[i] in kwargs else values[i] + + return field http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/layer.py ---------------------------------------------------------------------- diff --git a/tool/python/singa/layer.py b/tool/python/singa/layer.py index 1744d50..b391d26 100644 --- a/tool/python/singa/layer.py +++ b/tool/python/singa/layer.py @@ -1,300 +1,440 @@ #!/usr/bin/env python -from parameter import * -from utils.utility import * -from utils.message import * + +#/************************************************************ +#* +#* Licensed to the Apache Software Foundation (ASF) under one +#* or more contributor license agreements. See the NOTICE file +#* distributed with this work for additional information +#* regarding copyright ownership. The ASF licenses this file +#* to you under the Apache License, Version 2.0 (the +#* "License"); you may not use this file except in compliance +#* with the License. You may obtain a copy of the License at +#* +#* http://www.apache.org/licenses/LICENSE-2.0 +#* +#* Unless required by applicable law or agreed to in writing, +#* software distributed under the License is distributed on an +#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +#* KIND, either express or implied. See the License for the +#* specific language governing permissions and limitations +#* under the License. +#* +#*************************************************************/ + +''' +This script includes Layer class and its subclasses that +users can configure different types of layers for their model. +''' + +from singa.parameter import Parameter, set_param_field +from singa.initializations import get_init_values +from singa.utils.utility import setval, generate_name +from singa.utils.message import * from google.protobuf import text_format class Layer(object): - def __init__(self, **kwargs): - self.layer = Message('Layer', **kwargs).proto - # required - if not 'name' in kwargs: - setval(self.layer, name=generateName('layer', 1)) - # srclayers are set in Model.build() - self.is_datalayer = False + def __init__(self, **kwargs): + ''' + **kwargs (KEY=VALUE) + partition_dim = (int) // partition dimension for net + ''' + + self.layer = Message('Layer', **kwargs).proto + # required + if not 'name' in kwargs: + setval(self.layer, name=generate_name('layer', 1)) + + # srclayers are set in Model.build() + self.is_datalayer = False class Data(Layer): - def __init__(self, load, phase='train', checkpoint=None, - conf=None, **kwargs): - assert load != None, 'data type should be specified' - if load == 'kData': - super(Data, self).__init__(name=generateName('data'), user_type=load) - else: - self.layer_type = enumLayerType(load) - super(Data, self).__init__(name=generateName('data'), type=self.layer_type) - - # include/exclude - setval(self.layer, include=enumPhase(phase)) - #setval(self.layer, exclude=kTest if phase=='train' else kTrain) - - if conf == None: - if load == 'kData': - setval(self.layer.Extensions[data_conf], **kwargs) - else: - setval(self.layer.store_conf, **kwargs) - else: - setval(self.layer, store_conf=conf.proto) - self.is_datalayer = True - - self.checkpoint = checkpoint # checkpoint for training data + + def __init__(self, load, phase='train', checkpoint=None, + conf=None, **kwargs): + ''' + required + load = (string) // type of data + optional + phase = (string) // phase of data layer + checkpoint = (string) // checkpoint path + conf = (Store) // Store object + **kwargs (KEY=VALUE) + partition_dim = (int) // partition dimension for net + ''' + + assert load != None, 'data type should be specified' + if load == 'kData': + super(Data, self).__init__(name=generate_name('data'), + user_type=load) + else: + self.layer_type = enumLayerType(load) + super(Data, self).__init__(name=generate_name('data'), + type=self.layer_type) + self.is_datalayer = True + + # include/exclude + setval(self.layer, include=enumPhase(phase)) + #setval(self.layer, exclude=kTest if phase=='train' else kTrain) + + if conf == None: + if load == 'kData': + setval(self.layer.Extensions[data_conf], **kwargs) + else: + setval(self.layer.store_conf, **kwargs) + else: + setval(self.layer, store_conf=conf.proto) + + self.checkpoint = checkpoint # checkpoint for training data class Convolution2D(Layer): - def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0, - init=None, w_param=None, b_param=None, - activation=None, **kwargs): - ''' - required - nb_filter = (int) // the number of filters - kernel = (int) // the size of filter - optional - stride = (int) // the size of stride - pad = (int) // the size of padding - ''' - assert nb_filter > 0 and kernel > 0, 'should be set as positive int' - super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution) - fields = {'num_filters' : nb_filter, - 'kernel' : kernel, - 'stride' : stride, - 'pad' : pad} - setval(self.layer.convolution_conf, **fields) - - # parameter w - if w_param == None: - self.init = 'gaussian' if init==None else init - w_param = Parameter(init=self.init) - setParamField(w_param.param, 'w', True, **kwargs) - setval(self.layer, param=w_param.param) - - # parameter b - if b_param == None: - self.init = 'constant' if init==None else init - b_param = Parameter(init=self.init) # default: constant - setParamField(b_param.param, 'b', True, **kwargs) - setval(self.layer, param=b_param.param) - - # following layers: e.g., activation, dropout, etc. - if activation: - self.mask = Activation(activation=activation).layer + + def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0, + init=None, w_param=None, b_param=None, + activation=None, **kwargs): + ''' + required + nb_filter = (int) // the number of filters + kernel = (int) // the size of filter + optional + stride = (int) // the size of stride + pad = (int) // the size of padding + init = (string) // 'unirom', 'gaussian', 'constant' + w_param = (Parameter) // Parameter object for weight + b_param = (Parameter) // Parameter object for bias + **kwargs (KEY=VALUE) + w_lr = (float) // learning rate multiplier for weight, used to + // scale the learning rate when updating parameters. + w_wd = (float) // weight decay multiplier for weight, used to + // scale the weight decay when updating parameters. + b_lr = (float) // learning rate multiplier for bias + b_wd = (float) // weight decay multiplier for bias + ''' + + assert nb_filter > 0 and kernel > 0, 'should be set as positive int' + super(Convolution2D, self).__init__(name=generate_name('conv', 1), + type=kCConvolution) + fields = {'num_filters' : nb_filter, + 'kernel' : kernel, + 'stride' : stride, + 'pad' : pad} + setval(self.layer.convolution_conf, **fields) + + # parameter w + if w_param == None: + self.init = 'gaussian' if init == None else init + w_param = Parameter(init=self.init) + set_param_field(w_param.param, 'w', True, **kwargs) + setval(self.layer, param=w_param.param) + + # parameter b + if b_param == None: + self.init = 'constant' if init == None else init + b_param = Parameter(init=self.init) # default: constant + set_param_field(b_param.param, 'b', True, **kwargs) + setval(self.layer, param=b_param.param) + + # following layers: e.g., activation, dropout, etc. + if activation: + self.mask = Activation(activation=activation).layer class MaxPooling2D(Layer): - def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): - ''' - required - pool_size = (int|tuple) // the size for pooling - optional - stride = (int) // the size of striding - ignore_border = (bool) // flag for padding - **kwargs // fields for Layer class - ''' - assert pool_size != None, 'pool_size is required' - if type(pool_size) == int: - pool_size = (pool_size, pool_size) - assert type(pool_size) == tuple and \ - pool_size[0] == pool_size[1], 'pool size should be square in Singa' - super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) - fields = {'pool' : PoolingProto().MAX, - 'kernel' : pool_size[0], - 'stride' : stride, - 'pad' : 0 if ignore_border else 1} - setval(self.layer.pooling_conf, **fields) + + def __init__(self, pool_size=None, + stride=1, ignore_border=True, **kwargs): + ''' + Max Pooling layer + + required + pool_size = (int|tuple) // the size for pooling + optional + stride = (int) // the size of striding + ignore_border = (bool) // flag for padding + **kwargs // fields for Layer class + ''' + + assert pool_size != None, 'pool_size is required' + if type(pool_size) == int: + pool_size = (pool_size, pool_size) + assert type(pool_size) == tuple and pool_size[0] == pool_size[1], \ + 'pool size should be square in Singa' + super(MaxPooling2D, self).__init__(name=generate_name('pool'), + type=kCPooling, **kwargs) + fields = {'pool' : PoolingProto().MAX, + 'kernel' : pool_size[0], + 'stride' : stride, + 'pad' : 0 if ignore_border else 1} + setval(self.layer.pooling_conf, **fields) class AvgPooling2D(Layer): - def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs): - ''' - required - pool_size = (int|tuple) // size for pooling - optional - stride = (int) // size of striding - ignore_border = (bool) // flag for padding - **kwargs // fields for Layer class - ''' - assert pool_size != None, 'pool_size is required' - if type(pool_size) == int: - pool_size = (pool_size, pool_size) - assert type(pool_size) == tuple and \ - pool_size[0] == pool_size[1], 'pool size should be square in Singa' - super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs) - self.layer.pooling_conf.pool = PoolingProto().AVG - fields = {'pool' : PoolingProto().AVG, - 'kernel' : pool_size[0], - 'stride' : stride, - 'pad' : 0 if ignore_border else 1} - setval(self.layer.pooling_conf, **fields) + + def __init__(self, pool_size=None, + stride=1, ignore_border=True, **kwargs): + ''' + required + pool_size = (int|tuple) // size for pooling + optional + stride = (int) // size of striding + ignore_border = (bool) // flag for padding + **kwargs // fields for Layer class + ''' + + assert pool_size != None, 'pool_size is required' + if type(pool_size) == int: + pool_size = (pool_size, pool_size) + assert type(pool_size) == tuple and pool_size[0] == pool_size[1], \ + 'pool size should be square in Singa' + super(AvgPooling2D, self).__init__(name=generate_name('pool'), + type=kCPooling, **kwargs) + self.layer.pooling_conf.pool = PoolingProto().AVG + fields = {'pool' : PoolingProto().AVG, + 'kernel' : pool_size[0], + 'stride' : stride, + 'pad' : 0 if ignore_border else 1} + setval(self.layer.pooling_conf, **fields) class LRN2D(Layer): - def __init__(self, size=0, **kwargs): - super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN) - # required - assert size != 0, 'local size should be set' - self.layer.lrn_conf.local_size = size - init_value = initializations.get('lrn2d', **kwargs) - setval(self.layer.lrn_conf, **init_value) + + def __init__(self, size=0, **kwargs): + ''' + required + size = (int) // local size + ''' + + super(LRN2D, self).__init__(name=generate_name('norm'), type=kLRN) + # required + assert size != 0, 'local size should be set' + self.layer.lrn_conf.local_size = size + init_values = get_init_values('lrn2d', **kwargs) + setval(self.layer.lrn_conf, **init_values) class Activation(Layer): - def __init__(self, activation='stanh', topk=1): - self.name = activation - if activation == 'tanh': activation = 'stanh' # <-- better way to set? - self.layer_type = enumLayerType(activation) - super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type) - if activation == 'softmaxloss': - self.layer.softmaxloss_conf.topk = topk - -class Dropout(Layer): - def __init__(self, ratio=0.5): - self.name = 'dropout' - self.layer_type = kDropout - super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type) - self.layer.dropout_conf.dropout_ratio = ratio + + def __init__(self, activation='stanh', topk=1): + ''' + required + activation = (string) + optional + topk = (int) // the number of results + ''' + + self.name = activation + if activation == 'tanh': activation = 'stanh' # <-- better way to set? + self.layer_type = enumLayerType(activation) + super(Activation, self).__init__(name=generate_name(self.name), + type=self.layer_type) + if activation == 'softmaxloss': + self.layer.softmaxloss_conf.topk = topk + +class Dropout(Layer): + + def __init__(self, ratio=0.5): + ''' + required + ratio = (float) // ratio of drop out nodes + ''' + + self.name = 'dropout' + self.layer_type = enumLayerType(self.name) + super(Dropout, self).__init__(name=generate_name(self.name), + type=self.layer_type) + self.layer.dropout_conf.dropout_ratio = ratio class RGB(Layer): - def __init__(self, meanfile=None, **kwargs): - assert meanfile != None, 'meanfile should be specified' - self.name = 'rgb' - self.layer_type = kRGBImage - super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type) - self.layer.rgbimage_conf.meanfile = meanfile + + def __init__(self, meanfile=None, **kwargs): + ''' + required + meanfile = (string) // path to meanfile (depreciated) + ''' + + assert meanfile != None, 'meanfile should be specified' + self.name = 'rgb' + self.layer_type = kRGBImage + super(RGB, self).__init__(name=generate_name(self.name), + type=self.layer_type) + self.layer.rgbimage_conf.meanfile = meanfile class Dense(Layer): - def __init__(self, output_dim=0, activation=None, - init=None, w_param=None, b_param=None, input_dim=None, - **kwargs): - ''' - required - output_dim = (int) - optional - activation = (string) - init = (string) // 'unirom', 'gaussian', 'constant' - **kwargs - w_lr = (float) // learning rate for w - w_wd = (float) // weight decay for w - b_lr = (float) // learning rate for b - b_wd = (float) // weight decay for b - ''' - # required - assert output_dim > 0, 'output_dim should be set' - super(Dense, self).__init__(type=kInnerProduct, **kwargs) - self.layer.innerproduct_conf.num_output = output_dim - if 'transpose' in kwargs: - self.layer.innerproduct_conf.transpose = kwargs['transpose'] - - # parameter w (default: gaussian) - if w_param == None: - self.init = 'gaussian' if init==None else init - w_param = Parameter(init=self.init) - setParamField(w_param.param, 'w', False, **kwargs) - setval(self.layer, param=w_param.param) - - # parameter b (default: constant) - if b_param == None: - self.init = 'constant' if init==None else init - b_param = Parameter(init=self.init) - setParamField(b_param.param, 'b', False, **kwargs) - setval(self.layer, param=b_param.param) - - # following layers: e.g., activation, dropout, etc. - if activation: - self.mask = Activation(activation=activation).layer - - -''' Class to deal with multiple layers + + def __init__(self, output_dim=0, activation=None, + init=None, w_param=None, b_param=None, input_dim=None, + **kwargs): + ''' + required + output_dim = (int) + optional + activation = (string) + init = (string) // 'unirom', 'gaussian', 'constant' + w_param = (Parameter) // Parameter object for weight + b_param = (Parameter) // Parameter object for bias + **kwargs + w_lr = (float) // learning rate multiplier for weight, used to + // scale the learning rate when updating parameters. + w_wd = (float) // weight decay multiplier for weight, used to + // scale the weight decay when updating parameters. + b_lr = (float) // learning rate multiplier for bias + b_wd = (float) // weight decay multiplier for bias + ''' + # required + assert output_dim > 0, 'output_dim should be set' + super(Dense, self).__init__(type=kInnerProduct, **kwargs) + self.layer.innerproduct_conf.num_output = output_dim + if 'transpose' in kwargs: + self.layer.innerproduct_conf.transpose = kwargs['transpose'] + + # parameter w (default: gaussian) + if w_param == None: + self.init = 'gaussian' if init == None else init + w_param = Parameter(init=self.init) + set_param_field(w_param.param, 'w', False, **kwargs) + setval(self.layer, param=w_param.param) + + # parameter b (default: constant) + if b_param == None: + self.init = 'constant' if init == None else init + b_param = Parameter(init=self.init) + set_param_field(b_param.param, 'b', False, **kwargs) + setval(self.layer, param=b_param.param) + + # following layers: e.g., activation, dropout, etc. + if activation: + self.mask = Activation(activation=activation).layer + + +''' Classes to deal with multiple layers ''' class Autoencoder(object): - def __init__(self, hid_dim=None, out_dim=0, activation=None, - param_share=True, **kwargs): - # required - assert out_dim > 0, 'out_dim should be set' - self.out_dim = out_dim - assert hid_dim != None, 'hid_dim should be set' - self.hid_dim = [hid_dim] if type(hid_dim)==int else hid_dim - - self.layer_type = 'AutoEncoder' - self.activation = activation - self.param_share = param_share + + def __init__(self, hid_dim=None, out_dim=0, + activation=None, param_share=True): + ''' + Generate a set of layers (like MLP) for encoder and decoder + The layers are expanded and added in Sequential.add() + + required + hid_dim = (int/list) // the number of nodes in hidden layers + out_dim = (int) // the number of nodes in the top layer + optional + activation = (string) + param_share = (bool) // to share params in encoder and decoder + ''' + + # required + assert out_dim > 0, 'out_dim should be set' + self.out_dim = out_dim + assert hid_dim != None, 'hid_dim should be set' + self.hid_dim = [hid_dim] if type(hid_dim) == int else hid_dim + + self.layer_type = 'AutoEncoder' + self.activation = activation + self.param_share = param_share class RBM(Layer): - def __init__(self, out_dim=None, w_param=None, b_param=None, sampling=None, **kwargs): - ''' - Generate layers (like MLP) according to the number of elements in out_dim, and - on top of it, two layers RBMVis and RBMHid with bidirectional connection - - required - out_dim = (int) or (int list) // the number of hidden nodes - optional - sampling = (string) - ''' - assert out_dim > 0, 'out_dim should be set' - self.out_dim = [out_dim] if type(out_dim)==int else out_dim - - self.name = kwargs['name'] if 'name' in kwargs else 'RBMVis' - self.layer_type = kwargs['type'] if 'type' in kwargs else kRBMVis - super(RBM, self).__init__(name=generateName(self.name, withnumber=False), type=self.layer_type) - setval(self.layer.rbm_conf, hdim=self.out_dim[-1]) - if self.layer_type == kRBMHid and sampling != None: - if sampling == 'gaussian': - setval(self.layer.rbm_conf, gaussian=True) - - # parameter w - if w_param == None: - w_param = Parameter(init='gaussian', **kwargs) - setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs) - else: - if self.layer_type == kRBMHid: - del kwargs['name'] - else: - setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs) - setval(self.layer, param=w_param.param) - - # parameter b - if b_param == None: - b_param = Parameter(init='constant', **kwargs) - setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs) - else: - if self.layer_type == kRBMHid: - pass - else: - setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs) - setval(self.layer, param=b_param.param) - - if self.layer_type == kRBMVis: - wname = w_param.param.name - parw = Parameter(name=wname+"_", init='none', share_from=wname) - bname = b_param.param.name - parb = Parameter(name=bname+"2", wd=0, init='constant') - self.bidirect = RBM(self.out_dim, name='RBMHid', type=kRBMHid, - w_param=parw, b_param=parb, sampling=sampling).layer - - + + def __init__(self, out_dim=None, w_param=None, b_param=None, + sampling=None, **kwargs): + ''' + Generate a set of layers (like MLP) according to the number of elements + in out_dim, and on top of it, two layers RBMVis and RBMHid with + bidirectional connection + The layers are expanded and added in Energy.add() + + required + out_dim = (int) or (int list) // the number of hidden nodes + optional + w_param = (Parameter) // Parameter object for weight + b_param = (Parameter) // Parameter object for bias + sampling = (string) + ''' + + assert out_dim > 0, 'out_dim should be set' + self.out_dim = [out_dim] if type(out_dim) == int else out_dim + + self.name = kwargs['name'] if 'name' in kwargs else 'RBMVis' + self.layer_type = kwargs['type'] if 'type' in kwargs else kRBMVis + super(RBM, self).__init__(name=generate_name(self.name, + withnumber=False), type=self.layer_type) + setval(self.layer.rbm_conf, hdim=self.out_dim[-1]) + if self.layer_type == kRBMHid and sampling != None: + if sampling == 'gaussian': + setval(self.layer.rbm_conf, gaussian=True) + + # parameter w + if w_param == None: + w_param = Parameter(init='gaussian', **kwargs) + set_param_field(w_param.param, 'w', withnumber=False, + level=len(self.out_dim), **kwargs) + else: + if self.layer_type == kRBMHid: + del kwargs['name'] + else: + set_param_field(w_param.param, 'w', withnumber=False, + level=len(self.out_dim), **kwargs) + setval(self.layer, param=w_param.param) + + # parameter b + if b_param == None: + b_param = Parameter(init='constant', **kwargs) + set_param_field(b_param.param, 'b', withnumber=False, + level=len(self.out_dim), **kwargs) + else: + if self.layer_type == kRBMHid: + pass + else: + set_param_field(b_param.param, 'b', withnumber=False, + level=len(self.out_dim), **kwargs) + setval(self.layer, param=b_param.param) + + if self.layer_type == kRBMVis: + wname = w_param.param.name + parw = Parameter(name=wname+"_", init='none', share_from=wname) + bname = b_param.param.name + parb = Parameter(name=bname+"2", wd=0, init='constant') + self.bidirect = RBM(self.out_dim, name='RBMHid', type=kRBMHid, + w_param=parw, b_param=parb, sampling=sampling).layer + class Embedding(Layer): - def __init__(self, in_dim, out_dim, w_param=None, **kwargs): - super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding') - fields = { 'vocab_size': in_dim, - 'word_dim': out_dim } - setval(self.layer.Extensions[embedding_conf], **fields) - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - setParamField(w_param.param, 'w', True, **kwargs) - setval(self.layer, param=w_param.param) - + + def __init__(self, in_dim, out_dim, w_param=None, **kwargs): + + super(Embedding, self).__init__(name=generate_name('embedding', 1), + user_type='kEmbedding') + fields = {'vocab_size': in_dim, + 'word_dim': out_dim} + setval(self.layer.Extensions[embedding_conf], **fields) + if w_param == None: + # default: uniform + w_param = Parameter(name=generate_name('w'), init=init) + else: + set_param_field(w_param.param, 'w', True, **kwargs) + setval(self.layer, param=w_param.param) + class RNNLM(Layer): - def __init__(self, dim, w_param=None, **kwargs): - super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden') - if w_param == None: - w_param = Parameter(name=generateName('w'), init=init) # default: uniform - else: - setParamField(w_param.param, 'w', True, **kwargs) - setval(self.layer, param=w_param.param) + + def __init__(self, dim, w_param=None, **kwargs): + + super(RNNLM, self).__init__(name=generate_name('hidden', 1), + user_type='kHidden') + if w_param == None: + # default: uniform + w_param = Parameter(name=generate_name('w'), init=init) + else: + set_param_field(w_param.param, 'w', True, **kwargs) + setval(self.layer, param=w_param.param) class UserLossRNNLM(Layer): - def __init__(self, **kwargs): - super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss') - self.layer.Extensions[loss_conf].nclass = kwargs['nclass'] - self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size'] - setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', scale=0.3).param) - setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', scale=0.3).param) + def __init__(self, **kwargs): + super(UserLossRNNLM, self).__init__(name=generate_name('loss', 1), + user_type='kLoss') + self.layer.Extensions[loss_conf].nclass = kwargs['nclass'] + self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size'] + setval(self.layer, param=Parameter(name=generate_name('w'), + init='uniform', scale=0.3).param) + setval(self.layer, param=Parameter(name=generate_name('w', 1), + init='uniform', scale=0.3).param)
