This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new 42dd35b [MINOR] Addition of LSTM Sequence Replication Example
42dd35b is described below
commit 42dd35ba9d3b6cf523cdff38c870d09b0447730a
Author: Romeo Kienzler <[email protected]>
AuthorDate: Thu Mar 7 07:54:27 2019 +0530
[MINOR] Addition of LSTM Sequence Replication Example
Input sequence:
1.000 2.000 3.000 4.000 5.000
Normalized Input:
0.000 0.200 0.400 0.600 0.800
Iter log:
iter=0 loss=0.6437524920758813
iter=900 losss=2.368937593307822E-4
output sequence:
1.000 1.964 3.013 4.055 4.919
Closes #628
---
scripts/nn/examples/lstm_replicate_sequence.dml | 103 ++++++++++++++++++++++++
1 file changed, 103 insertions(+)
diff --git a/scripts/nn/examples/lstm_replicate_sequence.dml
b/scripts/nn/examples/lstm_replicate_sequence.dml
new file mode 100644
index 0000000..255a420
--- /dev/null
+++ b/scripts/nn/examples/lstm_replicate_sequence.dml
@@ -0,0 +1,103 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/**
+ * LSTM Replicate Sequence
+ *
+ * This script replicates a sequence of integer numbers.
+ *
+ * Inputs:
+ * - X : A sequence of integer values, of shape (n, 1)
+ *
+ * Outputs:
+ * - X_gen: The sequence generated by the NN based on training with X
+ *
+ * Data:
+ * A straightforward way to create the training data using pyspark
+ * is as follows
+ * from pyspark.sql.types import *
+ * X = spark.createDataFrame([1,2,3,4,5,6],IntegerType())
+ *
+ *
+ */
+
+# source relevant scripts
+source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss
+source("nn/layers/l2_loss.dml") as l2_loss
+source("nn/layers/lstm.dml") as lstm
+source("nn/layers/sigmoid.dml") as sigmoid
+source("nn/optim/sgd_nesterov.dml") as sgd_nesterov
+source("nn/optim/rmsprop.dml") as rmsprop
+
+# input sequence of [1, 2, 3, 4, 5]
+X = matrix("1 2 3 4 5", rows=5, cols=1);
+
+# initialization
+N = 1 # number of examples
+T = length(X) # length of training sequence
+D = 1 # number of features (dimension of X)
+M = 1 # number of neurons
+
+# data preparation / normalization
+X = t(X)
+min = min(X)
+max = max(X)
+X = (X-min)/max
+
+# debug print of training data
+print(toString(X))
+
+# init weight matrices of LSTM layer
+[W, b, out0, c0] = lstm::init(N,D,M)
+
+# init properties for gradient descent training
+max_iterations = 1000
+iter = 0
+
+# init adaptive learning rate component
+rmspropCache = rmsprop::init(W)
+learningRate = 0.01
+decayRate = 0.95
+
+while( iter < max_iterations ) {
+ # forward pass
+ [X_gen, c, c_out, c_c, c_ifog] = lstm::forward(X, W, b, T, D, TRUE, out0,
c0)
+ loss = l2_loss::forward(X_gen, X)
+
+ # debug every 100 iterations
+ if(iter %% 100 == 0) print("iter=" + iter + " loss=" + loss)
+
+ # calculate backward pass and gradient
+ loss_grad = l2_loss::backward(X_gen, X)
+ [dX, dW, db, dout0, dc0] = lstm::backward(loss_grad, c0, X, W, b, T, D,
TRUE, out0, c0, c_out, c_c, c_ifog)
+
+ # update weights
+ [W, rmspropCache] = rmsprop::update(W, dW, learningRate, decayRate, 1e-6,
rmspropCache)
+
+ iter = iter + 1
+}
+
+# final forward pass on trained network
+[X_gen, c, c_out, c_c, c_ifog] = lstm::forward(X, W, b, T, D, TRUE, out0, c0)
+
+# de-normalizaion and output of final result
+X_gen = X_gen*max+min
+print(toString(X_gen))
