Repository: incubator-systemml Updated Branches: refs/heads/master 468e472a6 -> 76553ebf0
[SYSTEMML-777] Restricted Boltzmann Machines train/predict scripts Closes #188. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/76553ebf Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/76553ebf Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/76553ebf Branch: refs/heads/master Commit: 76553ebf00828b3829c65596d997a6c7fa407703 Parents: 468e472 Author: nmanchev <[email protected]> Authored: Mon Jul 11 23:56:54 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Jul 11 23:56:54 2016 -0700 ---------------------------------------------------------------------- scripts/staging/rbm_minibatch.dml | 162 +++++++++++++++++++++++++++++++++ scripts/staging/rbm_predict.dml | 89 ++++++++++++++++++ 2 files changed, 251 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/76553ebf/scripts/staging/rbm_minibatch.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/rbm_minibatch.dml b/scripts/staging/rbm_minibatch.dml new file mode 100644 index 0000000..d7e45bb --- /dev/null +++ b/scripts/staging/rbm_minibatch.dml @@ -0,0 +1,162 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# THIS SCRIPT IMPLEMENTS A TRAINING ALGORITHM FOR RESTRICTED BOLTZMANN MACHINES +# +# The training algorithm uses one-step Contrastive Divergence as suggested in [Bengio, Y., Learning Deep +# Architectures for AI, Foundations and Trends in Machine Learning, Volume 2 Issue 1, January 2009, p. 1â127]. +# This implemention is slightly modified and uses mini-batches to speed up the learning process. +# +# INPUT PARAMETERS: +# -------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# -------------------------------------------------------------------------------------------- +# X String ---- Location (on HDFS) to read the matrix X of feature vectors +# W String ---- Location to store the estimated RBM weights +# A String ---- Location to store the visible units bias vector +# B String ---- Location to store the hidden units bias vector +# batchsize Int 100 Number of observations in a single batch +# vis Int ---- Number of units in the visible layer. If not specified this will be set +# to the number of features in X +# hid Int 2 Number of units in the hidden layer +# epochs Int 10 Number of training epochs +# alpha Double 0.1 Learning rate +# fmt String text Matrix output format for W,A, and B, usually "text" or "csv" +# -------------------------------------------------------------------------------------------- +# OUTPUT: Matrices containing weights and biases for the trained network +# +# OUTPUT SIZE: OUTPUT CONTENTS: +# W vis x hid RBM weights +# A 1 x vis Visible units biases +# B 1 x hid Hidden units biases +# +# To run an observation through the trained network use the rbm_run.dml script. +# Alternatively, you can simply compute P(h=1|v) using 1.0 / (1 + exp(-(X %*% W + B)) and +# get the values for the hidden layer by sampling from P(h=1|v). +# + +# HOW TO INVOKE THIS SCRIPT - EXAMPLE: +# hadoop jar SystemML.jar -f rbm.dml -nvargs X=$INPUT_DIR/X W=$OUTPUT_DIR/w A=$OUTPUT_DIR/a B=$OUTPUT_DIR/b batchsize=100 vis=10 hid=2 epochs=10 alpha=0.1 + + +# -------------------------------------------- START OF RBM_MINIBATCH.DML -------------------------------------------- + +# Default values +learning_rate = ifdef($alpha, 0.1) +hidden_units_count = ifdef($hid, 2) +epoch_count = ifdef($epochs, 10) +batch_size = ifdef($batchsize, 100) +output_format = ifdef($fmt, "text") + +print("BEGIN RBM MINIBATCH TRAINING SCRIPT") + +print("Reading X...") +X = read($X) + +# If number of visible units is not set, +# set it to the number of features in X +visible_units_count = ifdef ($vis, ncol(X)) + +# Get the total number of observations +N = nrow(X) + +# Rescale to interval [0,1] +minX = min(X) +maxX = max(X) +if (minX < 0.0 | maxX > 1.0) { + print("X not in [0,1]. Rescaling...") + X = (X-minX)/(maxX-minX) +} + +# Initialise a weight matrix (visible_units_count x hidden_units_count) +w = 0.1 * rand(rows = visible_units_count, cols = hidden_units_count, pdf = "uniform") + +# Setup biases and initialise with 0 +a = matrix(0, rows = 1, cols = visible_units_count) # Visible units bias +b = matrix(0, rows = 1, cols = hidden_units_count) # Hidden units bias + +# Train the RBM +print("Training starts...") + +for (epoch in 1:epoch_count) { + + # Cumulative error for this epoch + epoch_err = 0 + + # Loop over each batch + for (batch_start in seq(1, N, batch_size)) { + + # Compute the end of the batch for indexing + batch_end = batch_start + batch_size - 1 + + # Is this the last batch? + if (batch_end > N) { + batch_end = N + } + + # Present the current minibatch to the visible layer + v_1 = X[batch_start:batch_end,] + + # Get the number of observations in v_1 + batch_N = nrow(v_1) + + # POSITIVE PHASE + + # Compute the probabilities P(h=1|v) + p_h1_given_v = 1.0 / (1 + exp(-(v_1 %*% w + b))) + + # Sample from P(h=1|v) + h1 = p_h1_given_v > rand(rows = batch_N, cols = hidden_units_count) + + # NEGATIVE PHASE + + # Compute the probabilities P(v2=1|h1) + p_v2_given_h1 = 1.0 / (1 + exp(-(h1 %*% t(w) + a))) + + # Compute the probabilities P(h2=1|v2) + p_h2_given_v2 = 1.0 / (1 + exp(-(p_v2_given_h1 %*% w + b))) + + # UPDATE WEIGHTS AND BIASES + w = w + learning_rate * ((t(v_1) %*% p_h1_given_v - t(p_v2_given_h1) %*% p_h2_given_v2) / batch_N) + a = a + (learning_rate / batch_N) * (colSums(v_1) - colSums(p_v2_given_h1)) + b = b + (learning_rate / batch_N) * (colSums(p_h1_given_v) - colSums(p_h2_given_v2)) + + # COMPUTE ERROR + minibatch_err = sum((v_1 - p_v2_given_h1) ^ 2) / batch_N + epoch_err = epoch_err + minibatch_err + } + + print("Epoch " + epoch + " completed. Cummulative error is " + epoch_err) + +} + +# Save the RBM model +print("Saving RBM weights...") +write(w, $W, format=output_format) + +print("Saving RBM biases...") +write(a, $A, format=output_format) +write(b, $B, format=output_format) + +print("END OF RBM MINIBATCH TRAINING SCRIPT") + +# -------------------------------------------- END OF RBM_MINIBATCH.DML -------------------------------------------- + http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/76553ebf/scripts/staging/rbm_predict.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/rbm_predict.dml b/scripts/staging/rbm_predict.dml new file mode 100644 index 0000000..287c17f --- /dev/null +++ b/scripts/staging/rbm_predict.dml @@ -0,0 +1,89 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# THIS SCRIPT USES A TRAINED RESTRICTED BOLTZMANN MACHINES TO PROCESS A SET OF OBSERVATIONS +# +# The RBM training is handled by the rbm_minibatch.dml script. +# +# INPUT PARAMETERS: +# -------------------------------------------------------------------------------------------- +# NAME TYPE DEFAULT MEANING +# -------------------------------------------------------------------------------------------- +# X String --- Location (on HDFS) to read the matrix of observations X +# W String --- Location (on HDFS) to read the RBM's weight vector +# A String --- Location (on HDFS) to read the RBM's visible units bias vector +# B String --- Location (on HDFS) to read the RBM's hidden units bias vector +# O String --- Location to store the processed observations +# fmt String csv Matrix output format for O, usually "text" or "csv" +# -------------------------------------------------------------------------------------------- +# OUTPUT: A matrix containing a sample of P(h=1|v) for each observation in X +# +# OUTPUT SIZE: OUTPUT CONTENTS: +# O N x ncol(W) RBM outputs for each observation in X +# + +# HOW TO INVOKE THIS SCRIPT - EXAMPLE: +# hadoop jar SystemML.jar -f rbm_run.dml -nvargs X=$INPUT_DIR/X W=$INPUT_DIR/w A=$INPUT_DIR/a B=$INPUT_DIR/b O=$OUTPUT_DIR/O fmt="csv" + + +# -------------------------------------------- START OF RBM_MINIBATCH.DML -------------------------------------------- + +# Default values +output_format = ifdef($fmt, "csv") + +print("BEGIN RBM SCRIPT") + +print("Reading model data...") +#Read data set +X = read($X) + +# Get the total number of observations +N = nrow(X) + +# Read model coefficients +w = read($W) +a = read($A) +b = read($B) + +hidden_units_count = ncol(w) + +# Rescale to interval [0,1] +minX = min(X) +maxX = max(X) +if (minX < 0.0 | maxX > 1.0) { + print("X not in [0,1]. Rescaling...") + X = (X-minX)/(maxX-minX) +} + +print("Running RBM...") + +# Compute the probabilities P(h=1|v) +p_h1_given_v = 1.0 / (1 + exp(-(X %*% w + b))) + +# Sample from P(h=1|v) +h1 = p_h1_given_v > rand(rows = N, cols = hidden_units_count) + +print("Saving hidden layer sample...") +write(h1, $O, format=output_format) + +print("END RBM SCRIPT") + +# -------------------------------------------- END OF RBM_RUN.DML --------------------------------------------
