Repository: systemml Updated Branches: refs/heads/master 23157be4d -> 42359f11c
[SYSTEMML-1437] Factorization Machines Binary classification script This patch adds the binary classification script built on top of factorization machines with a sample data script for evaluation. Closes #699. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/42359f11 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/42359f11 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/42359f11 Branch: refs/heads/master Commit: 42359f11c9c215107421606bab93f6db65fea2fa Parents: 23157be Author: Janardhan Pulivarthi <[email protected]> Authored: Wed May 9 13:46:48 2018 +0530 Committer: Janardhan Pulivarthi <[email protected]> Committed: Wed May 9 14:02:06 2018 +0530 ---------------------------------------------------------------------- scripts/nn/examples/fm-binclass-dummy-data.dml | 48 +++++ scripts/staging/fm-binclass.dml | 183 ++++++++++++++++++++ 2 files changed, 231 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/42359f11/scripts/nn/examples/fm-binclass-dummy-data.dml ---------------------------------------------------------------------- diff --git a/scripts/nn/examples/fm-binclass-dummy-data.dml b/scripts/nn/examples/fm-binclass-dummy-data.dml new file mode 100644 index 0000000..20fe7e1 --- /dev/null +++ b/scripts/nn/examples/fm-binclass-dummy-data.dml @@ -0,0 +1,48 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +#imports +source("staging/fm-binclass.dml") as fm_binclass + +# generate dummy data ( this is just a sample! ) +n = 1000; d = 7; k=2; +X = rand(rows=n, cols=d); +y = round(rand(rows=n, cols=1)); +X_val = rand(rows=100, cols=7); +y_val = round(rand(rows=100, cols=1)); + +# Train +[w0, W, V, loss] = fm_binclass::train(X, y, X_val, y_val); + +# Write model out +#write(w0, out_dir+"/w0"); +#write(W, out_dir+"/W"); +#write(V, out_dir+"/V"); + +# eval on test set +probs = fm_binclass::predict(X, w0, W, V); +[loss, accuracy] = fm_binclass::eval(probs, y); + +# Output results +print("Test Accuracy: " + accuracy) +#write(accuracy, out_dir+"/accuracy") + +print("") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/42359f11/scripts/staging/fm-binclass.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/fm-binclass.dml b/scripts/staging/fm-binclass.dml new file mode 100644 index 0000000..f777544 --- /dev/null +++ b/scripts/staging/fm-binclass.dml @@ -0,0 +1,183 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +/* + * Factorization Machines for binary classification. + */ + +# Imports +source("nn/optim/adam.dml") as adam +source("nn/layers/fm.dml") as fm +source("nn/layers/log_loss.dml") as log_loss +source("nn/layers/sigmoid.dml") as sigmoid +source("nn/layers/l2_reg.dml") as l2_reg +source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss + +train = function(matrix[double] X, matrix[double] y, matrix[double] X_val, matrix[double] y_val) + return (matrix[double] w0, matrix[double] W, matrix[double] V, double loss) { + /* + * Trains the FM model. + * + * Inputs: + * - X : n examples with d features, of shape (n, d). + * - y : label corresponds to n examples + * - lambda : regularization (5e-04) + * + * Outputs: + * - w0, W, V : updated model parameters. + * - loss : computed loss with log_loss. + * + * input propagation through layers + * fm::init -> adam::init -> fm::forward -> sigmoid::forward -> log_loss::forward \ + * adam::update <- fm::backward <- sigmoid::backward <- log_loss::backward <- + */ + + n = nrow(X); + d = ncol(X); + k = 2; # factorization dimensionality, only(=2) possible for now. + + # 1.initialize fm core + [w0, W, V] = fm::init(n, d, k); + + # 2.initialize adam optimizer + ## Default values for some parameters + lr = 0.001; + beta1 = 0.9; # [0, 1) + beta2 = 0.999; # [0, 1) + epsilon = 0.00000001; + t = 0; + + # [mX, vX] = adam::init(X); # to optimize input. + [mw0, vw0] = adam::init(w0); + [mW, vW] = adam::init(W); + [mV, vV] = adam::init(V); + + # Regularization + lambda = 5e-04 + + # Optimize + print("Starting optimization") + batch_size = 10 + iters = ceil(1000 / batch_size) + epochs = 100; N = n; + for (e in 1:epochs) { + for (i in 1:iters) { + # Get the next batch + beg = ((i-1) * batch_size) %% N + 1 + end = min(N, beg + batch_size - 1) + X_batch = X[beg:end,] + y_batch = y[beg:end,] + + # 3.Send inputs through fm::forward + y_res = fm::forward(X_batch, w0, W, V); + + # 4.Send the above result through sigmoid::forward + sfy = sigmoid::forward(y_res); + + # 5.Send the above result through log_loss::forward + loss = log_loss::forward(sfy, y_batch); + + # Compute loss & accuracy for training & validation data every 100 iterations. + if (i %% 100 == 0) { + # Compute training loss & accuracy + loss_data = log_loss::forward(sfy, y_batch); + loss_reg_w0 = l2_reg::forward(w0, lambda); + loss_reg_W = l2_reg::forward(W, lambda); + loss_reg_V = l2_reg::forward(V, lambda); + + accuracy = mean((sfy<0.5) == (y_batch<0.5)); + loss = loss_data + loss_reg_w0 + loss_reg_W + loss_reg_V; + + # Compute validation loss & accuracy + probs_val = predict(X_val, w0, W, V) + loss_val = log_loss::forward(probs_val, y_val) + accuracy_val = mean((probs_val<0.5) == (y_val<0.5)) + + # Output results + print("Epoch: " + e + ", Iter: " + i + ", Train Loss: " + loss + ", Train Accuracy: " + + accuracy + ", Val Loss: " + loss_val + ", Val Accuracy: " + accuracy_val) + } + + # 6.Send the result of sigmoid::forward and the correct labels y to log_loss::backward + dsfy = log_loss::backward(sfy, y_batch); + + # 7.Send the above result through sigmoid::backward + dy = sigmoid::backward(dsfy, y_res); + + # 8.Send the above result through fm::backward + [dw0, dW, dV] = fm::backward(dy, X_batch, w0, W, V); + + # 9. update the timestep + t = e * i - 1; + + # 10.Call adam::update for all parameters + + # Incase we want to optimize inputs (X) also, as in deep dream. + #[X, mX, vX] = adam::update(X, dX, lr, beta1, beta2, epsilon, t, mX, vX); + + [w0, mw0, vw0] = adam::update(w0, dw0, lr, beta1, beta2, epsilon, t, mw0, vw0); + [W, mW, vW] = adam::update(W, dW, lr, beta1, beta2, epsilon, t, mW, vW ); + [V, mV, vV] = adam::update(V, dV, lr, beta1, beta2, epsilon, t, mV, vV ); + } + } +} + +predict = function(matrix[double] X, matrix[double] w0, matrix[double] W, matrix[double] V) + return (matrix[double] out) { + /* + * Computes the predictions for the given inputs. + * + * Inputs: + * - X : n examples with d features, of shape (n, d). + * - w0, W, V : trained model parameters. + * + * Outputs: + * - out : target vector, y. + */ + + # 1.initialize fm core + #[w0, W, V] = fm::init(d, k); + + # 2.Send inputs through fm::forward + y = fm::forward(X, w0, W, V); + + # 3.Send the above result through sigmoid::forward + out = sigmoid::forward(y); + + # 4.Send the above result through log_loss::forward + # loss = log_loss::forward(out); + +} + +eval = function(matrix[double] probs, matrix[double] y) + return (double loss, double accuracy) { + /** + * Computes loss and accuracy. + */ + + # 1. compute log loss + loss = log_loss::forward(probs, y); + + # 2. compute accuracy + accuracy = mean( (probs<0.5) == (y<0.5) ) +} +
