Repository: incubator-systemml Updated Branches: refs/heads/master d8a8c3b69 -> 3a3c16599
[SYSTEMML-618] SystemML-NN: Adding an MNIST softmax classifier example, including a DML script and a Jupyter PySpark notebook. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/60f3fe60 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/60f3fe60 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/60f3fe60 Branch: refs/heads/master Commit: 60f3fe607ac609576579202a9bca3a2723f1e6e2 Parents: d8a8c3b Author: Mike Dusenberry <[email protected]> Authored: Fri Jun 24 17:28:43 2016 -0700 Committer: Mike Dusenberry <[email protected]> Committed: Fri Jun 24 17:28:43 2016 -0700 ---------------------------------------------------------------------- scripts/staging/SystemML-NN/.gitignore | 2 - .../Example - MNIST Softmax Classifier.ipynb | 207 +++++++++++++++++++ .../SystemML-NN/examples/mnist_softmax.dml | 203 ++++++++++++++++++ 3 files changed, 410 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/60f3fe60/scripts/staging/SystemML-NN/.gitignore ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/.gitignore b/scripts/staging/SystemML-NN/.gitignore deleted file mode 100644 index a1b402b..0000000 --- a/scripts/staging/SystemML-NN/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -scratch_space -*.ipynb http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/60f3fe60/scripts/staging/SystemML-NN/Example - MNIST Softmax Classifier.ipynb ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/Example - MNIST Softmax Classifier.ipynb b/scripts/staging/SystemML-NN/Example - MNIST Softmax Classifier.ipynb new file mode 100644 index 0000000..5c1de15 --- /dev/null +++ b/scripts/staging/SystemML-NN/Example - MNIST Softmax Classifier.ipynb @@ -0,0 +1,207 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quick Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "# Add SystemML PySpark API file.\n", + "sc.addPyFile(\"https://raw.githubusercontent.com/apache/incubator-systemml/3d5f9b11741f6d6ecc6af7cbaa1069cde32be838/src/main/java/org/apache/sysml/api/python/SystemML.py\";)\n", + "\n", + "# Create a SystemML MLContext object\n", + "from SystemML import MLContext\n", + "ml = MLContext(sc)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download Data - MNIST" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The MNIST dataset contains labeled images of handwritten digits, where each example is a 28x28 pixel image of grayscale values scaled to [0,1] and stretched out as 784 pixels, and each label is a one-hot encoding over 10 possible digits. Here, we use TensorFlow's API for accessing the data, and retrieve 50,000 training examples, 5,000 validation examples, and 10,000 test examples. [Note: TensorFlow can easily be installed via [these instructions](https://www.tensorflow.org/versions/r0.9/get_started/os_setup.html#pip-installation).]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%%sh\n", + "mkdir -p examples/data/mnist/" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "scrolled": false + }, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "from tensorflow.examples.tutorials.mnist import input_data\n", + "\n", + "# Get MNIST data\n", + "mnist = input_data.read_data_sets('MNIST_data', one_hot=True)\n", + "\n", + "# Save to CSV\n", + "np.savetxt(\"examples/data/mnist/train_images.csv\", mnist.train.images, delimiter=\",\")\n", + "np.savetxt(\"examples/data/mnist/train_labels.csv\", mnist.train.labels, delimiter=\",\")\n", + "np.savetxt(\"examples/data/mnist/val_images.csv\", mnist.validation.images, delimiter=\",\")\n", + "np.savetxt(\"examples/data/mnist/val_labels.csv\", mnist.validation.labels, delimiter=\",\")\n", + "np.savetxt(\"examples/data/mnist/test_images.csv\", mnist.test.images, delimiter=\",\")\n", + "np.savetxt(\"examples/data/mnist/test_labels.csv\", mnist.test.labels, delimiter=\",\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SystemML Softmax Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Train" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "script = \"\"\"\n", + "source(\"examples/mnist_softmax.dml\") as mnist_softmax\n", + "\n", + "# Read data\n", + "X = read($X, format=\"csv\")\n", + "y = read($y, format=\"csv\")\n", + "\n", + "X_val = read($X_val, format=\"csv\")\n", + "y_val = read($y_val, format=\"csv\")\n", + "\n", + "# Train\n", + "[W, b] = mnist_softmax::train(X, y, X_val, y_val)\n", + "\n", + "# Write model out\n", + "write(W, $Wout)\n", + "write(b, $bout)\n", + "\n", + "print(\"\")\n", + "print(\"\")\n", + "\"\"\"\n", + "ml.reset()\n", + "out = ml.executeScript(script, {\"X\": \"examples/data/mnist/train_images.csv\", \n", + " \"y\": \"examples/data/mnist/train_labels.csv\",\n", + " \"X_val\": \"examples/data/mnist/val_images.csv\", \n", + " \"y_val\": \"examples/data/mnist/val_labels.csv\"},\n", + " outputs=[\"W\", \"b\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. Extract model from SystemML back into PySpark" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "W = out.getDF(sqlContext, \"W\").sort(\"ID\").drop(\"ID\")\n", + "b = out.getDF(sqlContext, \"b\").sort(\"ID\").drop(\"ID\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3. Compute Test Accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "script = \"\"\"\n", + "source(\"examples/mnist_softmax.dml\") as mnist_softmax\n", + "\n", + "# Read data & coefficients\n", + "X_test = read($X_test, format=\"csv\")\n", + "y_test = read($y_test, format=\"csv\")\n", + "W = read($W)\n", + "b = read($b)\n", + "\n", + "# Eval on test set\n", + "[loss, accuracy] = mnist_softmax::eval(X_test, y_test, W, b)\n", + "\n", + "print(\"Accuracy: \" + accuracy)\n", + "\n", + "print(\"\")\n", + "print(\"\")\n", + "\"\"\"\n", + "ml.reset()\n", + "out = ml.executeScript(script, {\"X_test\": \"examples/data/mnist/test_images.csv\",\n", + " \"y_test\": \"examples/data/mnist/test_labels.csv\",\n", + " \"W\": W, \"b\": b})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/60f3fe60/scripts/staging/SystemML-NN/examples/mnist_softmax.dml ---------------------------------------------------------------------- diff --git a/scripts/staging/SystemML-NN/examples/mnist_softmax.dml b/scripts/staging/SystemML-NN/examples/mnist_softmax.dml new file mode 100644 index 0000000..f3c47e9 --- /dev/null +++ b/scripts/staging/SystemML-NN/examples/mnist_softmax.dml @@ -0,0 +1,203 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +/* + * MNIST Softmax Example + */ +# Imports +source("nn/layers/affine.dml") as affine +source("nn/layers/cross_entropy_loss.dml") as cross_entropy_loss +source("nn/layers/softmax.dml") as softmax +source("nn/optim/sgd_nesterov.dml") as sgd_nesterov + +train = function(matrix[double] X, matrix[double] y, + matrix[double] X_val, matrix[double] y_val) + return (matrix[double] W, matrix[double] b) { + /* + * Trains a softmax classifier. The input matrix, X, has N examples, + * each with D features. The targets, y, have K classes. + * + * Inputs: + * - X: Input data matrix, of shape (N, D). + * - y: Target matrix, of shape (N, K). + * - X_val: Input validation data matrix, of shape (N, C*Hin*Win). + * - y_val: Target validation matrix, of shape (N, K). + * + * Outputs: + * - W: Weights (parameters) matrix, of shape (D, M). + * - b: Biases vector, of shape (1, M). + */ + N = nrow(X) # num examples + D = ncol(X) # num features + K = ncol(y) # num classes + + # Create softmax classifier: + # affine -> softmax + [W, b] = affine::init(D, K) + W = W / sqrt(2.0/(D)) * sqrt(1/(D)) + + # Initialize SGD w/ Nesterov momentum optimizer + lr = 0.2 # learning rate + mu = 0 # momentum + decay = 0.99 # learning rate decay constant + vW = sgd_nesterov::init(W) # optimizer momentum state for W + vb = sgd_nesterov::init(b) # optimizer momentum state for b + + # Optimize + print("Starting optimization") + batch_size = 50 + epochs = 1 + iters = 1000 #ceil(N / batch_size) + for (e in 1:epochs) { + for(i in 1:iters) { + # Get next batch + beg = ((i-1) * batch_size) %% N + 1 + end = min(N, beg + batch_size - 1) + X_batch = X[beg:end,] + y_batch = y[beg:end,] + + # Compute forward pass + ## affine & softmax: + out = affine::forward(X_batch, W, b) + probs = softmax::forward(out) + + # Compute loss & accuracy for training & validation data + loss = cross_entropy_loss::forward(probs, y_batch) + accuracy = mean(rowIndexMax(probs) == rowIndexMax(y_batch)) + [loss_val, accuracy_val] = eval(X_val, y_val, W, b) + print("Epoch: " + e + ", Iter: " + i + ", Train Loss: " + loss + ", Train Accuracy: " + accuracy + ", Val Loss: " + loss_val + ", Val Accuracy: " + accuracy_val) + + # Compute backward pass + ## loss: + dprobs = cross_entropy_loss::backward(probs, y_batch) + ## affine & softmax: + dout = softmax::backward(dprobs, out) + [dX_batch, dW, db] = affine::backward(dout, X_batch, W, b) + + # Optimize with SGD w/ Nesterov momentum + [W, vW] = sgd_nesterov::update(W, dW, lr, mu, vW) + [b, vb] = sgd_nesterov::update(b, db, lr, mu, vb) + } + # Anneal momentum towards 0.999 + mu = mu + (0.999 - mu)/(1+epochs-e) + # Decay learning rate + lr = lr * decay + } +} + +eval = function(matrix[double] X, matrix[double] y, matrix[double] W, matrix[double] b) + return (double loss, double accuracy) { + /* + * Evaluates a softmax classifier. The input matrix, X, has N + * examples, each with D features. The targets, y, have K classes. + * + * Inputs: + * - X: Input data matrix, of shape (N, D). + * - y: Target matrix, of shape (N, K). + * - W: Weights (parameters) matrix, of shape (D, M). + * - b: Biases vector, of shape (1, M). + * + * Outputs: + * - loss: Scalar loss, of shape (1). + * - accuracy: Scalar accuracy, of shape (1). + */ + # Compute forward pass + ## affine & softmax: + out = affine::forward(X, W, b) + probs = softmax::forward(out) + + # Compute loss & accuracy + loss = cross_entropy_loss::forward(probs, y) + correct_pred = rowIndexMax(probs) == rowIndexMax(y) + accuracy = mean(correct_pred) +} + +generate_dummy_data = function() + return (matrix[double] X, matrix[double] y, int C, int Hin, int Win) { + /* + * Generate a dummy dataset similar to the MNIST dataset. + * + * Outputs: + * - X: Input data matrix, of shape (N, D). + * - y: Target matrix, of shape (N, K). + * - C: Number of input channels (dimensionality of input depth). + * - Hin: Input height. + * - Win: Input width. + */ + # Generate dummy input data + N = 1024 # num examples + C = 1 # num input channels + Hin = 28 # input height + Win = 28 # input width + T = 10 # num targets + X = rand(rows=N, cols=C*Hin*Win, pdf="normal") + classes = round(rand(rows=N, cols=1, min=1, max=T, pdf="uniform")) + y = table(seq(1, N), classes) # one-hot encoding +} + + +# +# Main +# +# This runs if called as a script. +# +# Ex: +# ``` +# $SPARK_HOME/bin/spark-submit --master local[*] --driver-memory 5G +# --conf spark.driver.maxResultSize=0 --conf spark.akka.frameSize=128 +# $SYSTEMML_HOME/target/SystemML.jar -f examples/mnist_softmax.dml +# -nvargs X=examples/data/mnist/train_images.csv y=examples/data/mnist/train_labels.csv +# X_val=examples/data/mnist/val_images.csv y_val=examples/data/mnist/val_labels.csv +# X_test=examples/data/mnist/test_images.csv y_test=examples/data/mnist/test_labels.csv +# out_dir=examples/model/mnist_softmax +# ``` +# +# The MNIST dataset contains labeled images of handwritten digits, +# where each example is a 28x28 pixel image of grayscale values +# scaled to [0,1] and stretched out as 784 pixels, and each label +# is a one-hot encoding over 10 possible digits. +# + +# Read data +X = read($X, format="csv") +y = read($y, format="csv") +X_val = read($X_val, format="csv") +y_val = read($y_val, format="csv") +X_test = read($X_test, format="csv") +y_test = read($y_test, format="csv") + +# Train +[W, b] = train(X, y, X_val, y_val) + +# Write model out +write(W, $out_dir+"/W") +write(b, $out_dir+"/b") + +# Eval on test set +[loss, accuracy] = eval(X_test, y_test, W, b) + +# Output results +print("Accuracy: " + accuracy) +write(accuracy, $out_dir+"/accuracy") + +print("") +print("") +
