Repository: incubator-systemml Updated Branches: refs/heads/master e8fbc7539 -> 5aa6fb75b
[SYSTEMML-1571] New Jupyter Python Notebook to showcase SystemML 2-layer autoencoder for acoustic signal modeling Closes #478. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5aa6fb75 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5aa6fb75 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5aa6fb75 Branch: refs/heads/master Commit: 5aa6fb75b066228d723dfc88bc0ae24551812e92 Parents: e8fbc75 Author: fmakari <[email protected]> Authored: Mon May 1 11:50:23 2017 -0700 Committer: Glenn Weidner <[email protected]> Committed: Mon May 1 11:50:24 2017 -0700 ---------------------------------------------------------------------- samples/jupyter-notebooks/Autoencoder.ipynb | 188 +++++++++++++++++++++++ 1 file changed, 188 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5aa6fb75/samples/jupyter-notebooks/Autoencoder.ipynb ---------------------------------------------------------------------- diff --git a/samples/jupyter-notebooks/Autoencoder.ipynb b/samples/jupyter-notebooks/Autoencoder.ipynb new file mode 100644 index 0000000..ec9c2aa --- /dev/null +++ b/samples/jupyter-notebooks/Autoencoder.ipynb @@ -0,0 +1,188 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Autoencoder\n", + "This notebook demonstrates the invocation of the SystemML autoencoder script, and alternative ways of passing in/out data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "#!pip install --user systemml>0.13.0\n", + "!pip show systemml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from systemml import MLContext, dml\n", + "ml = MLContext(sc)\n", + "print(ml.info())\n", + "sc.version" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## SystemML Read/Write data from local file system" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "FsPath = \"/tmp/data/\"\n", + "inp = FsPath + \"Input/\"\n", + "outp = FsPath + \"Output/\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generate Data and write out to file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_pd = pd.DataFrame(range(1, 2001,1),dtype=float).values.reshape(100,20)\n", + "script =\"\"\"\n", + " write(X, $Xfile)\n", + "\"\"\"\n", + "prog = dml(script).input(X=X_pd).input(**{\"$Xfile\":inp+\"X.csv\"})\n", + "ml.execute(prog)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "!ls -l /tmp/data/Input" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "autoencoderURL = \"https://raw.githubusercontent.com/apache/incubator-systemml/master/scripts/staging/autoencoder-2layer.dml\"\n";, + "rets = (\"iter\", \"num_iters_per_epoch\", \"beg\", \"end\", \"o\")\n", + "\n", + "prog = dml(autoencoderURL).input(**{\"$X\":inp+\"X.csv\"}) \\\n", + " .input(**{\"$H1\":500, \"$H2\":2, \"$BATCH\":36, \"$EPOCH\":5 \\\n", + " , \"$W1_out\":outp+\"W1_out\", \"$b1_out\":outp+\"b1_out\" \\\n", + " , \"$W2_out\":outp+\"W2_out\", \"$b2_out\":outp+\"b2_out\" \\\n", + " , \"$W3_out\":outp+\"W3_out\", \"$b3_out\":outp+\"b3_out\" \\\n", + " , \"$W4_out\":outp+\"W4_out\", \"$b4_out\":outp+\"b4_out\" \\\n", + " }).output(*rets)\n", + "iter, num_iters_per_epoch, beg, end, o = ml.execute(prog).get(*rets)\n", + "print (iter, num_iters_per_epoch, beg, end, o)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "!ls -l /tmp/data/Output" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Alternatively to passing in/out file names, use Python variables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [], + "source": [ + "autoencoderURL = \"https://raw.githubusercontent.com/apache/incubator-systemml/master/scripts/staging/autoencoder-2layer.dml\"\n";, + "rets = (\"iter\", \"num_iters_per_epoch\", \"beg\", \"end\", \"o\")\n", + "rets2 = (\"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\", \"W4\", \"b4\")\n", + "\n", + "prog = dml(autoencoderURL).input(X=X_pd) \\\n", + " .input(**{ \"$H1\":500, \"$H2\":2, \"$BATCH\":36, \"$EPOCH\":5}) \\\n", + " .output(*rets) \\\n", + " .output(*rets2)\n", + "result = ml.execute(prog)\n", + "iter, num_iters_per_epoch, beg, end, o = result.get(*rets)\n", + "W1, b1, W2, b2, W3, b3, W4, b4 = result.get(*rets2)\n", + "\n", + "print (iter, num_iters_per_epoch, beg, end, o)\n", + "W1.toDF().head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}
