http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5144bcf1/doc/en/docs/notebook/model.ipynb
----------------------------------------------------------------------
diff --git a/doc/en/docs/notebook/model.ipynb b/doc/en/docs/notebook/model.ipynb
new file mode 100644
index 0000000..23a5553
--- /dev/null
+++ b/doc/en/docs/notebook/model.ipynb
@@ -0,0 +1,536 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# SINGA Model Classes\n",
+ "\n",
+ "<img src=\"http://singa.apache.org/en/_static/images/singav1-sw.png\";
width=\"500px\"/>"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Layer\n",
+ "\n",
+ "Typically, the life cycle of a layer instance includes:\n",
+ " 1. construct layer without input_sample_shapes, goto 2; or,\n",
+ " \n",
+ " construct layer with input_sample_shapes, goto 3;\n",
+ " 2. call setup to create the parameters and setup other meta fields;\n",
+ " 4. initialize the parameters of the layer\n",
+ " 3. call forward or access layer members\n",
+ " 4. call backward and get parameters for update"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "from singa import tensor, device, layer\n",
+ "\n",
+ "#help(layer.Layer)\n",
+ "layer.engine='singacpp'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Common layers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "from singa.layer import Dense, Conv2D, MaxPooling2D, Activation,
BatchNormalization, Softmax"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Dense Layer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2, 3) (3,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "dense = Dense('dense', 3, input_sample_shape=(2,))\n",
+ "#dense.param_names()\n",
+ "w, b = dense.param_values()\n",
+ "print w.shape, b.shape"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "w.gaussian(0, 0.1)\n",
+ "b.set_value(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([[ 0.02440065, -0.03396009, 0.01396658],\n",
+ " [ 0.00771775, 0.07841966, -0.05931653]], dtype=float32)"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "x = tensor.Tensor((2,2))\n",
+ "x.uniform(-1, 1)\n",
+ "y = dense.forward(True, x)\n",
+ "tensor.to_numpy(y)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(2, 2) (2, 3) (3,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "gx, [gw, gb] = dense.backward(True, y)\n",
+ "print gx.shape, gw.shape, gb.shape"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Convolution Layer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4, 6, 6)\n"
+ ]
+ }
+ ],
+ "source": [
+ "conv = Conv2D('conv', 4, 3, 1, input_sample_shape=(3, 6, 6))\n",
+ "print conv.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Pooling Layer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4, 3, 3)\n"
+ ]
+ }
+ ],
+ "source": [
+ "pool = MaxPooling2D('pool', 3, 2, input_sample_shape=(4, 6, 6))\n",
+ "print pool.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Branch layers"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "from singa.layer import Split, Merge, Slice, Concat"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[(4, 6, 6), (4, 6, 6)]\n"
+ ]
+ }
+ ],
+ "source": [
+ "split = Split('split', 2, input_sample_shape=(4, 6, 6))\n",
+ "print split.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4, 6, 6)\n"
+ ]
+ }
+ ],
+ "source": [
+ "merge = Merge('merge', input_sample_shape=(4, 6, 6))\n",
+ "print merge.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[(2, 6, 6), (2, 6, 6)]\n"
+ ]
+ }
+ ],
+ "source": [
+ "sli = Slice('slice', 1, [2], input_sample_shape=(4, 6, 6))\n",
+ "print sli.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(4, 6, 6)\n"
+ ]
+ }
+ ],
+ "source": [
+ "concat = Concat('concat', 1, input_sample_shapes=[(3, 6, 6), (1, 6,
6)])\n",
+ "print concat.get_output_sample_shape()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Metric and Loss"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[ 0.13973515 0.21827343 0.21068712 0.21905626 0.21224809]\n",
+ " [ 0.2354937 0.1047527 0.18490241 0.23617713 0.23867407]\n",
+ " [ 0.2659435 0.11397494 0.1659178 0.22683497 0.22732878]]\n",
+ "0.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "from singa import metric\n",
+ "import numpy as np\n",
+ "\n",
+ "x = tensor.Tensor((3, 5))\n",
+ "x.uniform(0, 1) # randomly genearte the prediction activation\n",
+ "x = tensor.softmax(x) # normalize the prediction into probabilities\n",
+ "print tensor.to_numpy(x)\n",
+ "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int)) # set the
truth\n",
+ "\n",
+ "f = metric.Accuracy()\n",
+ "acc = f.evaluate(x, y) # averaged accuracy over all 3 samples in x\n",
+ "print acc"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "1.80309379101\n",
+ "[[-0.78104687 0.18748793 0.16346708 0.24803984 0.18205206]\n",
+ " [ 0.21501946 -0.83683592 0.19003348 0.20714596 0.22463693]\n",
+ " [ 0.20000091 0.23285127 0.26842937 -0.87474263 0.17346108]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "from singa import loss\n",
+ "\n",
+ "x = tensor.Tensor((3, 5))\n",
+ "x.uniform(0, 1) # randomly genearte the prediction activation\n",
+ "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int)) # set the
truth\n",
+ "\n",
+ "f = loss.SoftmaxCrossEntropy()\n",
+ "l = f.forward(True, x, y) # l is tensor with 3 loss values\n",
+ "g = f.backward() # g is a tensor containing all gradients of x w.r.t
l\n",
+ "print l.l1()\n",
+ "print tensor.to_numpy(g)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Optimizer"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<singa.tensor.Tensor at 0x7f6a0c7cfe90>"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from singa import optimizer\n",
+ "\n",
+ "sgd = optimizer.SGD(lr=0.01, momentum=0.9, weight_decay=1e-4)\n",
+ "p = tensor.Tensor((3,5))\n",
+ "p.uniform(-1, 1)\n",
+ "g = tensor.Tensor((3,5))\n",
+ "g.gaussian(0, 0.01)\n",
+ "\n",
+ "sgd.apply(1, g, p, 'param') # use the global lr=0.1 for epoch 1\n",
+ "sgd.apply_with_lr(2, 0.03, g, p, 'param') # use lr=0.03 for epoch 2"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## FeedForwardNet"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "conv1 (32, 32, 32)\n",
+ "relu1 (32, 32, 32)\n",
+ "pool1 (32, 16, 16)\n",
+ "flat (8192,)\n",
+ "dense (10,)\n",
+ "[u'conv1_weight', u'conv1_bias', u'dense_weight', u'dense_bias']\n"
+ ]
+ }
+ ],
+ "source": [
+ "from singa import net as ffnet\n",
+ "layer.engine = 'singacpp'\n",
+ "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(),
metric.Accuracy())\n",
+ "net.add(layer.Conv2D('conv1', 32, 5, 1,
input_sample_shape=(3,32,32,)))\n",
+ "net.add(layer.Activation('relu1'))\n",
+ "net.add(layer.MaxPooling2D('pool1', 3, 2))\n",
+ "net.add(layer.Flatten('flat'))\n",
+ "net.add(layer.Dense('dense', 10))\n",
+ "\n",
+ "# init parameters\n",
+ "for p in net.param_values():\n",
+ " if len(p.shape) == 0:\n",
+ " p.set_value(0)\n",
+ " else:\n",
+ " p.gaussian(0, 0.01)\n",
+ "print net.param_names()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "conv1 (32, 32, 32)\n",
+ "relu1 (32, 32, 32)\n",
+ "pool1 (32, 16, 16)\n",
+ "flat (8192,)\n",
+ "dense (10,)\n"
+ ]
+ }
+ ],
+ "source": [
+ "layer.engine = 'cudnn'\n",
+ "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(),
metric.Accuracy())\n",
+ "net.add(layer.Conv2D('conv1', 32, 5, 1,
input_sample_shape=(3,32,32,)))\n",
+ "net.add(layer.Activation('relu1'))\n",
+ "net.add(layer.MaxPooling2D('pool1', 3, 2))\n",
+ "net.add(layer.Flatten('flat'))\n",
+ "net.add(layer.Dense('dense', 10))\n",
+ "\n",
+ "# init parameters\n",
+ "for p in net.param_values():\n",
+ " if len(p.shape) == 0:\n",
+ " p.set_value(0)\n",
+ " else:\n",
+ " p.gaussian(0, 0.01)\n",
+ " \n",
+ "# move net onto gpu\n",
+ "dev = device.create_cuda_gpu()\n",
+ "net.to_device(dev)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "collapsed": true
+ },
+ "source": [
+ "## Next: [Simple models](./regression.ipynb)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "anaconda-cloud": {},
+ "kernelspec": {
+ "display_name": "Python [conda env:conda]",
+ "language": "python",
+ "name": "conda-env-conda-py"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
![http://singa.apache.org/en/_static/images/singav1-sw.png\"](http://singa.apache.org/en/_static/images/singav1-sw.png\"){kind=link}