Author: buildbot
Date: Mon Jan 11 10:49:21 2016
New Revision: 977469
Log:
Staging update by buildbot for singa
Added:
websites/staging/singa/trunk/content/docs/general-rnn.html
websites/staging/singa/trunk/content/images/char-rnn-net.jpg (with props)
websites/staging/singa/trunk/content/images/char-rnn.png (with props)
Modified:
websites/staging/singa/trunk/content/ (props changed)
websites/staging/singa/trunk/content/docs/index.html
Propchange: websites/staging/singa/trunk/content/
------------------------------------------------------------------------------
--- cms:source-revision (original)
+++ cms:source-revision Mon Jan 11 10:49:21 2016
@@ -1 +1 @@
-1723990
+1724001
Added: websites/staging/singa/trunk/content/docs/general-rnn.html
==============================================================================
--- websites/staging/singa/trunk/content/docs/general-rnn.html (added)
+++ websites/staging/singa/trunk/content/docs/general-rnn.html Mon Jan 11
10:49:21 2016
@@ -0,0 +1,471 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2016-01-11
+ | Rendered using Apache Maven Fluido Skin 1.4
+-->
+<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20160111" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Apache SINGA – RNN in SINGA</title>
+ <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" />
+ <link rel="stylesheet" href="../css/site.css" />
+ <link rel="stylesheet" href="../css/print.css" media="print" />
+
+
+
+
+
+ <script type="text/javascript"
src="../js/apache-maven-fluido-1.4.min.js"></script>
+
+
+
+<script
src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
type="text/javascript"></script>
+
+
+<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath:
[['$','$'], ['\\(','\\)']]}});</script>
+
+
+<meta content="Apache SINGA" name="author"/>
+
+
+<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/>
+
+ </head>
+ <body class="topBarEnabled">
+
+
+
+
+
+
+ <a href="https://github.com/apache/incubator-singa";>
+ <img style="position: absolute; top: 0; right: 0; border: 0; z-index:
10000;"
+
src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png";
+ alt="Fork me on GitHub">
+ </a>
+
+
+
+
+
+ <div id="topbar" class="navbar navbar-fixed-top navbar-inverse">
+ <div class="navbar-inner">
+ <div class="container-fluid">
+ <a data-target=".nav-collapse" data-toggle="collapse" class="btn
btn-navbar">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+
+ <ul class="nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache
SINGA <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../docs/overview.html"
title="Introduction">Introduction</a>
+</li>
+
+ <li> <a href="../docs/quick-start.html"
title="Quick Start">Quick Start</a>
+</li>
+
+ <li> <a href="../downloads.html"
title="Downloads">Downloads</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle"
data-toggle="dropdown">Documentaion <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li class="dropdown-submenu">
+ <a href="../docs/index.html"
title="Latest">Latest</a>
+ <ul class="dropdown-menu">
+ <li> <a
href="../docs/installation.html" title="Installation">Installation</a>
+</li>
+ <li> <a
href="../docs/programmer-guide.html" title="Programmer Guide">Programmer
Guide</a>
+</li>
+ <li> <a
href="../docs/distributed-training.html" title="Distributed
Training">Distributed Training</a>
+</li>
+ <li> <a href="../docs/data.html"
title="Data Preparation">Data Preparation</a>
+</li>
+ <li> <a href="../docs/checkpoint.html"
title="Checkpoint">Checkpoint</a>
+</li>
+ <li> <a href="../docs/test.html"
title="Test">Test</a>
+</li>
+ <li> <a href="../docs/examples.html"
title="Examples">Examples</a>
+</li>
+ <li> <a href="../docs/debug.html"
title="Debug">Debug</a>
+</li>
+ </ul>
+ </li>
+
+ <li> <a href="../v0.1.0/index.html"
title="v0.1.0">v0.1.0</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development
<b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../develop/schedule.html"
title="Schedule">Schedule</a>
+</li>
+
+ <li class="dropdown-submenu">
+ <a href="../develop/how-contribute.html"
title="How to Contribute">How to Contribute</a>
+ <ul class="dropdown-menu">
+ <li> <a
href="../develop/contribute-code.html" title="Code">Code</a>
+</li>
+ <li> <a
href="../develop/contribute-docs.html" title="Documentation">Documentation</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community
<b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../community/source-repository.html"
title="Source Repository">Source Repository</a>
+</li>
+
+ <li> <a href="../community/mail-lists.html"
title="Mailing Lists">Mailing Lists</a>
+</li>
+
+ <li> <a href="../community/issue-tracking.html"
title="Issue Tracking">Issue Tracking</a>
+</li>
+
+ <li> <a href="../community/team-list.html"
title="SINGA Team">SINGA Team</a>
+</li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">External
Links <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="http://www.apache.org/";
title="Apache Software Foundation">Apache Software Foundation</a>
+</li>
+
+ <li> <a
href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS
Site</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="../index.html"
id="bannerLeft" title="Apache SINGA">
+
<img src="../images/singa-logo.png" alt="Apache SINGA"/>
+ </a>
+ </div>
+ <div class="pull-right"> <div id="bannerRight">
+
<img src="../images/singa-title.png" alt="Apache SINGA"/>
+ </div>
+ </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li class="">
+ <a href="../index.html" title="Apache SINGA">
+ Apache SINGA</a>
+ <span class="divider">/</span>
+ </li>
+ <li class="active ">RNN in SINGA</li>
+
+
+
+
+ </ul>
+ </div>
+
+
+ <div class="row-fluid">
+ <div id="leftColumn" class="span2">
+ <div class="well sidebar-nav">
+
+
+ <ul class="nav nav-list">
+ <li class="nav-header">Apache SINGA</li>
+
+ <li>
+
+ <a href="../docs/overview.html" title="Introduction">
+ <span class="none"></span>
+ Introduction</a>
+ </li>
+
+ <li>
+
+ <a href="../docs/quick-start.html" title="Quick
Start">
+ <span class="none"></span>
+ Quick Start</a>
+ </li>
+
+ <li>
+
+ <a href="../downloads.html" title="Downloads">
+ <span class="none"></span>
+ Downloads</a>
+ </li>
+ <li class="nav-header">Documentaion</li>
+
+ <li>
+
+ <a href="../docs/index.html" title="Latest">
+ <span class="icon-chevron-right"></span>
+ Latest</a>
+ </li>
+
+ <li>
+
+ <a href="../v0.1.0/index.html" title="v0.1.0">
+ <span class="none"></span>
+ v0.1.0</a>
+ </li>
+ <li class="nav-header">Development</li>
+
+ <li>
+
+ <a href="../develop/schedule.html" title="Schedule">
+ <span class="none"></span>
+ Schedule</a>
+ </li>
+
+ <li>
+
+ <a href="../develop/how-contribute.html" title="How
to Contribute">
+ <span class="icon-chevron-right"></span>
+ How to Contribute</a>
+ </li>
+ <li class="nav-header">Community</li>
+
+ <li>
+
+ <a href="../community/source-repository.html"
title="Source Repository">
+ <span class="none"></span>
+ Source Repository</a>
+ </li>
+
+ <li>
+
+ <a href="../community/mail-lists.html"
title="Mailing Lists">
+ <span class="none"></span>
+ Mailing Lists</a>
+ </li>
+
+ <li>
+
+ <a href="../community/issue-tracking.html"
title="Issue Tracking">
+ <span class="none"></span>
+ Issue Tracking</a>
+ </li>
+
+ <li>
+
+ <a href="../community/team-list.html" title="SINGA
Team">
+ <span class="none"></span>
+ SINGA Team</a>
+ </li>
+ <li class="nav-header">External Links</li>
+
+ <li>
+
+ <a href="http://www.apache.org/";
class="externalLink" title="Apache Software Foundation">
+ <span class="none"></span>
+ Apache Software Foundation</a>
+ </li>
+
+ <li>
+
+ <a
href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink"
title="NUS Site">
+ <span class="none"></span>
+ NUS Site</a>
+ </li>
+ </ul>
+
+
+
+ <hr />
+
+ <div id="poweredBy">
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+ <div class="clear"></div>
+
<a href="http://incubator.apache.org";
title="apache-incubator" class="builtBy">
+ <img class="builtBy" alt="Apache Incubator"
src="http://incubator.apache.org/images/egg-logo.png"; />
+ </a>
+ </div>
+ </div>
+ </div>
+
+
+ <div id="bodyColumn" class="span10" >
+
+ <h1>RNN in SINGA</h1>
+<hr />
+<p>Recurrent neural networks (RNN) are widely used for modelling sequential
data, e.g., natural language sentences. In this page, we describe how to
implement a RNN application (or model) using SINGA built-in RNN layers. We will
use the <a class="externalLink"
href="https://github.com/karpathy/char-rnn";>char-rnn modle</a> as an example,
which trains over setences or source code, with each character as an input
unit. Particularly, we will train a RNN using GRU over <a class="externalLink"
href="http://cs.stanford.edu/people/karpathy/char-rnn/";>Linux kernel source
code</a>. After training, we expect to generate meaningful code from the model,
like the one shown by <a class="externalLink"
href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/";>Karpathy</a>.
There is a <a href="rnn.html">vanilla RNN example</a> for language modelling
using user defined RNN layers, which is different to using built-in RNN layers
discribed in this page.</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">/*
+ * If this error is set, we will need anything right after that BSD.
+ */
+static void action_new_function(struct s_stat_info *wb)
+{
+ unsigned long flags;
+ int lel_idx_bit = e->edd, *sys & ~((unsigned long) *FIRST_COMPAT);
+ buf[0] = 0xFFFFFFFF & (bit << 4);
+ min(inc, slist->bytes);
+ printk(KERN_WARNING "Memory allocated %02x/%02x, "
+ "original MLL instead\n"),
+ min(min(multi_run - s->len, max) * num_data_in),
+ frame_pos, sz + first_seg);
+ div_u64_w(val, inb_p);
+ spin_unlock(&disk->queue_lock);
+ mutex_unlock(&s->sock->mutex);
+ mutex_unlock(&func->mutex);
+ return disassemble(info->pending_bh);
+}
+</pre></div></div>
+<div class="section">
+<h2><a name="User_configuration"></a>User configuration</h2>
+<p>The major diffences to the configuration of other models, e.g.,
feed-forward models include,</p>
+
+<ol style="list-style-type: decimal">
+
+<li>the training algorithm should be changed to BPTT (back-propagation through
time).</li>
+
+<li>the layers and their connections should be configured differently.</li>
+</ol>
+<p>The train one batch algorithm can be simply configured as</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">train_one_batch {
+ alg: kBPTT
+}
+</pre></div></div>
+<p>Next, we introduce the configuration of the neural net.</p>
+<p><img src="../images/char-rnn.png" style="width: 550px" alt="" />
+<p><b> Fig.1 - Illustration of the structure of the Char-RNN model</b></p>
+<p>Fig.1 illustrates the net structure of the char-rnn model. The input layer
buffers all training data (the Linux kernel code is about 6MB). For each
iteration, it reads <tt>unroll_len +1</tt> (<tt>unroll_len</tt> is configured
by users) successive characters, e.g., “int a;”, and passes the
first <tt>unroll_len</tt> characters to <tt>OneHotLayer</tt>s (one per layer).
Every <tt>OneHotLayer</tt> converts its character into the one-hot vector
representation. The input layer passes the last <tt>unroll_len</tt> characters
as labels to the <tt>RNNLabelLayer</tt> (the label of the i-th character is the
i+1 character, i.e., the objective is to predict the next character). Each
<tt>GRULayer</tt> receives an one-hot vector and the hidden feature vector from
its precedent layer. After some feature transformation, its own feature vector
is passed to an inner-product layer and its successive <tt>GRULayer</tt>. The
i-th SoftmaxLossLayer measures the cross-entropy loss for predicti
ng the i-th character. According to Karpathy, there could be another stack of
<tt>GRULayer</tt>s connecting the first stack of <tt>GRULayer</tt>s, which
improves the performance if there is enough training data. The layer
configuration is similar to that for other models, e.g., feed-forward models.
The major difference is on the connection configuration.</p>
+<div class="section">
+<h3><a name="Unrolling_length"></a>Unrolling length</h3>
+<p>To model the long dependency, recurrent layers need to be unrolled many
times, denoted as <tt>unroll_len</tt> (i.e., 50). According to our unified
neural net representation, the neural net should have configurations for
<tt>unroll_len</tt> recurrent layers. It is tedious to let users configure
these layers manually. Hence, SINGA makes it a configuration field for each
layer. For example, to unroll the <tt>GRULayer</tt>, users just configure it
as,</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">layer {
+ type: kGRU
+ unroll_len: 50
+}
+</pre></div></div>
+<p>Not only the <tt>GRULayer</tt> is unrolled, other layers like
<tt>InnerProductLayer</tt> and <tt>SoftmaxLossLayer</tt>, are also unrolled. To
simplify the configuration, SINGA provides a <tt>unroll_len</tt> field in the
net configuration, which sets the <tt>unroll_len</tt> of each layer
configuration if the <tt>unroll_len</tt> is not configured explicitly for that
layer. For instance, SINGA would set the <tt>unroll_len</tt> of the
<tt>GRULayer</tt> to 50 implicitly for the following configuration.</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">net {
+ unroll_len: 50
+ layer {
+ type: kCharRNNInput
+ unroll_len: 1 // configure it explicitly
+ }
+ layer {
+ type: kGRU
+ // no configuration for unroll_len
+ }
+ }
+</pre></div></div></div>
+<div class="section">
+<h3><a name="ConnectionType"></a>ConnectionType</h3>
+<p><img src="http://karpathy.github.io/assets/rnn/diags.jpeg"; style="width:
550px" alt="" />
+<p><b> Fig.1 - Different RNN structures from <a class="externalLink"
href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/";>Karpathy</a></b></p>
+<p>There would be many types of connections between layers in RNN models as
shown by Karpathy in Fig.2. For each <tt>srclayer</tt>, there is a
connection_type for it. Taking the i-th <tt>srclayer</tt> as an example, if its
connection type is,</p>
+
+<ul>
+
+<li>kOneToOne, then each unrolled layer is connected with one unrolled layer
from the i-th <tt>srclayer</tt>.</li>
+
+<li>kOneToALL, then each unrolled layer is connected with all unrolled layers
from the i-th <tt>srclayer</tt>.</li>
+</ul></div></div>
+<div class="section">
+<h2><a name="Implementation"></a>Implementation</h2>
+<div class="section">
+<h3><a name="Neural_net_configuration_preprocessing"></a>Neural net
configuration preprocessing</h3>
+<p>User configured neural net is preprocessed to unroll the recurrent layers,
i.e., duplicating the configuration of the <tt>GRULayer</tt>s, renaming the
name of each layer with unrolling index, and re-configuring the
<tt>srclayer</tt> field. After preprocessing, each layer’s name is
changed to <tt><unrolling_index>#<user_configured_name>.</tt>
Consequently, the (unrolled) neural net configuration passed to NeuralNet class
includes all layers and their connections. The NeuralNet class creates and
setup each layer in the same way as for other models. For example, after
partitioning, each layer’s name is changed to
<tt><layer_name>@<partition_index></tt>. One difference is that it
has some special code for sharing Param data and grad Blobs for layers unrolled
from the same original layer.</p>
+<p>Users can visualize the neural net structure using the Python script
<tt>tool/graph.py</tt> and the files in <i>WORKSPACE/visualization/</i>. For
example, after the training program is started,</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">python tool/graph.py
examples/char-rnn/visualization/train_net.json
+</pre></div></div>
+<p>The generated image file is shown in Fig.3 for <tt>unroll_len=5</tt>,</p>
+<p><img src="../images/char-rnn-net.jpg" style="width: 550px" alt="" />
+<p><b> Fig.3 - Net structure generated by SINGA</b></p></div>
+<div class="section">
+<h3><a name="BPTTWorker"></a>BPTTWorker</h3>
+<p>The BPTT (back-propagation through time) algorithm is typically used to
compute gradients of the objective loss w.r.t. parameters for RNN models. It
forwards propagates through all unrolled layers (i.e., timepoints) to compute
features of each layer, and backwards propagates to compute gradients of
parameters. It is the same as the BP algorithm for feed-forward models if the
recurrent layers are unrolled infinite times. In practice, due to the
constraint of memory, the truncated BPTT is widely used. It unrolls the
recurrent layers a fixed (truncated) times (controlled by <tt>unroll_len</tt>).
In SINGA, a BPTTWorker is provided to run the truncated BPTT algorithm for each
mini-batch (i.e., iteration). The pseudo code is</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">BPTTWorker::Forward(phase, net) {
+ for each layer in net
+ if layer.unroll_index() == 0
+ Get(layer.params()); // fetch params values from servers
+ srclayers = layer.srclayer();
+ if phase & kTest
+ srclayers.push_back(net->GetConextLayer(layer))
+ layer.ComputeFeature(phase, srclayers)
+}
+
+BPTTWorker::Backward(phase, net) {
+ for each layer in reverse(net.layers())
+ layer.ComputeGradient(layer.srclayers())
+ if layer.unroll_index() == 0
+ Update(layer.params()); // send params gradients to servers
+}
+</pre></div></div>
+<p>The testing phase is processed specially. Because the test phase may sample
a long sequence of data (e.g., sampling a piece of Linux kernel code), which
requires many unrolled layers (e.g., more than 1000 characters/layers). But we
cannot unroll the recurrent layers too many times due to memory constraint. The
special line add the 0-th unrolled layer as one of its own source layer.
Consequently, it dynamically adds a recurrent connection to the recurrent layer
(e.g., GRULayer). Then we can sample from the model for infinite times. Taking
the char-rnn model as an example, the test job can be configured as</p>
+
+<div class="source">
+<div class="source"><pre class="prettyprint">test_steps: 10000
+train_one_batch {
+ Alg: kBPTT
+}
+net {
+ // do not set the unroll_len
+ layer {
+ // do not set the unroll_len
+ }
+ …
+}
+</pre></div></div>
+<p>The instructions for <a href="test.html">running test</a> is the same for
feed-forward models.</p></div></div>
+ </div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container-fluid">
+ <div class="row-fluid">
+
+<p>Copyright © 2015 The Apache Software Foundation. All rights reserved.
Apache Singa, Apache, the Apache feather logo, and the Apache Singa project
logos are trademarks of The Apache Software Foundation. All other marks
mentioned may be trademarks or registered trademarks of their respective
owners.</p>
+ </div>
+
+
+ </div>
+ </footer>
+ </body>
+</html>
Modified: websites/staging/singa/trunk/content/docs/index.html
==============================================================================
--- websites/staging/singa/trunk/content/docs/index.html (original)
+++ websites/staging/singa/trunk/content/docs/index.html Mon Jan 11 10:49:21
2016
@@ -429,7 +429,9 @@
<li><a href="rbm.html">RBM + Auto-encoder</a></li>
-<li><a href="rnn.html">RNN</a></li>
+<li><a href="rnn.html">Vanilla RNN for language modelling</a></li>
+
+<li><a href="general-rnn.html">Char-RNN</a></li>
</ul></li>
</ul>
<hr />
Added: websites/staging/singa/trunk/content/images/char-rnn-net.jpg
==============================================================================
Binary file - no diff available.
Propchange: websites/staging/singa/trunk/content/images/char-rnn-net.jpg
------------------------------------------------------------------------------
svn:mime-type = image/jpeg
Added: websites/staging/singa/trunk/content/images/char-rnn.png
==============================================================================
Binary file - no diff available.
Propchange: websites/staging/singa/trunk/content/images/char-rnn.png
------------------------------------------------------------------------------
svn:mime-type = image/png
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}
![http://karpathy.github.io/assets/rnn/diags.jpeg"](http://karpathy.github.io/assets/rnn/diags.jpeg"){kind=link}
