Added: websites/staging/singa/trunk/content/v0.2.0/docker.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/docker.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/docker.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,491 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Building SINGA Docker container</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Building SINGA Docker container</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Building SINGA Docker container</h1> +<p>This guide explains how to set up a development environment for SINGA using Docker. It requires only Docker to be installed. The resulting image contains the complete working environment for SINGA. The image can then be used to set up cluster environment over one or multiple physical nodes. </p> + +<ol style="list-style-type: decimal"> + +<li><a href="#build_base">Build SINGA base</a></li> + +<li><a href="#build_mesos">Build SINGA with Mesos and Hadoop</a></li> + +<li><a href="#pre_built">Pre-built images</a></li> + +<li><a href="#launch_stand_alone">Launch and stop SINGA (stand alone mode)</a></li> + +<li><a href="#launch_pseudo">Launch pseudo-distributed SINGA on one node</a></li> + +<li><a href="#launch_distributed">Launch fully distributed SINGA on multiple nodes</a></li> +</ol> +<hr /> +<p><a name="build_base"></a></p> +<div class="section"> +<div class="section"> +<div class="section"> +<h4><a name="Build_SINGA_base_image"></a>Build SINGA base image</h4> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ cd tool/docker/singa +$ sudo docker build -t singa/base . +$ sudo docker images +REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE +singa/base latest XXXX XXX 2.01 GB +</pre></div></div> +<p>The result is the image containing a built version of SINGA. </p> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/images_base.png"; alt="singa/base" /></p> +<p><i>Figure 1. singa/base Docker image, containing library dependencies and SINGA built from source.</i></p> +<hr /> +<p><a name="build_mesos"></a></p></div> +<div class="section"> +<h4><a name="Build_SINGA_with_Mesos_and_Hadoop"></a>Build SINGA with Mesos and Hadoop</h4> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ cd tool/docker/mesos +$ sudo docker build -t singa/mesos . +$ sudo docker images +REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE +singa/mesos latest XXXX XXX 4.935 GB +</pre></div></div> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/images_mesos.png#1"; alt="singa/mesos" /></p> +<p><i>Figure 2. singa/mesos Docker image, containing Hadoop and Mesos built on top of SINGA. The default namenode address for Hadoop is <tt>node0:9000</tt></i></p> +<p><b>Notes</b> A common failure observed during the build process is caused by network failure occuring when downloading dependencies. Simply re-run the build command. </p> +<hr /> +<p><a name="pre_built"></a></p></div> +<div class="section"> +<h4><a name="Pre-built_images_on_epiC_cluster"></a>Pre-built images on epiC cluster</h4> +<p>For users with access to the <tt>epiC</tt> cluster, there are pre-built and loaded Docker images at the following nodes:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> ciidaa-c18 + ciidaa-c19 +</pre></div></div> +<p>The available images at those nodes are:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">REPOSITORY TAG IMAGE ID CREATED VIRTUAL SIZE +singa/base latest XXXX XXX 2.01 GB +singa/mesos latest XXXX XXX 4.935 GB +weaveworks/weaveexec 1.1.1 XXXX 11 days ago 57.8 MB +weaveworks/weave 1.1.1 XXXX 11 days ago 17.56 MB +</pre></div></div> +<hr /> +<p><a name="launch_stand_alone"></a></p></div> +<div class="section"> +<h4><a name="Launch_and_stop_SINGA_in_stand-alone_mode"></a>Launch and stop SINGA in stand-alone mode</h4> +<p>To launch a test environment for a single-node SINGA training, simply start a container from <tt>singa/base</tt> image. The following starts a container called <tt>XYZ</tt>, then launches a shell in the container: </p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ sudo docker run -dt --name XYZ singa/base /usr/bin/supervisord +$ sudo docker exec -it XYZ /bin/bash +</pre></div></div> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/images_standalone.png#1"; alt="Nothing" /></p> +<p><i>Figure 3. Launch SINGA in stand-alone mode: single node training</i></p> +<p>Inside the launched container, the SINGA source directory can be found at <tt>/root/incubator-singa</tt>. </p> +<p><b>Stopping the container</b></p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ sudo docker stop XYZ +$ sudo docker rm ZYZ +</pre></div></div> +<hr /> +<p><a name="launch_pseudo"></a></p></div> +<div class="section"> +<h4><a name="Launch_SINGA_on_pseudo-distributed_mode_single_node"></a>Launch SINGA on pseudo-distributed mode (single node)</h4> +<p>To simulate a distributed environment on a single node, one can repeat the previous step multiple times, each time giving a different name to the container. Network connections between these containers are already supported, thus SINGA instances/nodes in these container can readily communicate with each other. </p> +<p>The previous approach requires the user to start SINGA instances individually at each container. Although there’s a bash script for that, we provide a better way. In particular, multiple containers can be started from <tt>singa/mesos</tt> image which already bundles Mesos and Hadoop with SINGA. Using Mesos makes it easy to launch, stop and monitor the distributed execution from a single container. Figure 4 shows <tt>N+1</tt> containers running concurrently at the local host. </p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ sudo docker run -dt --name node0 singa/mesos /usr/bin/supervisord +$ sudo docker run -dt --name node1 singa/mesos /usr/bin/supervisord +... +</pre></div></div> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/images_pseudo.png#1"; alt="Nothing" /></p> +<p><i>Figure 4. Launch SINGA in pseudo-distributed mode : multiple SINGA nodes over one single machine</i></p> +<p><b>Starting SINGA distributed training</b></p> +<p>Refer to the <a href="mesos.html">Mesos guide</a> for details of how to start training with multiple SINGA instances. </p> +<p><b>Important:</b> the container that assumes the role of Hadoop’s namenode (and often Mesos’s and Zookeeper’s mater node as well) <b>must</b> be named <tt>node0</tt>. Otherwise, the user must log in to individual containers and change the Hadoop configuration separately. </p> +<hr /> +<p><a name="launch_distributed"></a></p></div> +<div class="section"> +<h4><a name="Launch_SINGA_on_fully_distributed_mode_multiple_nodes"></a>Launch SINGA on fully distributed mode (multiple nodes)</h4> +<p>The previous section has explained how to start a distributed environment on a single node. But running many containers on one node does not scale. When there are multiple physical hosts available, it is better to distribute the containers over them. </p> +<p>The only extra requirement for the fully distributed mode, as compared with the pseudo distributed mode, is that the containers from different hosts are able to transparently communicate with each other. In the pseudo distributed mode, the local docker engine takes care of such communication. Here, we rely on <a class="externalLink" href="http://weave.works/guides/weave-docker-ubuntu-simple.html";>Weave</a> to make the communication transparent. The resulting architecture is shown below. </p> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/images_full.png#1"; alt="Nothing" /></p> +<p><i>Figure 5. Launch SINGA in fully distributed mode: multiple SINGA nodes over multiple machines</i></p> +<p><b>Install Weave at all hosts</b></p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ curl -L git.io/weave -o /usr/local/bin/weave +$ chmod a+x /usr/local/bin/weave +</pre></div></div> +<p><b>Starting Weave</b></p> +<p>Suppose <tt>node0</tt> will be launched at host with IP <tt>111.222.111.222</tt>.</p> + +<ul> + +<li> +<p>At host <tt>111.222.111.222</tt>:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> $ weave launch + $ eval "$(weave env)" //if there's error, do `sudo -s` and try again +</pre></div></div></li> + +<li> +<p>At other hosts:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> $ weave launch 111.222.111.222 + $ eval "$(weave env)" //if there's error, do `sudo -s` and try again +</pre></div></div></li> +</ul> +<p><b>Starting containers</b></p> +<p>The user logs in to each host and starts the container (same as in <a href="#launch_pseudo">pseudo-distributed</a> mode). Note that container acting as the head node of the cluster must be named <tt>node0</tt> (and be running at the host with IP <tt>111.222.111.222</tt>, for example). </p> +<p><b><i>Important</i>:</b> when there are other containers sharing the same host as <tt>node0</tt>, say <tt>node1</tt> and <tt>node2</tt> for example, there’re additional changes to be made to <tt>node1</tt> and <tt>node2</tt>. Particularly, log in to each container and edit <tt>/etc/hosts</tt> file:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># modified by weave +... +X.Y.Z node0 node0.bridge //<- REMOVE this line +.. +</pre></div></div> +<p>This is to ensure that name resolutions (of <tt>node0</tt>’s address) from <tt>node1</tt> and <tt>node2</tt> are correct. By default, containers of the same host resolves each other’s addresses via the Docker bridge. Instead, we want they to use addressed given by Weave. </p> +<p><b>Starting SINGA distributed training</b></p> +<p>Refer to the <a href="mesos.html">Mesos guide</a> for details of how to start training with multiple SINGA instances. </p></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
Added: websites/staging/singa/trunk/content/v0.2.0/examples.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/examples.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/examples.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,371 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Example Models</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Example Models</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Example Models</h1> +<hr /> +<p>Different models are provided as examples to help users get familiar with SINGA. <a href="neural-net.html">Neural Network</a> gives details on the models that are supported by SINGA.</p> +<div class="section"> +<div class="section"> +<h3><a name="Feed-forward_neural_networks"></a>Feed-forward neural networks</h3> + +<ul> + +<li> +<p><a href="mlp.html">MultiLayer Perceptron</a> trained on MNIST dataset for handwritten digits recognition.</p></li> + +<li> +<p><a href="cnn.html">Convolutional Neural Network</a> trained on MNIST and CIFAR10 for image classification.</p></li> + +<li> +<p><a href="rbm.html">Deep Auto-Encoders</a> trained on MNIST for dimensionality</p></li> +</ul></div> +<div class="section"> +<h3><a name="Recurrent_neural_networks_RNN"></a>Recurrent neural networks (RNN)</h3> + +<ul> + +<li><a href="rnn.html">RNN language model</a> trained on plain text for language modelling.</li> +</ul></div> +<div class="section"> +<h3><a name="Energy_models"></a>Energy models</h3> + +<ul> + +<li><a href="rbm.html">RBM</a> used to pre-train deep auto-encoders for dimensionality reduction.</li> +</ul></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/v0.2.0/frameworks.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/frameworks.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/frameworks.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,431 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Distributed Training Framework</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Distributed Training Framework</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Distributed Training Framework</h1> +<hr /> +<div class="section"> +<h2><a name="Cluster_Topology_Configuration"></a>Cluster Topology Configuration</h2> +<p>Here we describe how to configure SINGA’s cluster topology to support different distributed training frameworks. The cluster topology is configured in the <tt>cluster</tt> field in <tt>JobProto</tt>. The <tt>cluster</tt> is of type <tt>ClusterProto</tt>:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">message ClusterProto { + optional int32 nworker_groups = 1; + optional int32 nserver_groups = 2; + optional int32 nworkers_per_group = 3 [default = 1]; + optional int32 nservers_per_group = 4 [default = 1]; + optional int32 nworkers_per_procs = 5 [default = 1]; + optional int32 nservers_per_procs = 6 [default = 1]; + + // servers and workers in different processes? + optional bool server_worker_separate = 20 [default = false]; + + ...... +} +</pre></div></div> +<p>The mostly used fields are as follows:</p> + +<ul> + +<li><tt>nworkers_per_group</tt> and <tt>nworkers_per_procs</tt>: decide the partitioning of worker side ParamShard.</li> + +<li><tt>nservers_per_group</tt> and <tt>nservers_per_procs</tt>: decide the partitioning of server side ParamShard.</li> + +<li><tt>server_worker_separate</tt>: separate servers and workers in different processes.</li> +</ul></div> +<div class="section"> +<h2><a name="Different_Training_Frameworks"></a>Different Training Frameworks</h2> +<p>In SINGA, worker groups run asynchronously and workers within one group run synchronously. Users can leverage this general design to run both <b>synchronous</b> and <b>asynchronous</b> training frameworks. Here we illustrate how to configure popular distributed training frameworks in SINGA.</p> +<p><img src="../images/frameworks.png" style="width: 800px" alt="" /> +<p><b> Fig.1 - Training frameworks in SINGA</b></p> +<div class="section"> +<h3><a name="Sandblaster"></a>Sandblaster</h3> +<p>This is a <b>synchronous</b> framework used by Google Brain. Fig.2(a) shows the Sandblaster framework implemented in SINGA. Its configuration is as follows:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cluster { + nworker_groups: 1 + nserver_groups: 1 + nworkers_per_group: 3 + nservers_per_group: 2 + server_worker_separate: true +} +</pre></div></div> +<p>A single server group is launched to handle all requests from workers. A worker computes on its partition of the model, and only communicates with servers handling related parameters.</p></div> +<div class="section"> +<h3><a name="AllReduce"></a>AllReduce</h3> +<p>This is a <b>synchronous</b> framework used by Baidu’s DeepImage. Fig.2(b) shows the AllReduce framework implemented in SINGA. Its configuration is as follows:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cluster { + nworker_groups: 1 + nserver_groups: 1 + nworkers_per_group: 3 + nservers_per_group: 3 + server_worker_separate: false +} +</pre></div></div> +<p>We bind each worker with a server on the same node, so that each node is responsible for maintaining a partition of parameters and collecting updates from all other nodes.</p></div> +<div class="section"> +<h3><a name="Downpour"></a>Downpour</h3> +<p>This is a <b>asynchronous</b> framework used by Google Brain. Fig.2(c) shows the Downpour framework implemented in SINGA. Its configuration is as follows:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cluster { + nworker_groups: 2 + nserver_groups: 1 + nworkers_per_group: 2 + nservers_per_group: 2 + server_worker_separate: true +} +</pre></div></div> +<p>Similar to the synchronous Sandblaster, all workers send requests to a global server group. We divide workers into several worker groups, each running independently and working on parameters from the last <i>update</i> response.</p></div> +<div class="section"> +<h3><a name="Distributed_Hogwild"></a>Distributed Hogwild</h3> +<p>This is a <b>asynchronous</b> framework used by Caffe. Fig.2(d) shows the Distributed Hogwild framework implemented in SINGA. Its configuration is as follows:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cluster { + nworker_groups: 3 + nserver_groups: 3 + nworkers_per_group: 1 + nservers_per_group: 1 + server_worker_separate: false +} +</pre></div></div> +<p>Each node contains a complete server group and a complete worker group. Parameter updates are done locally, so that communication cost during each training step is minimized. However, the server group must periodically synchronize with neighboring groups to improve the training convergence.</p></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/v0.2.0/general-rnn.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/general-rnn.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/general-rnn.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,473 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – RNN in SINGA</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">RNN in SINGA</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>RNN in SINGA</h1> +<hr /> +<p>Recurrent neural networks (RNN) are widely used for modelling sequential data, e.g., natural language sentences. In this page, we describe how to implement a RNN application (or model) using SINGA built-in RNN layers. We will use the <a class="externalLink" href="https://github.com/karpathy/char-rnn";>char-rnn modle</a> as an example, which trains over setences or source code, with each character as an input unit. Particularly, we will train a RNN using GRU over <a class="externalLink" href="http://cs.stanford.edu/people/karpathy/char-rnn/";>Linux kernel source code</a>. After training, we expect to generate meaningful code from the model, like the one shown by <a class="externalLink" href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/";>Karpathy</a>. There is a <a href="rnn.html">vanilla RNN example</a> for language modelling using user defined RNN layers, which is different to using built-in RNN layers discribed in this page.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">/* + * If this error is set, we will need anything right after that BSD. + */ +static void action_new_function(struct s_stat_info *wb) +{ + unsigned long flags; + int lel_idx_bit = e->edd, *sys & ~((unsigned long) *FIRST_COMPAT); + buf[0] = 0xFFFFFFFF & (bit << 4); + min(inc, slist->bytes); + printk(KERN_WARNING "Memory allocated %02x/%02x, " + "original MLL instead\n"), + min(min(multi_run - s->len, max) * num_data_in), + frame_pos, sz + first_seg); + div_u64_w(val, inb_p); + spin_unlock(&disk->queue_lock); + mutex_unlock(&s->sock->mutex); + mutex_unlock(&func->mutex); + return disassemble(info->pending_bh); +} +</pre></div></div> +<div class="section"> +<h2><a name="User_configuration"></a>User configuration</h2> +<p>The major diffences to the configuration of other models, e.g., feed-forward models include,</p> + +<ol style="list-style-type: decimal"> + +<li>the training algorithm should be changed to BPTT (back-propagation through time).</li> + +<li>the layers and their connections should be configured differently.</li> +</ol> +<p>The train one batch algorithm can be simply configured as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">train_one_batch { + alg: kBPTT +} +</pre></div></div> +<p>Next, we introduce the configuration of the neural net.</p> +<p><img src="../images/char-rnn.png" style="width: 550px" alt="" /> +<p><b> Fig.1 - Illustration of the structure of the Char-RNN model</b></p> +<p>Fig.1 illustrates the net structure of the char-rnn model. The input layer buffers all training data (the Linux kernel code is about 6MB). For each iteration, it reads <tt>unroll_len +1</tt> (<tt>unroll_len</tt> is configured by users) successive characters, e.g., “int a;”, and passes the first <tt>unroll_len</tt> characters to <tt>OneHotLayer</tt>s (one per layer). Every <tt>OneHotLayer</tt> converts its character into the one-hot vector representation. The input layer passes the last <tt>unroll_len</tt> characters as labels to the <tt>RNNLabelLayer</tt> (the label of the i-th character is the i+1 character, i.e., the objective is to predict the next character). Each <tt>GRULayer</tt> receives an one-hot vector and the hidden feature vector from its precedent layer. After some feature transformation, its own feature vector is passed to an inner-product layer and its successive <tt>GRULayer</tt>. The i-th SoftmaxLossLayer measures the cross-entropy loss for predicti ng the i-th character. According to Karpathy, there could be another stack of <tt>GRULayer</tt>s connecting the first stack of <tt>GRULayer</tt>s, which improves the performance if there is enough training data. The layer configuration is similar to that for other models, e.g., feed-forward models. The major difference is on the connection configuration.</p> +<div class="section"> +<h3><a name="Unrolling_length"></a>Unrolling length</h3> +<p>To model the long dependency, recurrent layers need to be unrolled many times, denoted as <tt>unroll_len</tt> (i.e., 50). According to our unified neural net representation, the neural net should have configurations for <tt>unroll_len</tt> recurrent layers. It is tedious to let users configure these layers manually. Hence, SINGA makes it a configuration field for each layer. For example, to unroll the <tt>GRULayer</tt>, users just configure it as,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + type: kGRU + unroll_len: 50 +} +</pre></div></div> +<p>Not only the <tt>GRULayer</tt> is unrolled, other layers like <tt>InnerProductLayer</tt> and <tt>SoftmaxLossLayer</tt>, are also unrolled. To simplify the configuration, SINGA provides a <tt>unroll_len</tt> field in the net configuration, which sets the <tt>unroll_len</tt> of each layer configuration if the <tt>unroll_len</tt> is not configured explicitly for that layer. For instance, SINGA would set the <tt>unroll_len</tt> of the <tt>GRULayer</tt> to 50 implicitly for the following configuration.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">net { + unroll_len: 50 + layer { + type: kCharRNNInput + unroll_len: 1 // configure it explicitly + } + layer { + type: kGRU + // no configuration for unroll_len + } + } +</pre></div></div></div> +<div class="section"> +<h3><a name="ConnectionType"></a>ConnectionType</h3> +<p><img src="http://karpathy.github.io/assets/rnn/diags.jpeg"; style="width: 550px" alt="" /> +<p><b> Fig.1 - Different RNN structures from <a class="externalLink" href="http://karpathy.github.io/2015/05/21/rnn-effectiveness/";>Karpathy</a></b></p> +<p>There would be many types of connections between layers in RNN models as shown by Karpathy in Fig.2. For each <tt>srclayer</tt>, there is a connection_type for it. Taking the i-th <tt>srclayer</tt> as an example, if its connection type is,</p> + +<ul> + +<li>kOneToOne, then each unrolled layer is connected with one unrolled layer from the i-th <tt>srclayer</tt>.</li> + +<li>kOneToALL, then each unrolled layer is connected with all unrolled layers from the i-th <tt>srclayer</tt>.</li> +</ul></div></div> +<div class="section"> +<h2><a name="Implementation"></a>Implementation</h2> +<div class="section"> +<h3><a name="Neural_net_configuration_preprocessing"></a>Neural net configuration preprocessing</h3> +<p>User configured neural net is preprocessed to unroll the recurrent layers, i.e., duplicating the configuration of the <tt>GRULayer</tt>s, renaming the name of each layer with unrolling index, and re-configuring the <tt>srclayer</tt> field. After preprocessing, each layer’s name is changed to <tt><unrolling_index>#<user_configured_name>.</tt> Consequently, the (unrolled) neural net configuration passed to NeuralNet class includes all layers and their connections. The NeuralNet class creates and setup each layer in the same way as for other models. For example, after partitioning, each layer’s name is changed to <tt><layer_name>@<partition_index></tt>. One difference is that it has some special code for sharing Param data and grad Blobs for layers unrolled from the same original layer.</p> +<p>Users can visualize the neural net structure using the Python script <tt>tool/graph.py</tt> and the files in <i>WORKSPACE/visualization/</i>. For example, after the training program is started,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">python tool/graph.py examples/char-rnn/visualization/train_net.json +</pre></div></div> +<p>The generated image file is shown in Fig.3 for <tt>unroll_len=5</tt>,</p> +<p><img src="../images/char-rnn-net.jpg" style="width: 550px" alt="" /> +<p><b> Fig.3 - Net structure generated by SINGA</b></p></div> +<div class="section"> +<h3><a name="BPTTWorker"></a>BPTTWorker</h3> +<p>The BPTT (back-propagation through time) algorithm is typically used to compute gradients of the objective loss w.r.t. parameters for RNN models. It forwards propagates through all unrolled layers (i.e., timepoints) to compute features of each layer, and backwards propagates to compute gradients of parameters. It is the same as the BP algorithm for feed-forward models if the recurrent layers are unrolled infinite times. In practice, due to the constraint of memory, the truncated BPTT is widely used. It unrolls the recurrent layers a fixed (truncated) times (controlled by <tt>unroll_len</tt>). In SINGA, a BPTTWorker is provided to run the truncated BPTT algorithm for each mini-batch (i.e., iteration). The pseudo code is</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">BPTTWorker::Forward(phase, net) { + for each layer in net + if layer.unroll_index() == 0 + Get(layer.params()); // fetch params values from servers + srclayers = layer.srclayer(); + if phase & kTest + srclayers.push_back(net->GetConextLayer(layer)) + layer.ComputeFeature(phase, srclayers) +} + +BPTTWorker::Backward(phase, net) { + for each layer in reverse(net.layers()) + layer.ComputeGradient(layer.srclayers()) + if layer.unroll_index() == 0 + Update(layer.params()); // send params gradients to servers +} +</pre></div></div> +<p>The testing phase is processed specially. Because the test phase may sample a long sequence of data (e.g., sampling a piece of Linux kernel code), which requires many unrolled layers (e.g., more than 1000 characters/layers). But we cannot unroll the recurrent layers too many times due to memory constraint. The special line add the 0-th unrolled layer as one of its own source layer. Consequently, it dynamically adds a recurrent connection to the recurrent layer (e.g., GRULayer). Then we can sample from the model for infinite times. Taking the char-rnn model as an example, the test job can be configured as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">test_steps: 10000 +train_one_batch { + Alg: kBPTT +} +net { + // do not set the unroll_len + layer { + // do not set the unroll_len + } + … +} +</pre></div></div> +<p>The instructions for <a href="test.html">running test</a> is the same for feed-forward models.</p></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/images_base.png"](http://www.comp.nus.edu.sg/~dinhtta/files/images_base.png"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/images_mesos.png#1"](http://www.comp.nus.edu.sg/~dinhtta/files/images_mesos.png#1"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/images_standalone.png#1"](http://www.comp.nus.edu.sg/~dinhtta/files/images_standalone.png#1"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/images_pseudo.png#1"](http://www.comp.nus.edu.sg/~dinhtta/files/images_pseudo.png#1"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/images_full.png#1"](http://www.comp.nus.edu.sg/~dinhtta/files/images_full.png#1"){kind=link}
![http://karpathy.github.io/assets/rnn/diags.jpeg"](http://karpathy.github.io/assets/rnn/diags.jpeg"){kind=link}