Added: websites/staging/singa/trunk/content/docs/neural-net.html ============================================================================== --- websites/staging/singa/trunk/content/docs/neural-net.html (added) +++ websites/staging/singa/trunk/content/docs/neural-net.html Wed Sep 2 10:31:57 2015 @@ -0,0 +1,680 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2015-09-02 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20150902" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Neural Net</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../index.html" title="Welcome">Welcome</a> +</li> + + <li> <a href="../introduction.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../quick-start.html" title="Quick Start">Quick Start</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/installation.html" title="Installation">Installation</a> +</li> + + <li class="dropdown-submenu"> + <a href="../docs/programmer-guide.html" title="Programmer Guide">Programmer Guide</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/model-config.html" title="Model Configuration">Model Configuration</a> +</li> + <li> <a href="../docs/neuralnet.html" title="Neural Network">Neural Network</a> +</li> + <li> <a href="../docs/layer.html" title="Layer">Layer</a> +</li> + <li> <a href="../docs/param.html" title="Param">Param</a> +</li> + </ul> + </li> + + <li class="dropdown-submenu"> + <a href="../docs/distributed-training.html" title="Distributed Training">Distributed Training</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/architecture.html" title="System Architecture">System Architecture</a> +</li> + <li> <a href="../docs/frameworks.html" title="Frameworks">Frameworks</a> +</li> + <li> <a href="../docs/communication.html" title="Communication">Communication</a> +</li> + </ul> + </li> + + <li> <a href="../docs/data.html" title="Data Preparation">Data Preparation</a> +</li> + + <li> <a href="../docs/checkpoint.html" title="Checkpoint">Checkpoint</a> +</li> + + <li> <a href="../docs/examples.html" title="Examples">Examples</a> +</li> + + <li> <a href="../docs/debug.html" title="Debug">Debug</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS School of Computing">NUS School of Computing</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Neural Net</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../index.html" title="Welcome"> + <span class="none"></span> + Welcome</a> + </li> + + <li> + + <a href="../introduction.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/installation.html" title="Installation"> + <span class="none"></span> + Installation</a> + </li> + + <li> + + <a href="../docs/programmer-guide.html" title="Programmer Guide"> + <span class="icon-chevron-down"></span> + Programmer Guide</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/model-config.html" title="Model Configuration"> + <span class="none"></span> + Model Configuration</a> + </li> + + <li> + + <a href="../docs/neuralnet.html" title="Neural Network"> + <span class="none"></span> + Neural Network</a> + </li> + + <li> + + <a href="../docs/layer.html" title="Layer"> + <span class="none"></span> + Layer</a> + </li> + + <li> + + <a href="../docs/param.html" title="Param"> + <span class="none"></span> + Param</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/distributed-training.html" title="Distributed Training"> + <span class="icon-chevron-down"></span> + Distributed Training</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/architecture.html" title="System Architecture"> + <span class="none"></span> + System Architecture</a> + </li> + + <li> + + <a href="../docs/frameworks.html" title="Frameworks"> + <span class="none"></span> + Frameworks</a> + </li> + + <li> + + <a href="../docs/communication.html" title="Communication"> + <span class="none"></span> + Communication</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/data.html" title="Data Preparation"> + <span class="none"></span> + Data Preparation</a> + </li> + + <li> + + <a href="../docs/checkpoint.html" title="Checkpoint"> + <span class="none"></span> + Checkpoint</a> + </li> + + <li> + + <a href="../docs/examples.html" title="Examples"> + <span class="none"></span> + Examples</a> + </li> + + <li> + + <a href="../docs/debug.html" title="Debug"> + <span class="none"></span> + Debug</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-down"></span> + How to Contribute</a> + <ul class="nav nav-list"> + + <li> + + <a href="../develop/contribute-code.html" title="Code"> + <span class="none"></span> + Code</a> + </li> + + <li> + + <a href="../develop/contribute-docs.html" title="Documentation"> + <span class="none"></span> + Documentation</a> + </li> + </ul> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS School of Computing"> + <span class="none"></span> + NUS School of Computing</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Neural Net</h1> +<p><tt>NeuralNet</tt> in SINGA represents an instance of user’s neural net model. As the neural net typically consists of a set of layers, <tt>NeuralNet</tt> comprises a set of unidirectionally connected <a class="externalLink" href="http://singa.incubator.apache.org/docs/layer";>Layer</a>s. This page describes how to convert an user’s neural net into the configuration of <tt>NeuralNet</tt>.</p> +<p><img src="http://singa.incubator.apache.org/assets/image/model-category.png"; align="center" width="200px" alt="" /> <span><b>Figure 1 - Categorization of popular deep learning models.</b></span></p> +<div class="section"> +<h2><a name="Net_structure_configuration"></a>Net structure configuration</h2> +<p>Users configure the <tt>NeuralNet</tt> by listing all layers of the neural net and specifying each layer’s source layer names. Popular deep learning models can be categorized as Figure 1. The subsequent sections give details for each category.</p> +<div class="section"> +<h3><a name="Feed-forward_models"></a>Feed-forward models</h3> + +<div align="left"> +<img src="http://singa.incubator.apache.org/assets/image/mlp-net.png"; align="center" width="200px" alt="" /> +<span><b>Figure 2 - Net structure of a MLP model.</b></span> +</div> +<p>Feed-forward models, e.g., CNN and MLP, can easily get configured as their layer connections are undirected without circles. The configuration for the MLP model shown in Figure 1 is as follows,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">net { + layer { + name : 'data" + type : kData + } + layer { + name : 'image" + type : kImage + srclayer: 'data' + } + layer { + name : 'label" + type : kLabel + srclayer: 'data' + } + layer { + name : 'hidden" + type : kHidden + srclayer: 'image' + } + layer { + name : 'softmax" + type : kSoftmaxLoss + srclayer: 'hidden' + srclayer: 'label' + } +} +</pre></div></div></div> +<div class="section"> +<h3><a name="Energy_models"></a>Energy models</h3> +<p><img src="http://singa.incubator.apache.org/assets/image/rbm-rnn.png"; align="center" width="500px" alt="" /> <span><b>Figure 3 - Convert connections in RBM and RNN.</b></span></p> +<p>For energy models including RBM, DBM, etc., their connections are undirected (i.e., Category B). To represent these models using <tt>NeuralNet</tt>, users can simply replace each connection with two directed connections, as shown in Figure 3a. In other words, for each pair of connected layers, their source layer field should include each other’s name. The full <a class="externalLink" href="http://singa.incubator.apache.org/docs/rbm";>RBM example</a> has detailed neural net configuration for a RBM model, which looks like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">net { + layer { + name : "vis" + type : kVisLayer + param { + name : "w1" + } + srclayer: "hid" + } + layer { + name : "hid" + type : kHidLayer + param { + name : "w2" + share_from: "w1" + } + srclayer: "vis" + } +} +</pre></div></div></div> +<div class="section"> +<h3><a name="RNN_models"></a>RNN models</h3> +<p>For recurrent neural networks (RNN), users can remove the recurrent connections by unrolling the recurrent layer. For example, in Figure 3b, the original layer is unrolled into a new layer with 4 internal layers. In this way, the model is like a normal feed-forward model, thus can be configured similarly. The <a class="externalLink" href="http://singa.incubator.apache.org/docs/rnn}";>RNN example</a> has a full neural net configuration for a RNN model.</p></div></div> +<div class="section"> +<h2><a name="Configuration_for_multiple_nets"></a>Configuration for multiple nets</h2> +<p>Typically, a training job includes three neural nets for training, validation and test phase respectively. The three neural nets share most layers except the data layer, loss layer or output layer, etc.. To avoid redundant configurations for the shared layers, users can uses the <tt>exclude</tt> filed to filter a layer in the neural net, e.g., the following layer will be filtered when creating the testing <tt>NeuralNet</tt>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + ... + exclude : kTest # filter this layer for creating test net +} +</pre></div></div></div> +<div class="section"> +<h2><a name="Neural_net_partitioning"></a>Neural net partitioning</h2> +<p>A neural net can be partitioned in different ways to distribute the training over multiple workers.</p> +<div class="section"> +<h3><a name="Batch_and_feature_dimension"></a>Batch and feature dimension</h3> +<p><img src="http://singa.incubator.apache.org/assets/image/partition_fc.png"; align="center" width="400px" alt="" /> <span><b>Figure 4 - Partitioning of a fully connected layer.</b></span></p> +<p>Every layer’s feature blob is considered a matrix whose rows are feature vectors. Thus, one layer can be split on two dimensions. Partitioning on dimension 0 (also called batch dimension) slices the feature matrix by rows. For instance, if the mini-batch size is 256 and the layer is partitioned into 2 sub-layers, each sub-layer would have 128 feature vectors in its feature blob. Partitioning on this dimension has no effect on the parameters, as every <a class="externalLink" href="http://singa.incubator.apache.org/docs/param";>Param</a> object is replicated in the sub-layers. Partitioning on dimension 1 (also called feature dimension) slices the feature matrix by columns. For example, suppose the original feature vector has 50 units, after partitioning into 2 sub-layers, each sub-layer would have 25 units. This partitioning may result in <a class="externalLink" href="http://singa.incubator.apache.org/docs/param";>Param</a> object being split, as shown in Figure 4. Both the bi as vector and weight matrix are partitioned into two sub-layers.</p></div> +<div class="section"> +<h3><a name="Partitioning_configuration"></a>Partitioning configuration</h3> +<p>There are 4 partitioning schemes, whose configurations are give below,</p> + +<ol style="list-style-type: decimal"> + +<li> +<p>Partitioning each singe layer into sub-layers on batch dimension (see below). It is enabled by configuring the partition dimension of the layer to 0, e.g.,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> # with other fields omitted + layer { + partition_dim: 0 + } +</pre></div></div></li> + +<li> +<p>Partitioning each singe layer into sub-layers on feature dimension (see below). It is enabled by configuring the partition dimension of the layer to 1, e.g.,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> # with other fields omitted + layer { + partition_dim: 1 + } +</pre></div></div></li> + +<li> +<p>Partitioning all layers into different subsets. It is enabled by configuring the location ID of a layer, e.g.,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> # with other fields omitted + layer { + location: 1 + } + layer { + location: 0 + } +</pre></div></div></li> +</ol> + +<ol style="list-style-type: decimal"> + +<li> +<p>Hybrid partitioning of strategy 1, 2 and 3. The hybrid partitioning is useful for large models. An example application is to implement the <a class="externalLink" href="http://arxiv.org/abs/1404.5997";>idea proposed by Alex</a>. Hybrid partitioning is configured like,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> # with other fields omitted + layer { + location: 1 + } + layer { + location: 0 + } + layer { + partition_dim: 0 + location: 0 + } + layer { + partition_dim: 1 + location: 0 + } +</pre></div></div></li> +</ol> +<p>Currently SINGA supports strategy-2 well. Other partitioning strategies are are under test and will be released in later version.</p></div></div> +<div class="section"> +<h2><a name="Parameter_sharing"></a>Parameter sharing</h2> +<p>Parameters can be shared in two cases,</p> + +<ul> + +<li> +<p>sharing parameters among layers via user configuration. For example, the visible layer and hidden layer of a RBM shares the weight matrix, which is configured through the <tt>share_from</tt> field as shown in the above RBM configuration. The configurations must be the same (except name) for shared parameters.</p></li> + +<li> +<p>due to neural net partitioning, some <tt>Param</tt> objects are replicated into different workers, e.g., partitioning one layer on batch dimension. These workers share parameter values. SINGA controls this kind of parameter sharing automatically, users do not need to do any configuration.</p></li> + +<li> +<p>the <tt>NeuralNet</tt> for training and testing (and validation) share most layers , thus share <tt>Param</tt> values.</p></li> +</ul> +<p>If the shared <tt>Param</tt> instances resident in the same process (may in different threads), they use the same chunk of memory space for their values. But they would have different memory spaces for their gradients. In fact, their gradients will be averaged by the <a href="">stub</a> or <a href="">server</a>.</p> +<p>{% comment %}</p></div> +<div class="section"> +<h2><a name="Advanced_user_guide"></a>Advanced user guide</h2> +<div class="section"> +<h3><a name="Creation"></a>Creation</h3> + +<div class="source"> +<div class="source"><pre class="prettyprint">static shared_ptr<NeuralNet> NeuralNet::Create(const NetProto& np, Phase phase, int num); +</pre></div></div> +<p>The above function creates a <tt>NeuralNet</tt> for a given phase, and returns a shared pointer to the <tt>NeuralNet</tt> instance. The phase is in {kTrain, kValidation, kTest}. <tt>num</tt> is used for net partitioning which indicates the number of partitions. Typically, a training job includes three neural nets for training, validation and test phase respectively. The three neural nets share most layers except the data layer, loss layer or output layer, etc.. The <tt>Create</tt> function takes in the full net configuration including layers for training, validation and test. It removes layers for phases other than the specified phase based on the <tt>exclude</tt> field in <a class="externalLink" href="http://singa.incubator.apache.org/docs/layer";>layer configuration</a>:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + ... + exclude : kTest # filter this layer for creating test net +} +</pre></div></div> +<p>The filtered net configuration is passed to the constructor of <tt>NeuralNet</tt>:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">NeuralNet::NeuralNet(NetProto netproto, int npartitions); +</pre></div></div> +<p>The constructor creates a graph representing the net structure firstly in</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions); +</pre></div></div> +<p>Next, it creates a layer for each node and connects layers if their nodes are connected.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions); +</pre></div></div> +<p>Since the <tt>NeuralNet</tt> instance may be shared among multiple workers, the <tt>Create</tt> function returns a shared pointer to the <tt>NeuralNet</tt> instance .</p></div> +<div class="section"> +<h3><a name="Parameter_sharing"></a>Parameter sharing</h3> +<p><tt>Param</tt> sharing is enabled by first sharing the Param configuration (in <tt>NeuralNet::Create</tt>) to create two similar (e.g., the same shape) Param objects, and then calling (in <tt>NeuralNet::CreateNetFromGraph</tt>),</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">void Param::ShareFrom(const Param& from); +</pre></div></div> +<p>It is also possible to share <tt>Param</tt>s of two nets, e.g., sharing parameters of the training net and the test net,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">void NeuralNet:ShareParamsFrom(shared_ptr<NeuralNet> other); +</pre></div></div> +<p>It will call <tt>Param::ShareFrom</tt> for each Param object.</p></div> +<div class="section"> +<h3><a name="Access_functions"></a>Access functions</h3> +<p><tt>NeuralNet</tt> provides a couple of access function to get the layers and params of the net:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">const std::vector<Layer*>& layers() const; +const std::vector<Param*>& params() const ; +Layer* name2layer(string name) const; +Param* paramid2param(int id) const; +</pre></div></div></div> +<div class="section"> +<h3><a name="Partitioning"></a>Partitioning</h3> +<div class="section"> +<h4><a name="Implementation"></a>Implementation</h4> +<p>SINGA partitions the neural net in <tt>CreateGraph</tt> function, which creates one node for each (partitioned) layer. For example, if one layer’s partition dimension is 0 or 1, then it creates <tt>npartition</tt> nodes for it; if the partition dimension is -1, a single node is created, i.e., no partitioning. Each node is assigned a partition (or location) ID. If the original layer is configured with a location ID, then the ID is assigned to each newly created node. These nodes are connected according to the connections of the original layers. Some connection layers will be added automatically. For instance, if two connected sub-layers are located at two different workers, then a pair of bridge layers is inserted to transfer the feature (and gradient) blob between them. When two layers are partitioned on different dimensions, a concatenation layer which concatenates feature rows (or columns) and a slice layer which slices feature rows (or columns) would be inserted. These connection layers help making the network communication and synchronization transparent to the users.</p></div> +<div class="section"> +<h4><a name="Dispatching_partitions_to_workers"></a>Dispatching partitions to workers</h4> +<p>Each (partitioned) layer is assigned a location ID, based on which it is dispatched to one worker. Particularly, the shared pointer to the <tt>NeuralNet</tt> instance is passed to every worker within the same group, but each worker only computes over the layers that have the same partition (or location) ID as the worker’s ID. When every worker computes the gradients of the entire model parameters (strategy-2), we refer to this process as data parallelism. When different workers compute the gradients of different parameters (strategy-3 or strategy-1), we call this process model parallelism. The hybrid partitioning leads to hybrid parallelism where some workers compute the gradients of the same subset of model parameters while other workers compute on different model parameters. For example, to implement the hybrid parallelism in for the <a class="externalLink" href="http://arxiv.org/abs/1404.5997";>DCNN model</a>, we set <tt>partition_dim = 0</tt> for lower layers and <tt>pa rtition_dim = 1</tt> for higher layers.</p> +<p>{% endcomment %}</p></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
Modified: websites/staging/singa/trunk/content/docs/neuralnet-partition.html ============================================================================== --- websites/staging/singa/trunk/content/docs/neuralnet-partition.html (original) +++ websites/staging/singa/trunk/content/docs/neuralnet-partition.html Wed Sep 2 10:31:57 2015 @@ -1,15 +1,15 @@ <!DOCTYPE html> <!-- - | Generated by Apache Maven Doxia at 2015-08-17 + | Generated by Apache Maven Doxia at 2015-09-02 | Rendered using Apache Maven Fluido Skin 1.4 --> <html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> <head> <meta charset="UTF-8" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="Date-Revision-yyyymmdd" content="20150817" /> + <meta name="Date-Revision-yyyymmdd" content="20150902" /> <meta http-equiv="Content-Language" content="en" /> - <title>Apache SINGA – Neural Network Partition</title> + <title>Apache SINGA – Neural Net Partition</title> <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> <link rel="stylesheet" href="../css/site.css" /> <link rel="stylesheet" href="../css/print.css" media="print" /> @@ -189,7 +189,7 @@ Apache SINGA</a> <span class="divider">/</span> </li> - <li class="active ">Neural Network Partition</li> + <li class="active ">Neural Net Partition</li> @@ -425,8 +425,7 @@ <div id="bodyColumn" class="span10" > - <div class="section"> -<h2><a name="Neural_Network_Partition"></a>Neural Network Partition</h2> + <h1>Neural Net Partition</h1> <hr /> <p>The purposes of partitioning neural network is to distribute the partitions onto different working units (e.g., threads or nodes, called workers in this article) and parallelize the processing. Another reason for partition is to handle large neural network which cannot be hold in a single node. For instance, to train models against images with high resolution we need large neural networks (in terms of training parameters).</p> <p>Since <i>Layer</i> is the first class citizen in SIGNA, we do the partition against layers. Specifically, we support partitions at two levels. First, users can configure the location (i.e., worker ID) of each layer. In this way, users assign one worker for each layer. Secondly, for one layer, we can partition its neurons or partition the instances (e.g, images). They are called layer partition and data partition respectively. We illustrate the two types of partitions using an simple convolutional neural network.</p> @@ -436,7 +435,7 @@ <p>The above figure shows the convolutional neural network after partitioning all layers except the DataLayer and ParserLayers, into 3 partitions using data partition. The read layers process 4 images of the batch, the black and blue layers process 2 images respectively. Some helper layers, i.e., SliceLayer, ConcateLayer, BridgeSrcLayer, BridgeDstLayer and SplitLayer, are added automatically by our partition algorithm. Layers of the same color resident in the same worker. There would be data transferring across different workers at the boundary layers (i.e., BridgeSrcLayer and BridgeDstLayer), e.g., between s-slice-mnist-conv1 and d-slice-mnist-conv1.</p> <p><img src="../images/conv-mnist-layerp.png" style="width: 1000px" alt="" /></p> <p>The above figure shows the convolutional neural network after partitioning all layers except the DataLayer and ParserLayers, into 2 partitions using layer partition. We can see that each layer processes all 8 images from the batch. But different partitions process different part of one image. For instance, the layer conv1-00 process only 4 channels. The other 4 channels are processed by conv1-01 which residents in another worker.</p> -<p>Since the partition is done at the layer level, we can apply different partitions for different layers to get a hybrid partition for the whole neural network. Moreover, we can also specify the layer locations to locate different layers to different workers.</p></div> +<p>Since the partition is done at the layer level, we can apply different partitions for different layers to get a hybrid partition for the whole neural network. Moreover, we can also specify the layer locations to locate different layers to different workers.</p> </div> </div> </div> Added: websites/staging/singa/trunk/content/docs/overview.html ============================================================================== --- websites/staging/singa/trunk/content/docs/overview.html (added) +++ websites/staging/singa/trunk/content/docs/overview.html Wed Sep 2 10:31:57 2015 @@ -0,0 +1,477 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2015-09-02 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20150902" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Introduction</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../index.html" title="Welcome">Welcome</a> +</li> + + <li> <a href="../introduction.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../quick-start.html" title="Quick Start">Quick Start</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/installation.html" title="Installation">Installation</a> +</li> + + <li class="dropdown-submenu"> + <a href="../docs/programmer-guide.html" title="Programmer Guide">Programmer Guide</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/model-config.html" title="Model Configuration">Model Configuration</a> +</li> + <li> <a href="../docs/neuralnet.html" title="Neural Network">Neural Network</a> +</li> + <li> <a href="../docs/layer.html" title="Layer">Layer</a> +</li> + <li> <a href="../docs/param.html" title="Param">Param</a> +</li> + </ul> + </li> + + <li class="dropdown-submenu"> + <a href="../docs/distributed-training.html" title="Distributed Training">Distributed Training</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/architecture.html" title="System Architecture">System Architecture</a> +</li> + <li> <a href="../docs/frameworks.html" title="Frameworks">Frameworks</a> +</li> + <li> <a href="../docs/communication.html" title="Communication">Communication</a> +</li> + </ul> + </li> + + <li> <a href="../docs/data.html" title="Data Preparation">Data Preparation</a> +</li> + + <li> <a href="../docs/checkpoint.html" title="Checkpoint">Checkpoint</a> +</li> + + <li> <a href="../docs/examples.html" title="Examples">Examples</a> +</li> + + <li> <a href="../docs/debug.html" title="Debug">Debug</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS School of Computing">NUS School of Computing</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Introduction</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../index.html" title="Welcome"> + <span class="none"></span> + Welcome</a> + </li> + + <li> + + <a href="../introduction.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/installation.html" title="Installation"> + <span class="none"></span> + Installation</a> + </li> + + <li> + + <a href="../docs/programmer-guide.html" title="Programmer Guide"> + <span class="icon-chevron-down"></span> + Programmer Guide</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/model-config.html" title="Model Configuration"> + <span class="none"></span> + Model Configuration</a> + </li> + + <li> + + <a href="../docs/neuralnet.html" title="Neural Network"> + <span class="none"></span> + Neural Network</a> + </li> + + <li> + + <a href="../docs/layer.html" title="Layer"> + <span class="none"></span> + Layer</a> + </li> + + <li> + + <a href="../docs/param.html" title="Param"> + <span class="none"></span> + Param</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/distributed-training.html" title="Distributed Training"> + <span class="icon-chevron-down"></span> + Distributed Training</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/architecture.html" title="System Architecture"> + <span class="none"></span> + System Architecture</a> + </li> + + <li> + + <a href="../docs/frameworks.html" title="Frameworks"> + <span class="none"></span> + Frameworks</a> + </li> + + <li> + + <a href="../docs/communication.html" title="Communication"> + <span class="none"></span> + Communication</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/data.html" title="Data Preparation"> + <span class="none"></span> + Data Preparation</a> + </li> + + <li> + + <a href="../docs/checkpoint.html" title="Checkpoint"> + <span class="none"></span> + Checkpoint</a> + </li> + + <li> + + <a href="../docs/examples.html" title="Examples"> + <span class="none"></span> + Examples</a> + </li> + + <li> + + <a href="../docs/debug.html" title="Debug"> + <span class="none"></span> + Debug</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-down"></span> + How to Contribute</a> + <ul class="nav nav-list"> + + <li> + + <a href="../develop/contribute-code.html" title="Code"> + <span class="none"></span> + Code</a> + </li> + + <li> + + <a href="../develop/contribute-docs.html" title="Documentation"> + <span class="none"></span> + Documentation</a> + </li> + </ul> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS School of Computing"> + <span class="none"></span> + NUS School of Computing</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Introduction</h1> +<p>SINGA is a general distributed deep learning platform for training big deep learning models over large datasets. It is designed with an intuitive programming model based on the layer abstraction. A variety of popular deep learning models are supported, namely feed-forward models including convolutional neural networks (CNN), energy models like restricted Boltzmann machine (RBM), and recurrent neural networks (RNN). Many built-in layers are provided for users. SINGA architecture is sufficiently flexible to run synchronous, asynchronous and hybrid training frameworks. SINGA also supports different neural net partitioning schemes to parallelize the training of large models, namely partitioning on batch dimension, feature dimension or hybrid partitioning.</p> +<div class="section"> +<h2><a name="Goals"></a>Goals</h2> +<p>As a distributed system, the first goal of SINGA is to have good scalability. In other words, SINGA is expected to reduce the total training time to achieve certain accuracy with more computing resources (i.e., machines).</p> +<p>The second goal is to make SINGA easy to use. It is non-trivial for programmers to develop and train models with deep and complex model structures. Distributed training further increases the burden of programmers, e.g., data and model partitioning, and network communication. Hence it is essential to provide an easy to use programming model so that users can implement their deep learning models/algorithms without much awareness of the underlying distributed platform.</p></div> +<div class="section"> +<h2><a name="Principles"></a>Principles</h2> +<p>Scalability is a challenging research problem for distributed deep learning training. SINGA provides a general architecture to exploit the scalability of different training frameworks. Synchronous training frameworks improve the efficiency of one training iteration, and asynchronous training frameworks improve the convergence rate. Given a fixed budget (e.g., cluster size), users can run a hybrid framework that maximizes the scalability by trading off between efficiency and convergence rate.</p> +<p>SINGA comes with a programming model designed based on the layer abstraction, which is intuitive for deep learning models. A variety of popular deep learning models can be expressed and trained using this programming model.</p> +<p>{% comment %} consists of multiple layers. Each layer is associated with a feature transformation function. After going through all layers, the raw input feature (e.g., pixels of images) would be converted into a high-level feature that is easier for tasks like classification. {% endcomment %}</p></div> +<div class="section"> +<h2><a name="System_overview"></a>System overview</h2> +<p><img src="http://singa.incubator.apache.org/assets/image/sgd.png"; align="center" width="400px" alt="" /> <span><b>Figure 1 - SGD flow.</b></span></p> +<p>Training a deep learning model is to find the optimal parameters involved in the transformation functions that generate good features for specific tasks. The goodness of a set of parameters is measured by a loss function, e.g., <a class="externalLink" href="https://en.wikipedia.org/wiki/Cross_entropy";>Cross-Entropy Loss</a>. Since the loss functions are usually non-linear and non-convex, it is difficult to get a closed form solution. Typically, people use the stochastic gradient descent (SGD) algorithm, which randomly initializes the parameters and then iteratively updates them to reduce the loss as shown in Figure 1.</p> +<p><img src="http://singa.incubator.apache.org/assets/image/overview.png"; align="center" width="400px" alt="" /> <span><b>Figure 2 - SINGA overview.</b></span></p> +<p>SGD is used in SINGA to train parameters of deep learning models. The training workload is distributed over worker and server units as shown in Figure 2. In each iteration, every worker calls <i>TrainOneBatch</i> function to compute parameter gradients. <i>TrainOneBatch</i> takes a <i>NeuralNet</i> object representing the neural net, and visits layers of the <i>NeuralNet</i> in certain order. The resultant gradients are sent to the local stub that aggregates the requests and forwards them to corresponding servers for updating. Servers reply to workers with the updated parameters for the next iteration.</p></div> +<div class="section"> +<h2><a name="Job_submission"></a>Job submission</h2> +<p>To submit a job in SINGA (i.e., training a deep learning model), users pass the job configuration to SINGA driver in the <a class="externalLink" href="http://singa.incubator.apache.org/docs/programming-guide";>main function</a>. The job configuration specifies the four major components in Figure 2,</p> + +<ul> + +<li>a <a class="externalLink" href="http://singa.incubator.apache.org/docs/neural-net";>NeuralNet</a> describing the neural net structure with the detailed layer setting and their connections;</li> + +<li>a <a class="externalLink" href="http://singa.incubator.apache.org/docs/train-one-batch";>TrainOneBatch</a> algorithm which is tailored for different model categories;</li> + +<li>an <a class="externalLink" href="http://singa.incubator.apache.org/docs/updater";>Updater</a> defining the protocol for updating parameters at the server side;</li> + +<li>a <a class="externalLink" href="http://singa.incubator.apache.org/docs/distributed-training";>Cluster Topology</a> specifying the distributed architecture of workers and servers.</li> +</ul> +<p>This process is like the job submission in Hadoop, where users configure their jobs in the main function to set the mapper, reducer, etc. In Hadoop, users can configure their jobs with their own (or built-in) mapper and reducer; in SINGA, users can configure their jobs with their own (or built-in) layer, updater, etc.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/docs/param.html ============================================================================== --- websites/staging/singa/trunk/content/docs/param.html (added) +++ websites/staging/singa/trunk/content/docs/param.html Wed Sep 2 10:31:57 2015 @@ -0,0 +1,628 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2015-09-02 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20150902" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Parameters</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../index.html" title="Welcome">Welcome</a> +</li> + + <li> <a href="../introduction.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../quick-start.html" title="Quick Start">Quick Start</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/installation.html" title="Installation">Installation</a> +</li> + + <li class="dropdown-submenu"> + <a href="../docs/programmer-guide.html" title="Programmer Guide">Programmer Guide</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/model-config.html" title="Model Configuration">Model Configuration</a> +</li> + <li> <a href="../docs/neuralnet.html" title="Neural Network">Neural Network</a> +</li> + <li> <a href="../docs/layer.html" title="Layer">Layer</a> +</li> + <li> <a href="../docs/param.html" title="Param">Param</a> +</li> + </ul> + </li> + + <li class="dropdown-submenu"> + <a href="../docs/distributed-training.html" title="Distributed Training">Distributed Training</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/architecture.html" title="System Architecture">System Architecture</a> +</li> + <li> <a href="../docs/frameworks.html" title="Frameworks">Frameworks</a> +</li> + <li> <a href="../docs/communication.html" title="Communication">Communication</a> +</li> + </ul> + </li> + + <li> <a href="../docs/data.html" title="Data Preparation">Data Preparation</a> +</li> + + <li> <a href="../docs/checkpoint.html" title="Checkpoint">Checkpoint</a> +</li> + + <li> <a href="../docs/examples.html" title="Examples">Examples</a> +</li> + + <li> <a href="../docs/debug.html" title="Debug">Debug</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS School of Computing">NUS School of Computing</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Parameters</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../index.html" title="Welcome"> + <span class="none"></span> + Welcome</a> + </li> + + <li> + + <a href="../introduction.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/installation.html" title="Installation"> + <span class="none"></span> + Installation</a> + </li> + + <li> + + <a href="../docs/programmer-guide.html" title="Programmer Guide"> + <span class="icon-chevron-down"></span> + Programmer Guide</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/model-config.html" title="Model Configuration"> + <span class="none"></span> + Model Configuration</a> + </li> + + <li> + + <a href="../docs/neuralnet.html" title="Neural Network"> + <span class="none"></span> + Neural Network</a> + </li> + + <li> + + <a href="../docs/layer.html" title="Layer"> + <span class="none"></span> + Layer</a> + </li> + + <li class="active"> + + <a href="#"><span class="none"></span>Param</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/distributed-training.html" title="Distributed Training"> + <span class="icon-chevron-down"></span> + Distributed Training</a> + <ul class="nav nav-list"> + + <li> + + <a href="../docs/architecture.html" title="System Architecture"> + <span class="none"></span> + System Architecture</a> + </li> + + <li> + + <a href="../docs/frameworks.html" title="Frameworks"> + <span class="none"></span> + Frameworks</a> + </li> + + <li> + + <a href="../docs/communication.html" title="Communication"> + <span class="none"></span> + Communication</a> + </li> + </ul> + </li> + + <li> + + <a href="../docs/data.html" title="Data Preparation"> + <span class="none"></span> + Data Preparation</a> + </li> + + <li> + + <a href="../docs/checkpoint.html" title="Checkpoint"> + <span class="none"></span> + Checkpoint</a> + </li> + + <li> + + <a href="../docs/examples.html" title="Examples"> + <span class="none"></span> + Examples</a> + </li> + + <li> + + <a href="../docs/debug.html" title="Debug"> + <span class="none"></span> + Debug</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-down"></span> + How to Contribute</a> + <ul class="nav nav-list"> + + <li> + + <a href="../develop/contribute-code.html" title="Code"> + <span class="none"></span> + Code</a> + </li> + + <li> + + <a href="../develop/contribute-docs.html" title="Documentation"> + <span class="none"></span> + Documentation</a> + </li> + </ul> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS School of Computing"> + <span class="none"></span> + NUS School of Computing</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Parameters</h1> +<p>A <tt>Param</tt> object in SINGA represents a set of parameters, e.g., a weight matrix or a bias vector. <i>Basic user guide</i> describes how to configure for a <tt>Param</tt> object, and <i>Advanced user guide</i> provides details on implementing users’ parameter initialization methods.</p> +<div class="section"> +<h2><a name="Basic_user_guide"></a>Basic user guide</h2> +<p>The configuration of a Param object is inside a layer configuration, as the <tt>Param</tt> are associated with layers. An example configuration is like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + ... + param { + name : "p1" + init { + type : kConstant + value: 1 + } + } +} +</pre></div></div> +<p>The <a class="externalLink" href="http://singa.incubator.apache.org/docs/overview";>SGD algorithm</a> starts with initializing all parameters according to user specified initialization method (the <tt>init</tt> field). For the above example, all parameters in <tt>Param</tt> “p1” will be initialized to constant value 1. The configuration fields of a Param object is defined in <a class="externalLink" href="http://singa.incubator.apache.org/api/classsinga_1_1ParamProto.html";>ParamProto</a>:</p> + +<ul> + +<li>name, an identifier string. It is an optional field. If not provided, SINGA will generate one based on layer name and its order in the layer.</li> + +<li>init, field for setting initialization methods.</li> + +<li>share_from, name of another <tt>Param</tt> object, from which this <tt>Param</tt> will share configurations and values.</li> + +<li>lr_scale, float value to be multiplied with the learning rate when <a class="externalLink" href="http://singa.incubator.apache.org/docs/updater";>updating the parameters</a></li> + +<li>wd_scale, float value to be multiplied with the weight decay when <a class="externalLink" href="http://singa.incubator.apache.org/docs/updater";>updating the parameters</a></li> +</ul> +<p>There are some other fields that are specific to initialization methods.</p> +<div class="section"> +<h3><a name="Initialization_methods"></a>Initialization methods</h3> +<p>Users can set the <tt>type</tt> of <tt>init</tt> use the following built-in initialization methods,</p> + +<ul> + +<li> +<p><tt>kConst</tt>, set all parameters of the Param object to a constant value</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kConst +value: float # default is 1 +</pre></div></div></li> + +<li> +<p><tt>kGaussian</tt>, initialize the parameters following a Gaussian distribution.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kGaussian +mean: float # mean of the Gaussian distribution, default is 0 +std: float # standard variance, default is 1 +value: float # default 0 +</pre></div></div></li> + +<li> +<p><tt>kUniform</tt>, initialize the parameters following an uniform distribution</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kUniform +low: float # lower boundary, default is -1 +high: float # upper boundary, default is 1 +value: float # default 0 +</pre></div></div></li> + +<li> +<p><tt>kGaussianSqrtFanIn</tt>, initialize <tt>Param</tt> objects with two dimensions (i.e., matrix) using <tt>kGaussian</tt> and then multiple each parameter with <tt>1/sqrt(fan_in)</tt>, where<tt>fan_in</tt> is the number of columns of the matrix.</p></li> + +<li> +<p><tt>kUniformSqrtFanIn</tt>, the same as <tt>kGaussianSqrtFanIn</tt> except that the distribution is an uniform distribution.</p></li> + +<li> +<p><tt>kUniformFanInOut</tt>, initialize matrix <tt>Param</tt> objects using <tt>kUniform</tt> and then multiple each parameter with <tt>sqrt(6/(fan_in + fan_out))</tt>, where<tt>fan_in + + fan_out</tt> sums up the number of columns and rows of the matrix.</p></li> +</ul> +<p>For all above initialization methods except <tt>kConst</tt>, if their <tt>value</tt> is not 1, every parameter will be multiplied with <tt>value</tt>. Users can also implement their own initialization method following the <i>Advanced user guide</i>.</p></div></div> +<div class="section"> +<h2><a name="Advanced_user_guide"></a>Advanced user guide</h2> +<p>This sections describes the details on implementing new parameter initialization methods.</p> +<div class="section"> +<h3><a name="Base_ParamGenerator"></a>Base ParamGenerator</h3> +<p>All initialization methods are implemented as subclasses of the base <tt>ParamGenerator</tt> class.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class ParamGenerator { + public: + virtual void Init(const ParamGenProto&); + void Fill(Param*); + + protected: + ParamGenProto proto_; +}; +</pre></div></div> +<p>Configurations of the initialization method is in <tt>ParamGenProto</tt>. The <tt>Fill</tt> function fills the <tt>Param</tt> object (passed in as an argument).</p></div> +<div class="section"> +<h3><a name="New_ParamGenerator_subclass"></a>New ParamGenerator subclass</h3> +<p>Similar to implement a new Layer subclass, users can define a configuration protocol message,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in user.proto +message FooParamProto { + optional int32 x = 1; +} +extend ParamGenProto { + optional FooParamProto fooparam_conf =101; +} +</pre></div></div> +<p>The configuration of <tt>Param</tt> would be</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">param { + ... + init { + user_type: 'FooParam" # must use user_type for user defined methods + [fooparam_conf] { # must use brackets for configuring user defined messages + x: 10 + } + } +} +</pre></div></div> +<p>The subclass could be declared as,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class FooParamGen : public ParamGenerator { + public: + void Fill(Param*) override; +}; +</pre></div></div> +<p>Users can access the configuration fields in <tt>Fill</tt> by</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">int x = proto_.GetExtension(fooparam_conf).x(); +</pre></div></div> +<p>To use the new initialization method, users need to register it in the <a class="externalLink" href="http://singa.incubator.apache.org/docs/programming-guide";>main function</a>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">driver.RegisterParamGenerator<FooParamGen>("FooParam") # must be consistent with the user_type in configuration +</pre></div></div> +<p>{% comment %}</p></div> +<div class="section"> +<h3><a name="Base_Param_class"></a>Base Param class</h3></div> +<div class="section"> +<h3><a name="Members"></a>Members</h3> + +<div class="source"> +<div class="source"><pre class="prettyprint">int local_version_; +int slice_start_; +vector<int> slice_offset_, slice_size_; + +shared_ptr<Blob<float>> data_; +Blob<float> grad_; +ParamProto proto_; +</pre></div></div> +<p>Each Param object has a local version and a global version (inside the data Blob). These two versions are used for synchronization. If multiple Param objects share the same values, they would have the same <tt>data_</tt> field. Consequently, their global version is the same. The global version is updated by <a class="externalLink" href="http://singa.incubator.apache.org/docs/communication";>the stub thread</a>. The local version is updated in <tt>Worker::Update</tt> function which assigns the global version to the local version. The <tt>Worker::Collect</tt> function is blocked until the global version is larger than the local version, i.e., when <tt>data_</tt> is updated. In this way, we synchronize workers sharing parameters.</p> +<p>In Deep learning models, some Param objects are 100 times larger than others. To ensure the load-balance among servers, SINGA slices large Param objects. The slicing information is recorded by <tt>slice_*</tt>. Each slice is assigned a unique ID starting from 0. <tt>slice_start_</tt> is the ID of the first slice of this Param object. <tt>slice_offset_[i]</tt> is the offset of the i-th slice in this Param object. <tt>slice_size_[i]</tt> is the size of the i-th slice. These slice information is used to create messages for transferring parameter values or gradients to different servers.</p> +<p>Each Param object has a <tt>grad_</tt> field for gradients. Param objects do not share this Blob although they may share <tt>data_</tt>. Because each layer containing a Param object would contribute gradients. E.g., in RNN, the recurrent layers share parameters values, and the gradients used for updating are averaged from all recurrent these recurrent layers. In SINGA, the [stub thread] will aggregate local gradients for the same Param object. The server will do a global aggregation of gradients for the same Param object.</p> +<p>The <tt>proto_</tt> field has some meta information, e.g., name and ID. It also has a field called <tt>owner</tt> which is the ID of the Param object that shares parameter values with others.</p></div> +<div class="section"> +<h3><a name="Functions"></a>Functions</h3> +<p>The base Param class implements two sets of functions,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">virtual void InitValues(int version = 0); // initialize values according to `init_method` +void ShareFrom(const Param& other); // share `data_` from `other` Param +-------------- +virtual Msg* GenGetMsg(bool copy, int slice_idx); +virtual Msg* GenPutMsg(bool copy, int slice_idx); +... // other message related functions. +</pre></div></div> +<p>Besides the functions for processing the parameter values, there is a set of functions for generating and parsing messages. These messages are for transferring parameter values or gradients between workers and servers. Each message corresponds to one Param slice. If <tt>copy</tt> is false, it means the receiver of this message is in the same process as the sender. In such case, only pointers to the memory of parameter value (or gradient) are wrapped in the message; otherwise, the parameter values (or gradients) should be copied into the message.</p></div></div> +<div class="section"> +<h2><a name="Implementing_Param_subclass"></a>Implementing Param subclass</h2> +<p>Users can extend the base Param class to implement their own parameter initialization methods and message transferring protocols. Similar to implementing a new Layer subclasses, users can create google protocol buffer messages for configuring the Param subclass. The subclass, denoted as FooParam should be registered in main.cc,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">dirver.RegisterParam<FooParam>(kFooParam); // kFooParam should be different to 0, which is for the base Param type +</pre></div></div> + +<ul> + +<li>type, an integer representing the <tt>Param</tt> type. Currently SINGA provides one <tt>Param</tt> implementation with type 0 (the default type). If users want to use their own Param implementation, they should extend the base Param class and configure this field with <tt>kUserParam</tt></li> +</ul> +<p>{% endcomment %}</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/model-category.png"](http://singa.incubator.apache.org/assets/image/model-category.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/mlp-net.png"](http://singa.incubator.apache.org/assets/image/mlp-net.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/rbm-rnn.png"](http://singa.incubator.apache.org/assets/image/rbm-rnn.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/partition_fc.png"](http://singa.incubator.apache.org/assets/image/partition_fc.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/sgd.png"](http://singa.incubator.apache.org/assets/image/sgd.png"){kind=link}
![http://singa.incubator.apache.org/assets/image/overview.png"](http://singa.incubator.apache.org/assets/image/overview.png"){kind=link}