Added: incubator/singa/site/trunk/en/docs/net.html URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/en/docs/net.html?rev=1858059&view=auto ============================================================================== --- incubator/singa/site/trunk/en/docs/net.html (added) +++ incubator/singa/site/trunk/en/docs/net.html Wed Apr 24 14:57:35 2019 @@ -0,0 +1,558 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>FeedForward Net — incubator-singa 1.1.0 documentation</title> + + + + + + + + + <script type="text/javascript" src="../_static/js/modernizr.min.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + <script type="text/javascript" src="../_static/language_data.js"></script> + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="Initializer" href="initializer.html" /> + <link rel="prev" title="Layer" href="layer.html" /> + <link href="../_static/style.css" rel="stylesheet" type="text/css"> + <!--link href="../_static/fontawesome-all.min.css" rel="stylesheet" type="text/css"--> + <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"; integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous"> + <style> + .fa:hover { + opacity: 0.7; + } + .fab:hover { + opacity: 0.7; + } + </style> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search" > + + + + <a href="../index.html" class="icon icon-home"> incubator-singa + + + + + <img src="../_static/singa.png" class="logo" alt="Logo"/> + + </a> + + + + + <div class="version"> + latest + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <ul class="current"> +<li class="toctree-l1 current"><a class="reference internal" href="index.html">Documentation</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="installation.html">Installation</a></li> +<li class="toctree-l2"><a class="reference internal" href="software_stack.html">Software Stack</a></li> +<li class="toctree-l2"><a class="reference internal" href="device.html">Device</a></li> +<li class="toctree-l2"><a class="reference internal" href="tensor.html">Tensor</a></li> +<li class="toctree-l2"><a class="reference internal" href="layer.html">Layer</a></li> +<li class="toctree-l2 current"><a class="current reference internal" href="#">FeedForward Net</a></li> +<li class="toctree-l2"><a class="reference internal" href="initializer.html">Initializer</a></li> +<li class="toctree-l2"><a class="reference internal" href="loss.html">Loss</a></li> +<li class="toctree-l2"><a class="reference internal" href="metric.html">Metric</a></li> +<li class="toctree-l2"><a class="reference internal" href="optimizer.html">Optimizer</a></li> +<li class="toctree-l2"><a class="reference internal" href="autograd.html">Autograd in Singa</a></li> +<li class="toctree-l2"><a class="reference internal" href="data.html">Data</a></li> +<li class="toctree-l2"><a class="reference internal" href="image_tool.html">Image Tool</a></li> +<li class="toctree-l2"><a class="reference internal" href="snapshot.html">Snapshot</a></li> +<li class="toctree-l2"><a class="reference internal" href="converter.html">Caffe Converter</a></li> +<li class="toctree-l2"><a class="reference internal" href="utils.html">Utils</a></li> +<li class="toctree-l2"><a class="reference internal" href="model_zoo/index.html">Model Zoo</a></li> +<li class="toctree-l2"><a class="reference internal" href="security.html">Security</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../downloads.html">Download SINGA</a></li> +</ul> +<p class="caption"><span class="caption-text">Development</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../develop/schedule.html">Development Schedule</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-contribute.html">How to Contribute to SINGA</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-contribute.html#how-to-become-a-singa-committer">How to become a SINGA committer</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/contribute-code.html">How to Contribute Code</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/contribute-docs.html">How to Contribute to Documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-to-release.html">How to prepare a release</a></li> +</ul> +<p class="caption"><span class="caption-text">Community</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../community/source-repository.html">Source Repository</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/mail-lists.html">Project Mailing Lists</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/issue-tracking.html">Issue Tracking</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/team-list.html">The SINGA Team</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">incubator-singa</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">Documentation</a> »</li> + + <li>FeedForward Net</li> + + + <li class="wy-breadcrumbs-aside"> + + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> + <div itemprop="articleBody"> + + <div class="section" id="module-singa.net"> +<span id="feedforward-net"></span><h1>FeedForward Net<a class="headerlink" href="#module-singa.net" title="Permalink to this headline">¶</a></h1> +<p>Nerual net class for constructing the nets using layers and providing access +functions for net info, e.g., parameters.</p> +<p>Example usages:</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">singa</span> <span class="k">import</span> <span class="n">net</span> <span class="k">as</span> <span class="n">ffnet</span> +<span class="kn">from</span> <span class="nn">singa</span> <span class="k">import</span> <span class="n">metric</span> +<span class="kn">from</span> <span class="nn">singa</span> <span class="k">import</span> <span class="n">loss</span> +<span class="kn">from</span> <span class="nn">singa</span> <span class="k">import</span> <span class="n">layer</span> +<span class="kn">from</span> <span class="nn">singa</span> <span class="k">import</span> <span class="n">device</span> + +<span class="c1"># create net and add layers</span> +<span class="n">net</span> <span class="o">=</span> <span class="n">ffnet</span><span class="o">.</span><span class="n">FeedForwardNet</span><span class="p">(</span><span class="n">loss</span><span class="o">.</span><span class="n">SoftmaxCrossEntropy</span><span class="p">(),</span> <span class="n">metric</span><span class="o">.</span><span class="n">Accuracy</span><span class="p">())</span> +<span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layer</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="s1">'conv1'</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">input_sample_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span><span class="mi">32</span><span class="p">,</span><span class="mi">32</span><span class="p">,)))</span> +<span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layer</span><span class="o">.</span><span class="n">Activation</span><span class="p">(</span><span class="s1">'relu1'</span><span class="p">))</span> +<span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layer</span><span class="o">.</span><span class="n">MaxPooling2D</span><span class="p">(</span><span class="s1">'pool1'</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span> +<span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layer</span><span class="o">.</span><span class="n">Flatten</span><span class="p">(</span><span class="s1">'flat'</span><span class="p">))</span> +<span class="n">net</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layer</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="s1">'dense'</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span> + +<span class="c1"># init parameters</span> +<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">net</span><span class="o">.</span><span class="n">param_values</span><span class="p">():</span> + <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">p</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> + <span class="n">p</span><span class="o">.</span><span class="n">set_value</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> + <span class="k">else</span><span class="p">:</span> + <span class="n">p</span><span class="o">.</span><span class="n">gaussian</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.01</span><span class="p">)</span> + +<span class="c1"># move net onto gpu</span> +<span class="n">dev</span> <span class="o">=</span> <span class="n">device</span><span class="o">.</span><span class="n">create_cuda_gpu</span><span class="p">()</span> +<span class="n">net</span><span class="o">.</span><span class="n">to_device</span><span class="p">(</span><span class="n">dev</span><span class="p">)</span> + +<span class="c1"># training (skipped)</span> + +<span class="c1"># do prediction after training</span> +<span class="n">x</span> <span class="o">=</span> <span class="n">tensor</span><span class="o">.</span><span class="n">Tensor</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">),</span> <span class="n">dev</span><span class="p">)</span> +<span class="n">x</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span> +<span class="n">y</span> <span class="o">=</span> <span class="n">net</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> +<span class="nb">print</span> <span class="n">tensor</span><span class="o">.</span><span class="n">to_numpy</span><span class="p">(</span><span class="n">y</span><span class="p">)</span> +</pre></div> +</div> +<dl class="class"> +<dt id="singa.net.FeedForwardNet"> +<em class="property">class </em><code class="descclassname">singa.net.</code><code class="descname">FeedForwardNet</code><span class="sig-paren">(</span><em>loss=None</em>, <em>metric=None</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet" title="Permalink to this definition">¶</a></dt> +<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p> +<dl class="method"> +<dt id="singa.net.FeedForwardNet.to_device"> +<code class="descname">to_device</code><span class="sig-paren">(</span><em>dev</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.to_device" title="Permalink to this definition">¶</a></dt> +<dd><p>Move the net onto the given device, including +all parameters and intermediate data.</p> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.add"> +<code class="descname">add</code><span class="sig-paren">(</span><em>lyr</em>, <em>src=None</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.add" title="Permalink to this definition">¶</a></dt> +<dd><p>Append a layer into the layer list.</p> +<p>This function will get the sample shape from the src layers to setup the +newly added layer. For the first layer, it is setup outside. The calling +function should ensure the correctness of the layer order. If src is +None, the last layer is the src layer. If there are multiple src layers, +the src is a list of the src layers.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>lyr</strong> (<a class="reference internal" href="layer.html#singa.layer.Layer" title="singa.layer.Layer"><em>Layer</em></a>) â the layer to be added</p></li> +<li><p><strong>src</strong> (<a class="reference internal" href="layer.html#singa.layer.Layer" title="singa.layer.Layer"><em>Layer</em></a>) â the source layer of lyr</p></li> +</ul> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.param_values"> +<code class="descname">param_values</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.param_values" title="Permalink to this definition">¶</a></dt> +<dd><p>Return a list of tensors for all parameters</p> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.param_specs"> +<code class="descname">param_specs</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.param_specs" title="Permalink to this definition">¶</a></dt> +<dd><p>Return a list of ParamSpec for all parameters</p> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.param_names"> +<code class="descname">param_names</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.param_names" title="Permalink to this definition">¶</a></dt> +<dd><p>Return a list for the names of all params</p> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.train"> +<code class="descname">train</code><span class="sig-paren">(</span><em>x</em>, <em>y</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.train" title="Permalink to this definition">¶</a></dt> +<dd><p>Run BP for one iteration. +This method is deprecated. It is only kept for backward compatibility. +The name of this method is confusing since it does not update parameters. +Please use backprob() instead. +The back progagation algorithm computes gradients but it does not train.</p> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.backprob"> +<code class="descname">backprob</code><span class="sig-paren">(</span><em>x</em>, <em>y</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.backprob" title="Permalink to this definition">¶</a></dt> +<dd><p>Run BP for one iteration.</p> +<p>Currently only support nets with a single output layer, and a single +loss objective and metric. +For multiple outputs (with multiple loss/metric), please manually +call forward, compute loss/metric and call backward. backward() is also +more memory efficient than this function.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>x</strong> â input data, a single input Tensor or a dict: layer name -> Tensor</p></li> +<li><p><strong>y</strong> â label data, a single input Tensor.</p></li> +</ul> +</dd> +<dt class="field-even">Returns</dt> +<dd class="field-even"><p>gradients of parameters and the loss and metric values.</p> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.evaluate"> +<code class="descname">evaluate</code><span class="sig-paren">(</span><em>x</em>, <em>y</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.evaluate" title="Permalink to this definition">¶</a></dt> +<dd><p>Evaluate the loss and metric of the given data.</p> +<p>Currently only support nets with a single output layer, and a single +loss objective and metric. +TODO(wangwei) consider multiple loss objectives and metrics.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>x</strong> â input data, a single input Tensor or a dict: layer name -> Tensor</p></li> +<li><p><strong>y</strong> â label data, a single input Tensor.</p></li> +</ul> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.predict"> +<code class="descname">predict</code><span class="sig-paren">(</span><em>x</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.predict" title="Permalink to this definition">¶</a></dt> +<dd><p>Forward the input data through each layer to get the values of the +output layers.</p> +<p>Currently only support nets with a single output layer</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><p><strong>x</strong> â input data, a single input Tensor or a dict: layer name -> Tensor</p> +</dd> +<dt class="field-even">Returns</dt> +<dd class="field-even"><p>a single output tensor as the prediction result.</p> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.topo_sort"> +<code class="descname">topo_sort</code><span class="sig-paren">(</span><em>layers</em>, <em>src_of_layer</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.topo_sort" title="Permalink to this definition">¶</a></dt> +<dd><p>Topology sort of layers.</p> +<p>It would try to preserve the orders of the input layers.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>layers</strong> â a list of layers; the layers from the output of the same +layer (e.g., slice layer) should be added by users in correct +order; This function would not change their order.</p></li> +<li><p><strong>src_of_layer</strong> â a dictionary: src layer name -> a list of src layers</p></li> +</ul> +</dd> +<dt class="field-even">Returns</dt> +<dd class="field-even"><p>A list of ordered layer</p> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.forward"> +<code class="descname">forward</code><span class="sig-paren">(</span><em>flag</em>, <em>x</em>, <em>output=[]</em>, <em>freeze=None</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.forward" title="Permalink to this definition">¶</a></dt> +<dd><p>Forward the input(s) through every layer.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>flag</strong> â True for training; False for evaluation; could also be +model_pb2.kTrain or model_pb2.kEval, or other values for future +use.</p></li> +<li><p><strong>x</strong> â a single SINGA tensor if there is a single input; otherwise, a +dictionary: layer name-> singa tensor, for each layer accepting +input data. Do not associate a layer with input tensor if it is +connected from another layer. For such case, use a Dummy() layer +to accept the input data and connect the dummy layer to this +layer.</p></li> +<li><p><strong>output</strong> (<em>list</em>) â a list of layer names whose output would be returned +in addition to the default output.</p></li> +<li><p><strong>freeze</strong> (<em>str</em>) â layer name, freeze all layers before this layer; flag +is set to false for these layers.</p></li> +</ul> +</dd> +<dt class="field-even">Returns</dt> +<dd class="field-even"><p><dl class="simple"> +<dt>if there is only one output layer and output arg is empty, return</dt><dd><p>the result from the single output layer; otherwise, return a +dictionary: layer name -> output tensor(s)</p> +</dd> +</dl> +</p> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.backward"> +<code class="descname">backward</code><span class="sig-paren">(</span><em>dy</em>, <em>output=[]</em>, <em>freeze=None</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.backward" title="Permalink to this definition">¶</a></dt> +<dd><p>Run back-propagation after forward-propagation.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>dy</strong> â a single tensor if there is a single loss function; otherwise, +a dictionary maps the name of the layer connecting to the loss +function -> gradient from the loss function. Do not associate a +layer with gradient tensor if it is connecting to another layer. +For such case, connect this layer to a Dummy() layer and use the +dummy layer to accept the gradient.</p></li> +<li><p><strong>output</strong> (<em>list</em>) â a list of layer names whose output gradient would be +returned in addition to the param gradient</p></li> +<li><p><strong>freeze</strong> (<em>str</em>) â layer name, stop backward after this layer.</p></li> +</ul> +</dd> +<dt class="field-even">Returns</dt> +<dd class="field-even"><p>a geneartor iterator that generates +(param_names, param_values, param_grads, layer_grads) after +processing each layer h, where the first three lists are for h +and the last item is a dictionary which maps +layer name -> its output gradient tensor(s). At the end of this +function, the key set includes all layers in the output arg.</p> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.save"> +<code class="descname">save</code><span class="sig-paren">(</span><em>f</em>, <em>buffer_size=10</em>, <em>use_pickle=False</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.save" title="Permalink to this definition">¶</a></dt> +<dd><p>Save model parameters using io/snapshot.</p> +<dl class="field-list simple"> +<dt class="field-odd">Parameters</dt> +<dd class="field-odd"><ul class="simple"> +<li><p><strong>f</strong> â file name</p></li> +<li><p><strong>buffer_size</strong> â size (MB) of the IO, default setting is 10MB; Please +make sure it is larger than any single parameter object.</p></li> +<li><p><strong>use_pickle</strong> (<em>Boolean</em>) â if true, it would use pickle for dumping; +otherwise, it would use protobuf for serialization, which uses +less space.</p></li> +</ul> +</dd> +</dl> +</dd></dl> + +<dl class="method"> +<dt id="singa.net.FeedForwardNet.load"> +<code class="descname">load</code><span class="sig-paren">(</span><em>f</em>, <em>buffer_size=10</em>, <em>use_pickle=False</em><span class="sig-paren">)</span><a class="headerlink" href="#singa.net.FeedForwardNet.load" title="Permalink to this definition">¶</a></dt> +<dd><p>Load model parameters using io/snapshot.</p> +<p>Please refer to the argument description in save().</p> +</dd></dl> + +</dd></dl> + +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="initializer.html" class="btn btn-neutral float-right" title="Initializer" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="layer.html" class="btn btn-neutral float-left" title="Layer" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2019 The Apache Software Foundation. All rights reserved. Apache SINGA, Apache, the Apache feather logo, and the Apache SINGA project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners. + + </p> + </div> + Built with <a href="http://sphinx-doc.org/";>Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> + + + + + + +<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> + <span class="rst-current-version" data-toggle="rst-current-version"> + <span class="fa fa-book"> incubator-singa </span> + v: latest + <span class="fa fa-caret-down"></span> + </span> + <div class="rst-other-versions"> + <dl> + <dt>Languages</dt> + <dd><a href="../../en/index.html">English</a></dd> + <dd><a href="../../zh/index.html">ä¸æ</a></dd> + </dl> + <dl> + <dt>Versions</dt> + <dd><a href="http://singa.apache.org/v0.3.0/";>0.3</a></dd> + <dd><a href="http://singa.apache.org/v1.1.0/";>1.1</a></dd> + </dl> + + </div> + <a href="http://incubator.apache.org/";> <img src= "../_static/apache.png" style="background-color:white;"> </a> + + <a href="https://github.com/apache/incubator-singa"; class="fa fa-github" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://aws.amazon.com/marketplace/seller-profile?id=5bcac385-12c4-4802-aec7-351e09b77b4c"; class="fab fa-aws" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://hub.docker.com/r/apache/singa/"; class="fab fa-docker" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.linkedin.com/groups/13550034"; class="fa fa-linkedin" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://twitter.com/ApacheSinga"; class="fa fa-twitter" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.facebook.com/Apache-SINGA-347284219056544/"; class="fa fa-facebook" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.researchgate.net/project/Apache-SINGA"; class="fab fa-researchgate" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + +</div> + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> +</a> + + + + +</body> +</html> \ No newline at end of file
Added: incubator/singa/site/trunk/en/docs/neural-net.html URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/en/docs/neural-net.html?rev=1858059&view=auto ============================================================================== --- incubator/singa/site/trunk/en/docs/neural-net.html (added) +++ incubator/singa/site/trunk/en/docs/neural-net.html Wed Apr 24 14:57:35 2019 @@ -0,0 +1,598 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>Neural Net — incubator-singa 1.1.0 documentation</title> + + + + + + + + + <script type="text/javascript" src="../_static/js/modernizr.min.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + <script type="text/javascript" src="../_static/language_data.js"></script> + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link href="../_static/style.css" rel="stylesheet" type="text/css"> + <!--link href="../_static/fontawesome-all.min.css" rel="stylesheet" type="text/css"--> + <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"; integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous"> + <style> + .fa:hover { + opacity: 0.7; + } + .fab:hover { + opacity: 0.7; + } + </style> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search" > + + + + <a href="../index.html" class="icon icon-home"> incubator-singa + + + + + <img src="../_static/singa.png" class="logo" alt="Logo"/> + + </a> + + + + + <div class="version"> + latest + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <ul> +<li class="toctree-l1"><a class="reference internal" href="index.html">Documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../downloads.html">Download SINGA</a></li> +</ul> +<p class="caption"><span class="caption-text">Development</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../develop/schedule.html">Development Schedule</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-contribute.html">How to Contribute to SINGA</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-contribute.html#how-to-become-a-singa-committer">How to become a SINGA committer</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/contribute-code.html">How to Contribute Code</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/contribute-docs.html">How to Contribute to Documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../develop/how-to-release.html">How to prepare a release</a></li> +</ul> +<p class="caption"><span class="caption-text">Community</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../community/source-repository.html">Source Repository</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/mail-lists.html">Project Mailing Lists</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/issue-tracking.html">Issue Tracking</a></li> +<li class="toctree-l1"><a class="reference internal" href="../community/team-list.html">The SINGA Team</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">incubator-singa</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li>Neural Net</li> + + + <li class="wy-breadcrumbs-aside"> + + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> + <div itemprop="articleBody"> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--><div class="section" id="neural-net"> +<h1>Neural Net<a class="headerlink" href="#neural-net" title="Permalink to this headline">¶</a></h1> +<p><code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> in SINGA represents an instance of userâs neural net model. As the +neural net typically consists of a set of layers, <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> comprises +a set of unidirectionally connected <a class="reference external" href="layer.html">Layer</a>s. +This page describes how to convert an userâs neural net into +the configuration of <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code>.</p> +<img src="../_static/images/model-category.png" align="center" width="200px"/> +<span><strong>Figure 1 - Categorization of popular deep learning models.</strong></span><div class="section" id="net-structure-configuration"> +<h2>Net structure configuration<a class="headerlink" href="#net-structure-configuration" title="Permalink to this headline">¶</a></h2> +<p>Users configure the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> by listing all layers of the neural net and +specifying each layerâs source layer names. Popular deep learning models can be +categorized as Figure 1. The subsequent sections give details for each +category.</p> +<div class="section" id="feed-forward-models"> +<h3>Feed-forward models<a class="headerlink" href="#feed-forward-models" title="Permalink to this headline">¶</a></h3> +<div align = "left"> +<img src="../_static/images/mlp-net.png" align="center" width="200px"/> +<span><strong>Figure 2 - Net structure of a MLP model.</strong></span> +</div><p>Feed-forward models, e.g., CNN and MLP, can easily get configured as their layer +connections are undirected without circles. The +configuration for the MLP model shown in Figure 1 is as follows,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">net</span> <span class="p">{</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s1">'data"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kData</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s1">'image"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kImage</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s1">'data'</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s1">'label"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kLabel</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s1">'data'</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s1">'hidden"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kHidden</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s1">'image'</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s1">'softmax"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kSoftmaxLoss</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s1">'hidden'</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s1">'label'</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="energy-models"> +<h3>Energy models<a class="headerlink" href="#energy-models" title="Permalink to this headline">¶</a></h3> +<img src="../_static/images/rbm-rnn.png" align="center" width="500px"/> +<span><strong>Figure 3 - Convert connections in RBM and RNN.</strong></span><p>For energy models including RBM, DBM, +etc., their connections are undirected (i.e., Category B). To represent these models using +<code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code>, users can simply replace each connection with two directed +connections, as shown in Figure 3a. In other words, for each pair of connected layers, their source +layer field should include each otherâs name. +The full <a class="reference external" href="rbm.html">RBM example</a> has +detailed neural net configuration for a RBM model, which looks like</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">net</span> <span class="p">{</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s2">"vis"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kVisLayer</span> + <span class="n">param</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s2">"w1"</span> + <span class="p">}</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s2">"hid"</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s2">"hid"</span> + <span class="nb">type</span> <span class="p">:</span> <span class="n">kHidLayer</span> + <span class="n">param</span> <span class="p">{</span> + <span class="n">name</span> <span class="p">:</span> <span class="s2">"w2"</span> + <span class="n">share_from</span><span class="p">:</span> <span class="s2">"w1"</span> + <span class="p">}</span> + <span class="n">srclayer</span><span class="p">:</span> <span class="s2">"vis"</span> + <span class="p">}</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="rnn-models"> +<h3>RNN models<a class="headerlink" href="#rnn-models" title="Permalink to this headline">¶</a></h3> +<p>For recurrent neural networks (RNN), users can remove the recurrent connections +by unrolling the recurrent layer. For example, in Figure 3b, the original +layer is unrolled into a new layer with 4 internal layers. In this way, the +model is like a normal feed-forward model, thus can be configured similarly. +The <a class="reference external" href="rnn.html">RNN example</a> has a full neural net +configuration for a RNN model.</p> +</div> +</div> +<div class="section" id="configuration-for-multiple-nets"> +<h2>Configuration for multiple nets<a class="headerlink" href="#configuration-for-multiple-nets" title="Permalink to this headline">¶</a></h2> +<p>Typically, a training job includes three neural nets for +training, validation and test phase respectively. The three neural nets share most +layers except the data layer, loss layer or output layer, etc.. To avoid +redundant configurations for the shared layers, users can uses the <code class="docutils literal notranslate"><span class="pre">exclude</span></code> +filed to filter a layer in the neural net, e.g., the following layer will be +filtered when creating the testing <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code>.</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">layer</span> <span class="p">{</span> + <span class="o">...</span> + <span class="n">exclude</span> <span class="p">:</span> <span class="n">kTest</span> <span class="c1"># filter this layer for creating test net</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +<div class="section" id="neural-net-partitioning"> +<h2>Neural net partitioning<a class="headerlink" href="#neural-net-partitioning" title="Permalink to this headline">¶</a></h2> +<p>A neural net can be partitioned in different ways to distribute the training +over multiple workers.</p> +<div class="section" id="batch-and-feature-dimension"> +<h3>Batch and feature dimension<a class="headerlink" href="#batch-and-feature-dimension" title="Permalink to this headline">¶</a></h3> +<img src="../_static/images/partition_fc.png" align="center" width="400px"/> +<span><strong>Figure 4 - Partitioning of a fully connected layer.</strong></span><p>Every layerâs feature blob is considered a matrix whose rows are feature +vectors. Thus, one layer can be split on two dimensions. Partitioning on +dimension 0 (also called batch dimension) slices the feature matrix by rows. +For instance, if the mini-batch size is 256 and the layer is partitioned into 2 +sub-layers, each sub-layer would have 128 feature vectors in its feature blob. +Partitioning on this dimension has no effect on the parameters, as every +<a class="reference external" href="param.html">Param</a> object is replicated in the sub-layers. Partitioning on dimension +1 (also called feature dimension) slices the feature matrix by columns. For +example, suppose the original feature vector has 50 units, after partitioning +into 2 sub-layers, each sub-layer would have 25 units. This partitioning may +result in <a class="reference external" href="param.html">Param</a> object being split, as shown in +Figure 4. Both the bias vector and weight matrix are +partitioned into two sub-layers.</p> +</div> +<div class="section" id="partitioning-configuration"> +<h3>Partitioning configuration<a class="headerlink" href="#partitioning-configuration" title="Permalink to this headline">¶</a></h3> +<p>There are 4 partitioning schemes, whose configurations are give below,</p> +<ol> +<li><p>Partitioning each singe layer into sub-layers on batch dimension (see +below). It is enabled by configuring the partition dimension of the layer to +0, e.g.,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="c1"># with other fields omitted</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">partition_dim</span><span class="p">:</span> <span class="mi">0</span> + <span class="p">}</span> +</pre></div> +</div> +</li> +<li><p>Partitioning each singe layer into sub-layers on feature dimension (see +below). It is enabled by configuring the partition dimension of the layer to +1, e.g.,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="c1"># with other fields omitted</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">partition_dim</span><span class="p">:</span> <span class="mi">1</span> + <span class="p">}</span> +</pre></div> +</div> +</li> +<li><p>Partitioning all layers into different subsets. It is enabled by +configuring the location ID of a layer, e.g.,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="c1"># with other fields omitted</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">1</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">0</span> + <span class="p">}</span> +</pre></div> +</div> +</li> +<li><p>Hybrid partitioning of strategy 1, 2 and 3. The hybrid partitioning is +useful for large models. An example application is to implement the +<a class="reference external" href="http://arxiv.org/abs/1404.5997";>idea proposed by Alex</a>. +Hybrid partitioning is configured like,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="c1"># with other fields omitted</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">1</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">0</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">partition_dim</span><span class="p">:</span> <span class="mi">0</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">0</span> + <span class="p">}</span> + <span class="n">layer</span> <span class="p">{</span> + <span class="n">partition_dim</span><span class="p">:</span> <span class="mi">1</span> + <span class="n">location</span><span class="p">:</span> <span class="mi">0</span> + <span class="p">}</span> +</pre></div> +</div> +</li> +</ol> +<p>Currently SINGA supports strategy-2 well. Other partitioning strategies are +are under test and will be released in later version.</p> +</div> +</div> +<div class="section" id="parameter-sharing"> +<h2>Parameter sharing<a class="headerlink" href="#parameter-sharing" title="Permalink to this headline">¶</a></h2> +<p>Parameters can be shared in two cases,</p> +<ul class="simple"> +<li><p>sharing parameters among layers via user configuration. For example, the +visible layer and hidden layer of a RBM shares the weight matrix, which is configured through +the <code class="docutils literal notranslate"><span class="pre">share_from</span></code> field as shown in the above RBM configuration. The +configurations must be the same (except name) for shared parameters.</p></li> +<li><p>due to neural net partitioning, some <code class="docutils literal notranslate"><span class="pre">Param</span></code> objects are replicated into +different workers, e.g., partitioning one layer on batch dimension. These +workers share parameter values. SINGA controls this kind of parameter +sharing automatically, users do not need to do any configuration.</p></li> +<li><p>the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> for training and testing (and validation) share most layers +, thus share <code class="docutils literal notranslate"><span class="pre">Param</span></code> values.</p></li> +</ul> +<p>If the shared <code class="docutils literal notranslate"><span class="pre">Param</span></code> instances resident in the same process (may in different +threads), they use the same chunk of memory space for their values. But they +would have different memory spaces for their gradients. In fact, their +gradients will be averaged by the stub or server.</p> +</div> +<div class="section" id="advanced-user-guide"> +<h2>Advanced user guide<a class="headerlink" href="#advanced-user-guide" title="Permalink to this headline">¶</a></h2> +<div class="section" id="creation"> +<h3>Creation<a class="headerlink" href="#creation" title="Permalink to this headline">¶</a></h3> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">static</span> <span class="n">NeuralNet</span><span class="o">*</span> <span class="n">NeuralNet</span><span class="p">::</span><span class="n">Create</span><span class="p">(</span><span class="n">const</span> <span class="n">NetProto</span><span class="o">&</span> <span class="n">np</span><span class="p">,</span> <span class="n">Phase</span> <span class="n">phase</span><span class="p">,</span> <span class="nb">int</span> <span class="n">num</span><span class="p">);</span> +</pre></div> +</div> +<p>The above function creates a <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> for a given phase, and returns a +pointer to the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> instance. The phase is in {kTrain, +kValidation, kTest}. <code class="docutils literal notranslate"><span class="pre">num</span></code> is used for net partitioning which indicates the +number of partitions. Typically, a training job includes three neural nets for +training, validation and test phase respectively. The three neural nets share most +layers except the data layer, loss layer or output layer, etc.. The <code class="docutils literal notranslate"><span class="pre">Create</span></code> +function takes in the full net configuration including layers for training, +validation and test. It removes layers for phases other than the specified +phase based on the <code class="docutils literal notranslate"><span class="pre">exclude</span></code> field in +<a class="reference external" href="layer.html">layer configuration</a>:</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">layer</span> <span class="p">{</span> + <span class="o">...</span> + <span class="n">exclude</span> <span class="p">:</span> <span class="n">kTest</span> <span class="c1"># filter this layer for creating test net</span> +<span class="p">}</span> +</pre></div> +</div> +<p>The filtered net configuration is passed to the constructor of <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code>:</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">NeuralNet</span><span class="p">::</span><span class="n">NeuralNet</span><span class="p">(</span><span class="n">NetProto</span> <span class="n">netproto</span><span class="p">,</span> <span class="nb">int</span> <span class="n">npartitions</span><span class="p">);</span> +</pre></div> +</div> +<p>The constructor creates a graph representing the net structure firstly in</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Graph</span><span class="o">*</span> <span class="n">NeuralNet</span><span class="p">::</span><span class="n">CreateGraph</span><span class="p">(</span><span class="n">const</span> <span class="n">NetProto</span><span class="o">&</span> <span class="n">netproto</span><span class="p">,</span> <span class="nb">int</span> <span class="n">npartitions</span><span class="p">);</span> +</pre></div> +</div> +<p>Next, it creates a layer for each node and connects layers if their nodes are +connected.</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">void</span> <span class="n">NeuralNet</span><span class="p">::</span><span class="n">CreateNetFromGraph</span><span class="p">(</span><span class="n">Graph</span><span class="o">*</span> <span class="n">graph</span><span class="p">,</span> <span class="nb">int</span> <span class="n">npartitions</span><span class="p">);</span> +</pre></div> +</div> +<p>Since the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> instance may be shared among multiple workers, the +<code class="docutils literal notranslate"><span class="pre">Create</span></code> function returns a pointer to the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> instance .</p> +</div> +<div class="section" id="id1"> +<h3>Parameter sharing<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3> +<p><code class="docutils literal notranslate"><span class="pre">Param</span></code> sharing +is enabled by first sharing the Param configuration (in <code class="docutils literal notranslate"><span class="pre">NeuralNet::Create</span></code>) +to create two similar (e.g., the same shape) Param objects, and then calling +(in <code class="docutils literal notranslate"><span class="pre">NeuralNet::CreateNetFromGraph</span></code>),</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">void</span> <span class="n">Param</span><span class="p">::</span><span class="n">ShareFrom</span><span class="p">(</span><span class="n">const</span> <span class="n">Param</span><span class="o">&</span> <span class="n">from</span><span class="p">);</span> +</pre></div> +</div> +<p>It is also possible to share <code class="docutils literal notranslate"><span class="pre">Param</span></code>s of two nets, e.g., sharing parameters of +the training net and the test net,</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">void</span> <span class="n">NeuralNet</span><span class="p">:</span><span class="n">ShareParamsFrom</span><span class="p">(</span><span class="n">NeuralNet</span><span class="o">*</span> <span class="n">other</span><span class="p">);</span> +</pre></div> +</div> +<p>It will call <code class="docutils literal notranslate"><span class="pre">Param::ShareFrom</span></code> for each Param object.</p> +</div> +<div class="section" id="access-functions"> +<h3>Access functions<a class="headerlink" href="#access-functions" title="Permalink to this headline">¶</a></h3> +<p><code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> provides a couple of access function to get the layers and params +of the net:</p> +<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">const</span> <span class="n">std</span><span class="p">::</span><span class="n">vector</span><span class="o"><</span><span class="n">Layer</span><span class="o">*>&</span> <span class="n">layers</span><span class="p">()</span> <span class="n">const</span><span class="p">;</span> +<span class="n">const</span> <span class="n">std</span><span class="p">::</span><span class="n">vector</span><span class="o"><</span><span class="n">Param</span><span class="o">*>&</span> <span class="n">params</span><span class="p">()</span> <span class="n">const</span> <span class="p">;</span> +<span class="n">Layer</span><span class="o">*</span> <span class="n">name2layer</span><span class="p">(</span><span class="n">string</span> <span class="n">name</span><span class="p">)</span> <span class="n">const</span><span class="p">;</span> +<span class="n">Param</span><span class="o">*</span> <span class="n">paramid2param</span><span class="p">(</span><span class="nb">int</span> <span class="nb">id</span><span class="p">)</span> <span class="n">const</span><span class="p">;</span> +</pre></div> +</div> +</div> +<div class="section" id="partitioning"> +<h3>Partitioning<a class="headerlink" href="#partitioning" title="Permalink to this headline">¶</a></h3> +<div class="section" id="implementation"> +<h4>Implementation<a class="headerlink" href="#implementation" title="Permalink to this headline">¶</a></h4> +<p>SINGA partitions the neural net in <code class="docutils literal notranslate"><span class="pre">CreateGraph</span></code> function, which creates one +node for each (partitioned) layer. For example, if one layerâs partition +dimension is 0 or 1, then it creates <code class="docutils literal notranslate"><span class="pre">npartition</span></code> nodes for it; if the +partition dimension is -1, a single node is created, i.e., no partitioning. +Each node is assigned a partition (or location) ID. If the original layer is +configured with a location ID, then the ID is assigned to each newly created node. +These nodes are connected according to the connections of the original layers. +Some connection layers will be added automatically. +For instance, if two connected sub-layers are located at two +different workers, then a pair of bridge layers is inserted to transfer the +feature (and gradient) blob between them. When two layers are partitioned on +different dimensions, a concatenation layer which concatenates feature rows (or +columns) and a slice layer which slices feature rows (or columns) would be +inserted. These connection layers help making the network communication and +synchronization transparent to the users.</p> +</div> +<div class="section" id="dispatching-partitions-to-workers"> +<h4>Dispatching partitions to workers<a class="headerlink" href="#dispatching-partitions-to-workers" title="Permalink to this headline">¶</a></h4> +<p>Each (partitioned) layer is assigned a location ID, based on which it is dispatched to one +worker. Particularly, the pointer to the <code class="docutils literal notranslate"><span class="pre">NeuralNet</span></code> instance is passed +to every worker within the same group, but each worker only computes over the +layers that have the same partition (or location) ID as the workerâs ID. When +every worker computes the gradients of the entire model parameters +(strategy-2), we refer to this process as data parallelism. When different +workers compute the gradients of different parameters (strategy-3 or +strategy-1), we call this process model parallelism. The hybrid partitioning +leads to hybrid parallelism where some workers compute the gradients of the +same subset of model parameters while other workers compute on different model +parameters. For example, to implement the hybrid parallelism in for the +<a class="reference external" href="http://arxiv.org/abs/1404.5997";>DCNN model</a>, we set <code class="docutils literal notranslate"><span class="pre">partition_dim</span> <span class="pre">=</span> <span class="pre">0</span></code> for +lower layers and <code class="docutils literal notranslate"><span class="pre">partition_dim</span> <span class="pre">=</span> <span class="pre">1</span></code> for higher layers.</p> +</div> +</div> +</div> +</div> + + + </div> + + </div> + <footer> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2019 The Apache Software Foundation. All rights reserved. Apache SINGA, Apache, the Apache feather logo, and the Apache SINGA project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners. + + </p> + </div> + Built with <a href="http://sphinx-doc.org/";>Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> + + + + + + +<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> + <span class="rst-current-version" data-toggle="rst-current-version"> + <span class="fa fa-book"> incubator-singa </span> + v: latest + <span class="fa fa-caret-down"></span> + </span> + <div class="rst-other-versions"> + <dl> + <dt>Languages</dt> + <dd><a href="../../en/index.html">English</a></dd> + <dd><a href="../../zh/index.html">ä¸æ</a></dd> + </dl> + <dl> + <dt>Versions</dt> + <dd><a href="http://singa.apache.org/v0.3.0/";>0.3</a></dd> + <dd><a href="http://singa.apache.org/v1.1.0/";>1.1</a></dd> + </dl> + + </div> + <a href="http://incubator.apache.org/";> <img src= "../_static/apache.png" style="background-color:white;"> </a> + + <a href="https://github.com/apache/incubator-singa"; class="fa fa-github" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://aws.amazon.com/marketplace/seller-profile?id=5bcac385-12c4-4802-aec7-351e09b77b4c"; class="fab fa-aws" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://hub.docker.com/r/apache/singa/"; class="fab fa-docker" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.linkedin.com/groups/13550034"; class="fa fa-linkedin" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://twitter.com/ApacheSinga"; class="fa fa-twitter" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.facebook.com/Apache-SINGA-347284219056544/"; class="fa fa-facebook" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.researchgate.net/project/Apache-SINGA"; class="fab fa-researchgate" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + +</div> + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> +</a> + + + + +</body> +</html> \ No newline at end of file Added: incubator/singa/site/trunk/en/docs/notebook/README.html URL: http://svn.apache.org/viewvc/incubator/singa/site/trunk/en/docs/notebook/README.html?rev=1858059&view=auto ============================================================================== --- incubator/singa/site/trunk/en/docs/notebook/README.html (added) +++ incubator/singa/site/trunk/en/docs/notebook/README.html Wed Apr 24 14:57:35 2019 @@ -0,0 +1,282 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title><no title> — incubator-singa 1.1.0 documentation</title> + + + + + + + + + <script type="text/javascript" src="../../_static/js/modernizr.min.js"></script> + + + <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script> + <script type="text/javascript" src="../../_static/jquery.js"></script> + <script type="text/javascript" src="../../_static/underscore.js"></script> + <script type="text/javascript" src="../../_static/doctools.js"></script> + <script type="text/javascript" src="../../_static/language_data.js"></script> + + <script type="text/javascript" src="../../_static/js/theme.js"></script> + + + + + <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../../genindex.html" /> + <link rel="search" title="Search" href="../../search.html" /> + <link href="../../_static/style.css" rel="stylesheet" type="text/css"> + <!--link href="../../_static/fontawesome-all.min.css" rel="stylesheet" type="text/css"--> + <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.0.13/css/all.css"; integrity="sha384-DNOHZ68U8hZfKXOrtjWvjxusGo9WQnrNx2sqG0tfsghAvtVlRW3tvkXWZh58N9jp" crossorigin="anonymous"> + <style> + .fa:hover { + opacity: 0.7; + } + .fab:hover { + opacity: 0.7; + } + </style> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search" > + + + + <a href="../../index.html" class="icon icon-home"> incubator-singa + + + + + <img src="../../_static/singa.png" class="logo" alt="Logo"/> + + </a> + + + + + <div class="version"> + latest + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <ul> +<li class="toctree-l1"><a class="reference internal" href="../index.html">Documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../downloads.html">Download SINGA</a></li> +</ul> +<p class="caption"><span class="caption-text">Development</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../develop/schedule.html">Development Schedule</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../develop/how-contribute.html">How to Contribute to SINGA</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../develop/how-contribute.html#how-to-become-a-singa-committer">How to become a SINGA committer</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../develop/contribute-code.html">How to Contribute Code</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../develop/contribute-docs.html">How to Contribute to Documentation</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../develop/how-to-release.html">How to prepare a release</a></li> +</ul> +<p class="caption"><span class="caption-text">Community</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../../community/source-repository.html">Source Repository</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../community/mail-lists.html">Project Mailing Lists</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../community/issue-tracking.html">Issue Tracking</a></li> +<li class="toctree-l1"><a class="reference internal" href="../../community/team-list.html">The SINGA Team</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../../index.html">incubator-singa</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../../index.html">Docs</a> »</li> + + <li><no title></li> + + + <li class="wy-breadcrumbs-aside"> + + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article";> + <div itemprop="articleBody"> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--><p>These are some examples in IPython notebooks.</p> +<p>You can open them in <a class="reference external" href="http://nbviewer.jupyter.org/github/apache/incubator-singa/blob/master/doc/en/docs/notebook/index.ipynb";>notebook viewer</a>.</p> + + + </div> + + </div> + <footer> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2019 The Apache Software Foundation. All rights reserved. Apache SINGA, Apache, the Apache feather logo, and the Apache SINGA project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners. + + </p> + </div> + Built with <a href="http://sphinx-doc.org/";>Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme";>theme</a> provided by <a href="https://readthedocs.org";>Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> + + + + + + +<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> + <span class="rst-current-version" data-toggle="rst-current-version"> + <span class="fa fa-book"> incubator-singa </span> + v: latest + <span class="fa fa-caret-down"></span> + </span> + <div class="rst-other-versions"> + <dl> + <dt>Languages</dt> + <dd><a href="../../../en/index.html">English</a></dd> + <dd><a href="../../../zh/index.html">ä¸æ</a></dd> + </dl> + <dl> + <dt>Versions</dt> + <dd><a href="http://singa.apache.org/v0.3.0/";>0.3</a></dd> + <dd><a href="http://singa.apache.org/v1.1.0/";>1.1</a></dd> + </dl> + + </div> + <a href="http://incubator.apache.org/";> <img src= "../../_static/apache.png" style="background-color:white;"> </a> + + <a href="https://github.com/apache/incubator-singa"; class="fa fa-github" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://aws.amazon.com/marketplace/seller-profile?id=5bcac385-12c4-4802-aec7-351e09b77b4c"; class="fab fa-aws" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://hub.docker.com/r/apache/singa/"; class="fab fa-docker" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.linkedin.com/groups/13550034"; class="fa fa-linkedin" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://twitter.com/ApacheSinga"; class="fa fa-twitter" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.facebook.com/Apache-SINGA-347284219056544/"; class="fa fa-facebook" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + <a href="https://www.researchgate.net/project/Apache-SINGA"; class="fab fa-researchgate" style="padding: 10px; font-size: 20px; width: 30px; text-align: center; text-decoration: none; margin: 5px 2px;"></a> + +</div> + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> +</a> + + + + +</body> +</html> \ No newline at end of file
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}