Added: websites/staging/singa/trunk/content/v0.2.0/param.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/param.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/param.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,525 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Parameters</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Parameters</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Parameters</h1> +<hr /> +<p>A <tt>Param</tt> object in SINGA represents a set of parameters, e.g., a weight matrix or a bias vector. <i>Basic user guide</i> describes how to configure for a <tt>Param</tt> object, and <i>Advanced user guide</i> provides details on implementing users’ parameter initialization methods.</p> +<div class="section"> +<h2><a name="Basic_user_guide"></a>Basic user guide</h2> +<p>The configuration of a Param object is inside a layer configuration, as the <tt>Param</tt> are associated with layers. An example configuration is like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + ... + param { + name : "p1" + init { + type : kConstant + value: 1 + } + } +} +</pre></div></div> +<p>The <a href="overview.html">SGD algorithm</a> starts with initializing all parameters according to user specified initialization method (the <tt>init</tt> field). For the above example, all parameters in <tt>Param</tt> “p1” will be initialized to constant value 1. The configuration fields of a Param object is defined in <a href="../api/classsinga_1_1ParamProto.html">ParamProto</a>:</p> + +<ul> + +<li>name, an identifier string. It is an optional field. If not provided, SINGA will generate one based on layer name and its order in the layer.</li> + +<li>init, field for setting initialization methods.</li> + +<li>share_from, name of another <tt>Param</tt> object, from which this <tt>Param</tt> will share configurations and values.</li> + +<li>lr_scale, float value to be multiplied with the learning rate when <a href="updater.html">updating the parameters</a></li> + +<li>wd_scale, float value to be multiplied with the weight decay when <a href="updater.html">updating the parameters</a></li> +</ul> +<p>There are some other fields that are specific to initialization methods.</p> +<div class="section"> +<h3><a name="Initialization_methods"></a>Initialization methods</h3> +<p>Users can set the <tt>type</tt> of <tt>init</tt> use the following built-in initialization methods,</p> + +<ul> + +<li> +<p><tt>kConst</tt>, set all parameters of the Param object to a constant value</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kConst +value: float # default is 1 +</pre></div></div></li> + +<li> +<p><tt>kGaussian</tt>, initialize the parameters following a Gaussian distribution.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kGaussian +mean: float # mean of the Gaussian distribution, default is 0 +std: float # standard variance, default is 1 +value: float # default 0 +</pre></div></div></li> + +<li> +<p><tt>kUniform</tt>, initialize the parameters following an uniform distribution</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kUniform +low: float # lower boundary, default is -1 +high: float # upper boundary, default is 1 +value: float # default 0 +</pre></div></div></li> + +<li> +<p><tt>kGaussianSqrtFanIn</tt>, initialize <tt>Param</tt> objects with two dimensions (i.e., matrix) using <tt>kGaussian</tt> and then multiple each parameter with <tt>1/sqrt(fan_in)</tt>, where<tt>fan_in</tt> is the number of columns of the matrix.</p></li> + +<li> +<p><tt>kUniformSqrtFanIn</tt>, the same as <tt>kGaussianSqrtFanIn</tt> except that the distribution is an uniform distribution.</p></li> + +<li> +<p><tt>kUniformFanInOut</tt>, initialize matrix <tt>Param</tt> objects using <tt>kUniform</tt> and then multiple each parameter with <tt>sqrt(6/(fan_in + fan_out))</tt>, where<tt>fan_in + + fan_out</tt> sums up the number of columns and rows of the matrix.</p></li> +</ul> +<p>For all above initialization methods except <tt>kConst</tt>, if their <tt>value</tt> is not 1, every parameter will be multiplied with <tt>value</tt>. Users can also implement their own initialization method following the <i>Advanced user guide</i>.</p></div></div> +<div class="section"> +<h2><a name="Advanced_user_guide"></a>Advanced user guide</h2> +<p>This sections describes the details on implementing new parameter initialization methods.</p> +<div class="section"> +<h3><a name="Base_ParamGenerator"></a>Base ParamGenerator</h3> +<p>All initialization methods are implemented as subclasses of the base <tt>ParamGenerator</tt> class.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class ParamGenerator { + public: + virtual void Init(const ParamGenProto&); + void Fill(Param*); + + protected: + ParamGenProto proto_; +}; +</pre></div></div> +<p>Configurations of the initialization method is in <tt>ParamGenProto</tt>. The <tt>Fill</tt> function fills the <tt>Param</tt> object (passed in as an argument).</p></div> +<div class="section"> +<h3><a name="New_ParamGenerator_subclass"></a>New ParamGenerator subclass</h3> +<p>Similar to implement a new Layer subclass, users can define a configuration protocol message,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in user.proto +message FooParamProto { + optional int32 x = 1; +} +extend ParamGenProto { + optional FooParamProto fooparam_conf =101; +} +</pre></div></div> +<p>The configuration of <tt>Param</tt> would be</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">param { + ... + init { + user_type: 'FooParam" # must use user_type for user defined methods + [fooparam_conf] { # must use brackets for configuring user defined messages + x: 10 + } + } +} +</pre></div></div> +<p>The subclass could be declared as,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class FooParamGen : public ParamGenerator { + public: + void Fill(Param*) override; +}; +</pre></div></div> +<p>Users can access the configuration fields in <tt>Fill</tt> by</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">int x = proto_.GetExtension(fooparam_conf).x(); +</pre></div></div> +<p>To use the new initialization method, users need to register it in the <a href="programming-guide.html">main function</a>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">driver.RegisterParamGenerator<FooParamGen>("FooParam") # must be consistent with the user_type in configuration +</pre></div></div> +<p>{% comment %}</p></div> +<div class="section"> +<h3><a name="Base_Param_class"></a>Base Param class</h3></div> +<div class="section"> +<h3><a name="Members"></a>Members</h3> + +<div class="source"> +<div class="source"><pre class="prettyprint">int local_version_; +int slice_start_; +vector<int> slice_offset_, slice_size_; + +shared_ptr<Blob<float>> data_; +Blob<float> grad_; +ParamProto proto_; +</pre></div></div> +<p>Each Param object has a local version and a global version (inside the data Blob). These two versions are used for synchronization. If multiple Param objects share the same values, they would have the same <tt>data_</tt> field. Consequently, their global version is the same. The global version is updated by <a href="communication.html">the stub thread</a>. The local version is updated in <tt>Worker::Update</tt> function which assigns the global version to the local version. The <tt>Worker::Collect</tt> function is blocked until the global version is larger than the local version, i.e., when <tt>data_</tt> is updated. In this way, we synchronize workers sharing parameters.</p> +<p>In Deep learning models, some Param objects are 100 times larger than others. To ensure the load-balance among servers, SINGA slices large Param objects. The slicing information is recorded by <tt>slice_*</tt>. Each slice is assigned a unique ID starting from 0. <tt>slice_start_</tt> is the ID of the first slice of this Param object. <tt>slice_offset_[i]</tt> is the offset of the i-th slice in this Param object. <tt>slice_size_[i]</tt> is the size of the i-th slice. These slice information is used to create messages for transferring parameter values or gradients to different servers.</p> +<p>Each Param object has a <tt>grad_</tt> field for gradients. Param objects do not share this Blob although they may share <tt>data_</tt>. Because each layer containing a Param object would contribute gradients. E.g., in RNN, the recurrent layers share parameters values, and the gradients used for updating are averaged from all recurrent these recurrent layers. In SINGA, the stub thread will aggregate local gradients for the same Param object. The server will do a global aggregation of gradients for the same Param object.</p> +<p>The <tt>proto_</tt> field has some meta information, e.g., name and ID. It also has a field called <tt>owner</tt> which is the ID of the Param object that shares parameter values with others.</p></div> +<div class="section"> +<h3><a name="Functions"></a>Functions</h3> +<p>The base Param class implements two sets of functions,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">virtual void InitValues(int version = 0); // initialize values according to `init_method` +void ShareFrom(const Param& other); // share `data_` from `other` Param +-------------- +virtual Msg* GenGetMsg(bool copy, int slice_idx); +virtual Msg* GenPutMsg(bool copy, int slice_idx); +... // other message related functions. +</pre></div></div> +<p>Besides the functions for processing the parameter values, there is a set of functions for generating and parsing messages. These messages are for transferring parameter values or gradients between workers and servers. Each message corresponds to one Param slice. If <tt>copy</tt> is false, it means the receiver of this message is in the same process as the sender. In such case, only pointers to the memory of parameter value (or gradient) are wrapped in the message; otherwise, the parameter values (or gradients) should be copied into the message.</p></div></div> +<div class="section"> +<h2><a name="Implementing_Param_subclass"></a>Implementing Param subclass</h2> +<p>Users can extend the base Param class to implement their own parameter initialization methods and message transferring protocols. Similar to implementing a new Layer subclasses, users can create google protocol buffer messages for configuring the Param subclass. The subclass, denoted as FooParam should be registered in main.cc,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">dirver.RegisterParam<FooParam>(kFooParam); // kFooParam should be different to 0, which is for the base Param type +</pre></div></div> + +<ul> + +<li>type, an integer representing the <tt>Param</tt> type. Currently SINGA provides one <tt>Param</tt> implementation with type 0 (the default type). If users want to use their own Param implementation, they should extend the base Param class and configure this field with <tt>kUserParam</tt></li> +</ul> +<p>{% endcomment %}</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
Added: websites/staging/singa/trunk/content/v0.2.0/programming-guide.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/programming-guide.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/programming-guide.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,410 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Programming Guide</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Programming Guide</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Programming Guide</h1> +<hr /> +<p>To submit a training job, users must provide the configuration of the four components shown in Figure 1:</p> + +<ul> + +<li>a <a href="neural-net.html">NeuralNet</a> describing the neural net structure with the detailed layer setting and their connections;</li> + +<li>a <a href="train-one-batch.html">TrainOneBatch</a> algorithm which is tailored for different model categories;</li> + +<li>an <a href="updater.html">Updater</a> defining the protocol for updating parameters at the server side;</li> + +<li>a <a href="distributed-training.html">Cluster Topology</a> specifying the distributed architecture of workers and servers.</li> +</ul> +<p>The <i>Basic user guide</i> section describes how to submit a training job using built-in components; while the <i>Advanced user guide</i> section presents details on writing user’s own main function to register components implemented by themselves. In addition, the training data must be prepared, which has the same <a href="data.html">process</a> for both advanced users and basic users.</p> +<p><img src="../images/overview.png" align="center" width="400px" alt="" /> <span><b>Figure 1 - SINGA overview.</b></span></p> +<div class="section"> +<h2><a name="Basic_user_guide"></a>Basic user guide</h2> +<p>Users can use the default main function provided SINGA to submit the training job. For this case, a job configuration file written as a google protocol buffer message for the <a href="../api/classsinga_1_1JobProto.html">JobProto</a> must be provided in the command line,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -conf <path to job conf> [-resume] +</pre></div></div> +<p><tt>-resume</tt> is for continuing the training from last <a href="checkpoint.html">checkpoint</a>. The <a href="mlp.html">MLP</a> and <a href="cnn.html">CNN</a> examples use built-in components. Please read the corresponding pages for their job configuration files. The subsequent pages will illustrate the details on each component of the configuration.</p></div> +<div class="section"> +<h2><a name="Advanced_user_guide"></a>Advanced user guide</h2> +<p>If a user’s model contains some user-defined components, e.g., <a href="updater.html">Updater</a>, he has to write a main function to register these components. It is similar to Hadoop’s main function. Generally, the main function should</p> + +<ul> + +<li> +<p>initialize SINGA, e.g., setup logging.</p></li> + +<li> +<p>register user-defined components.</p></li> + +<li> +<p>create and pass the job configuration to SINGA driver</p></li> +</ul> +<p>An example main function is like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">#include "singa.h" +#include "user.h" // header for user code + +int main(int argc, char** argv) { + singa::Driver driver; + driver.Init(argc, argv); + bool resume; + // parse resume option from argv. + + // register user defined layers + driver.RegisterLayer<FooLayer>(kFooLayer); + // register user defined updater + driver.RegisterUpdater<FooUpdater>(kFooUpdater); + ... + auto jobConf = driver.job_conf(); + // update jobConf + + driver.Train(resume, jobConf); + return 0; +} +</pre></div></div> +<p>The Driver class’ <tt>Init</tt> method will load a job configuration file provided by users as a command line argument (<tt>-conf <job conf></tt>). It contains at least the cluster topology and returns the <tt>jobConf</tt> for users to update or fill in configurations of neural net, updater, etc. If users define subclasses of Layer, Updater, Worker and Param, they should register them through the driver. Finally, the job configuration is submitted to the driver which starts the training.</p> +<p>We will provide helper functions to make the configuration easier in the future, like <a class="externalLink" href="https://github.com/fchollet/keras";>keras</a>.</p> +<p>Users need to compile and link their code (e.g., layer implementations and the main file) with SINGA library (<i>.libs/libsinga.so</i>) to generate an executable file, e.g., with name <i>mysinga</i>. To launch the program, users just pass the path of the <i>mysinga</i> and base job configuration to <i>./bin/singa-run.sh</i>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -conf <path to job conf> -exec <path to mysinga> [other arguments] +</pre></div></div> +<p>The <a href="rnn.html">RNN application</a> provides a full example of implementing the main function for training a specific RNN model.</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/v0.2.0/python.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/python.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/python.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,757 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Python Binding</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Python Binding</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Python Binding</h1> +<hr /> +<p>Python binding provides APIs for configuring a training job following <a class="externalLink" href="http://keras.io/";>keras</a>, including the configuration of neural net, training algorithm, etc. It replaces the configuration file (e.g., <i>job.conf</i>) in protobuf format, which is typically long and error-prone to prepare. In later version, we will add python functions to interact with the layer and neural net objects, which would enable users to train and debug their models interactively.</p> +<p>Here is the layout of python related code,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">SINGAROOT/tool/python +|-- pb2 (has job_pb2.py) +|-- singa + |-- model.py + |-- layer.py + |-- parameter.py + |-- initialization.py + |-- utils + |-- utility.py + |-- message.py +|-- examples + |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, mnist_ae.py, etc. + |-- datasets + |-- cifar10.py + |-- mnist.py +</pre></div></div> +<div class="section"> +<h2><a name="Compiling_and_running_instructions"></a>Compiling and running instructions</h2> +<p>In order to use the Python APIs, users need to add the following arguments when compiling SINGA,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./configure --enable-python --with-python=PYTHON_DIR +make +</pre></div></div> +<p>where PYTHON_DIR has Python.h</p> +<p>The training program is launched by</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">bin/singa-run.sh -exec <user_main.py> +</pre></div></div> +<p>where user_main.py creates the JobProto object and passes it to Driver::Train to start the training.</p> +<p>For example,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cd SINGAROOT +bin/singa-run.sh -exec tool/python/examples/cifar10_cnn.py +</pre></div></div></div> +<div class="section"> +<h2><a name="Examples"></a>Examples</h2> +<div class="section"> +<h3><a name="MLP_Example"></a>MLP Example</h3> +<p>This example uses python APIs to configure and train a MLP model over the MNIST dataset. The configuration content is the same as that written in <i>SINGAROOT/examples/mnist/job.conf</i>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">X_train, X_test, workspace = mnist.load_data() + +m = Sequential('mlp', sys.argv) + +m.add(Dense(2500, init='uniform', activation='tanh')) +m.add(Dense(2000, init='uniform', activation='tanh')) +m.add(Dense(1500, init='uniform', activation='tanh')) +m.add(Dense(1000, init='uniform', activation='tanh')) +m.add(Dense(500, init='uniform', activation='tanh')) +m.add(Dense(10, init='uniform', activation='softmax')) + +sgd = SGD(lr=0.001, lr_type='step') +topo = Cluster(workspace) +m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo) +m.fit(X_train, nb_epoch=1000, with_test=True) +result = m.evaluate(X_test, batch_size=100, test_steps=10, test_freq=60) +</pre></div></div></div> +<div class="section"> +<h3><a name="CNN_Example"></a>CNN Example</h3> +<p>This example uses python APIs to configure and train a CNN model over the Cifar10 dataset. The configuration content is the same as that written in <i>SINGAROOT/examples/cifar10/job.conf</i>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">X_train, X_test, workspace = cifar10.load_data() + +m = Sequential('cnn', sys.argv) + +m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2)) +m.add(MaxPooling2D(pool_size=(3,3), stride=2)) +m.add(Activation('relu')) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution2D(32, 5, 1, 2, b_lr=2)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution2D(64, 5, 1, 2)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size=(3,3), stride=2)) + +m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax')) + +sgd = SGD(decay=0.004, lr_type='manual', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001)) +topo = Cluster(workspace) +m.compile(updater=sgd, cluster=topo) +m.fit(X_train, nb_epoch=1000, with_test=True) +result = m.evaluate(X_test, 1000, test_steps=30, test_freq=300) +</pre></div></div></div> +<div class="section"> +<h3><a name="RBM_Example"></a>RBM Example</h3> +<p>This example uses python APIs to configure and train a RBM model over the MNIST dataset. The configuration content is the same as that written in <i>SINGAROOT/examples/rbm</i>.conf*.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">rbmid = 3 +X_train, X_test, workspace = mnist.load_data(nb_rbm=rbmid) +m = Energy('rbm'+str(rbmid), sys.argv) + +out_dim = [1000, 500, 250] +m.add(RBM(out_dim, w_std=0.1, b_wd=0)) + +sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8) +topo = Cluster(workspace) +m.compile(optimizer=sgd, cluster=topo) +m.fit(X_train, alg='cd', nb_epoch=6000) +</pre></div></div></div> +<div class="section"> +<h3><a name="AutoEncoder_Example"></a>AutoEncoder Example</h3> +<p>This example uses python APIs to configure and train an autoencoder model over the MNIST dataset. The configuration content is the same as that written in <i>SINGAROOT/examples/autoencoder.conf</i>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">rbmid = 4 +X_train, X_test, workspace = mnist.load_data(nb_rbm=rbmid+1) +m = Sequential('autoencoder', sys.argv) + +hid_dim = [1000, 500, 250, 30] +m.add(Autoencoder(hid_dim, out_dim=784, activation='sigmoid', param_share=True)) + +agd = AdaGrad(lr=0.01) +topo = Cluster(workspace) +m.compile(loss='mean_squared_error', optimizer=agd, cluster=topo) +m.fit(X_train, alg='bp', nb_epoch=12200) +</pre></div></div></div> +<div class="section"> +<h3><a name="To_run_SINGA_on_GPU"></a>To run SINGA on GPU</h3> +<p>Users need to set a list of gpu ids to <tt>device</tt> field in fit() or evaluate(). The number of GPUs must be the same to the number of workers configured for cluster topology.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">gpu_id = [0] +m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id) +</pre></div></div></div> +<div class="section"> +<h3><a name="TIPS"></a>TIPS</h3> +<p>Hidden layers for MLP can be configured as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">for n in [2500, 2000, 1500, 1000, 500]: + m.add(Dense(n, init='uniform', activation='tanh')) +m.add(Dense(10, init='uniform', activation='softmax')) +</pre></div></div> +<p>Activation layer can be specified separately</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">m.add(Dense(2500, init='uniform')) +m.add(Activation('tanh')) +</pre></div></div> +<p>Users can explicitly specify hyper-parameters of weight and bias</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">par = Parameter(init='uniform', scale=0.05) +m.add(Dense(2500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(2000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(1000, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(500, w_param=par, b_param=par, activation='tanh')) +m.add(Dense(10, w_param=par, b_param=par, activation='softmax')) +</pre></div></div> + +<div class="source"> +<div class="source"><pre class="prettyprint">parw = Parameter(init='gauss', std=0.0001) +parb = Parameter(init='const', value=0) +m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2)) +m.add(MaxPooling2D(pool_size(3,3), stride=2)) +m.add(Activation('relu')) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +parw.update(std=0.01) +m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size(3,3), stride=2)) +m.add(LRN2D(3, alpha=0.00005, beta=0.75)) + +m.add(Convolution(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1)) +m.add(Activation('relu')) +m.add(AvgPooling2D(pool_size(3,3), stride=2)) + +m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax')) +</pre></div></div> +<p>Data can be added in this way,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">X_train, X_test = mnist.load_data() // parameter values are set in load_data() +m.fit(X_train, ...) // Data layer for training is added +m.evaluate(X_test, ...) // Data layer for testing is added +</pre></div></div> +<p>or this way,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">X_train, X_test = mnist.load_data() // parameter values are set in load_data() +m.add(X_train) // explicitly add Data layer +m.add(X_test) // explicitly add Data layer +</pre></div></div> + +<div class="source"> +<div class="source"><pre class="prettyprint">store = Store(path='train.bin', batch_size=64, ...) // parameter values are set explicitly +m.add(Data(load='recordinput', phase='train', conf=store)) // Data layer is added +store = Store(path='test.bin', batch_size=100, ...) // parameter values are set explicitly +m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added +</pre></div></div></div> +<div class="section"> +<h3><a name="Cases_to_run_SINGA"></a>Cases to run SINGA</h3> +<p>(1) Run SINGA for training</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">m.fit(X_train, nb_epoch=1000) +</pre></div></div> +<p>(2) Run SINGA for training and validation</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">m.fit(X_train, validate_data=X_valid, nb_epoch=1000) +</pre></div></div> +<p>(3) Run SINGA for test while training</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">m.fit(X_train, nb_epoch=1000, with_test=True) +result = m.evaluate(X_test, batch_size=100, test_steps=100) +</pre></div></div> +<p>(4) Run SINGA for test only Assume a checkpoint exists after training</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">result = m.evaluate(X_test, batch_size=100, checkpoint_path=workspace+'/checkpoint/step100-worker0') +</pre></div></div></div></div> +<div class="section"> +<h2><a name="Implementation_Details"></a>Implementation Details</h2> +<div class="section"> +<h3><a name="Layer_class_inherited"></a>Layer class (inherited)</h3> + +<ul> + +<li>Data</li> + +<li>Dense</li> + +<li>Activation</li> + +<li>Convolution2D</li> + +<li>MaxPooling2D</li> + +<li>AvgPooling2D</li> + +<li>LRN2D</li> + +<li>Dropout</li> + +<li>RBM</li> + +<li>Autoencoder</li> +</ul></div> +<div class="section"> +<h3><a name="Model_class"></a>Model class</h3> +<p>Model class has <tt>jobconf</tt> (JobProto) and <tt>layers</tt> (layer list)</p> +<p>Methods in Model class</p> + +<ul> + +<li> +<p>add</p> + +<ul> + +<li>add Layer into Model</li> + +<li>2 subclasses: Sequential model and Energy model</li> + </ul></li> + +<li> +<p>compile</p> + +<ul> + +<li>set Updater (i.e., optimizer) and Cluster (i.e., topology) components</li> + </ul></li> + +<li> +<p>fit</p> + +<ul> + +<li>set Training data and parameter values for the training + +<ul> + +<li>(optional) set Validatiaon data and parameter values</li> + </ul></li> + +<li>set Train_one_batch component</li> + +<li>specify <tt>with_test</tt> field if a user wants to run SINGA with test data simultaneously.</li> + +<li>[TODO] recieve train/validation results, e.g., accuracy, loss, ppl, etc.</li> + </ul></li> + +<li> +<p>evaluate</p> + +<ul> + +<li>set Testing data and parameter values for the testing</li> + +<li>specify <tt>checkpoint_path</tt> field if a user want to run SINGA only for testing.</li> + +<li>[TODO] recieve test results, e.g., accuracy, loss, ppl, etc.</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="Results"></a>Results</h3> +<p>fit() and evaluate() return train/test results, a dictionary containing</p> + +<ul> + +<li>[key]: step number</li> + +<li>[value]: a list of dictionay + +<ul> + +<li>‘acc’ for accuracy</li> + +<li>‘loss’ for loss</li> + +<li>‘ppl’ for ppl</li> + +<li>‘se’ for squred error</li> + </ul></li> +</ul></div> +<div class="section"> +<h3><a name="Parameter_class"></a>Parameter class</h3> +<p>Users need to set parameter and initial values. For example,</p> + +<ul> + +<li> +<p>Parameter (fields in Param proto)</p> + +<ul> + +<li>lr = (float) // learning rate multiplier, used to scale the learning rate when updating parameters.</li> + +<li>wd = (float) // weight decay multiplier, used to scale the weight decay when updating parameters.</li> + </ul></li> + +<li> +<p>Parameter initialization (fields in ParamGen proto)</p> + +<ul> + +<li>init = (string) // one of the types, ‘uniform’, ‘constant’, ‘gaussian’</li> + +<li>high = (float) // for ‘uniform’</li> + +<li>low = (float) // for ‘uniform’</li> + +<li>value = (float) // for ‘constant’</li> + +<li>mean = (float) // for ‘gaussian’</li> + +<li>std = (float) // for ‘gaussian’</li> + </ul></li> + +<li> +<p>Weight (<tt>w_param</tt>) is ‘gaussian’ with mean=0, std=0.01 at default</p></li> + +<li> +<p>Bias (<tt>b_param</tt>) is ‘constant’ with value=0 at default</p></li> + +<li> +<p>How to update the parameter fields</p> + +<ul> + +<li>for updating Weight, put <tt>w_</tt> in front of field name</li> + +<li>for updating Bias, put <tt>b_</tt> in front of field name</li> + </ul></li> +</ul> +<p>Several ways to set Parameter values</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">parw = Parameter(lr=2, wd=10, init='gaussian', std=0.1) +parb = Parameter(lr=1, wd=0, init='constant', value=0) +m.add(Convolution2D(10, w_param=parw, b_param=parb, ...) +</pre></div></div> + +<div class="source"> +<div class="source"><pre class="prettyprint">m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...) +</pre></div></div> + +<div class="source"> +<div class="source"><pre class="prettyprint">parw = Parameter(init='constant', mean=0) +m.add(Dense(10, w_param=parw, w_lr=1, w_wd=1, b_value=1, ...) +</pre></div></div></div> +<div class="section"> +<h3><a name="Other_classes"></a>Other classes</h3> + +<ul> + +<li>Store</li> + +<li>Algorithm</li> + +<li>Updater</li> + +<li>SGD</li> + +<li>AdaGrad</li> + +<li>Cluster</li> +</ul></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}