Added: websites/staging/singa/trunk/content/v0.2.0/layer.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/layer.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/layer.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,823 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Layers</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Layers</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Layers</h1> +<hr /> +<p>Layer is a core abstraction in SINGA. It performs a variety of feature transformations for extracting high-level features, e.g., loading raw features, parsing RGB values, doing convolution transformation, etc.</p> +<p>The <i>Basic user guide</i> section introduces the configuration of a built-in layer. <i>Advanced user guide</i> explains how to extend the base Layer class to implement users’ functions.</p> +<div class="section"> +<h2><a name="Basic_user_guide"></a>Basic user guide</h2> +<div class="section"> +<h3><a name="Layer_configuration"></a>Layer configuration</h3> +<p>Configuration of two example layers are shown below,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "data" + type: kCSVRecord + store_conf { } +} +layer{ + name: "fc1" + type: kInnerProduct + srclayers: "data" + innerproduct_conf{ } + param{ } +} +</pre></div></div> +<p>There are some common fields for all kinds of layers:</p> + +<ul> + +<li><tt>name</tt>: a string used to differentiate two layers in a neural net.</li> + +<li><tt>type</tt>: an integer used for identifying a specific Layer subclass. The types of built-in layers are listed in LayerType (defined in job.proto). For user-defined layer subclasses, <tt>user_type</tt> should be used instead of <tt>type</tt>.</li> + +<li><tt>srclayers</tt>: names of the source layers. In SINGA, all connections are <a href="neural-net.html">converted</a> to directed connections.</li> + +<li><tt>param</tt>: configuration for a <a href="param.html">Param</a> instance. There can be multiple Param objects in one layer.</li> +</ul> +<p>Different layers may have different configurations. These configurations are defined in <tt><type>_conf</tt>. E.g., “fc1” layer has <tt>innerproduct_conf</tt>. The subsequent sections explain the functionality of each built-in layer and how to configure it.</p></div> +<div class="section"> +<h3><a name="Built-in_Layer_subclasses"></a>Built-in Layer subclasses</h3> +<p>SINGA has provided many built-in layers, which can be used directly to create neural nets. These layers are categorized according to their functionalities,</p> + +<ul> + +<li>Input layers for loading records (e.g., images) from disk files, HDFS or network into memory.</li> + +<li>Neuron layers for feature transformation, e.g., <a href="../api/classsinga_1_1ConvolutionLayer.html">convolution</a>, <a href="../api/classsinga_1_1PoolingLayer.html">pooling</a>, dropout, etc.</li> + +<li>Loss layers for measuring the training objective loss, e.g., Cross Entropy loss or Euclidean loss.</li> + +<li>Output layers for outputting the prediction results (e.g., probabilities of each category) or features into persistent storage, e.g., disk or HDFS.</li> + +<li>Connection layers for connecting layers when the neural net is partitioned.</li> +</ul> +<div class="section"> +<h4><a name="Input_layers"></a>Input layers</h4> +<p>Input layers load training/test data from disk or other places (e.g., HDFS or network) into memory.</p> +<div class="section"> +<h5><a name="StoreInputLayer"></a>StoreInputLayer</h5> +<p><a href="../api/classsinga_1_1StoreInputLayer.html">StoreInputLayer</a> is a base layer for loading data from data store. The data store can be a KVFile or TextFile (LMDB, LevelDB, HDFS, etc., will be supported later). Its <tt>ComputeFeature</tt> function reads batchsize (string:key, string:value) tuples. Each tuple is parsed by a <tt>Parse</tt> function implemented by its subclasses.</p> +<p>The configuration for this layer is in <tt>store_conf</tt>,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">store_conf { + backend: # "kvfile" or "textfile" + path: # path to the data store + batchsize : + ... +} +</pre></div></div></div> +<div class="section"> +<h5><a name="SingleLabelRecordLayer"></a>SingleLabelRecordLayer</h5> +<p>It is a subclass of StoreInputLayer. It assumes the (key, value) tuple loaded from a data store contains a feature vector (and a label) for one data instance. All feature vectors are of the same fixed length. The shape of one instance is configured through the <tt>shape</tt> field, e.g., the following configuration specifies the shape for the CIFAR10 images.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">store_conf { + shape: 3 #channels + shape: 32 #height + shape: 32 #width +} +</pre></div></div> +<p>It may do some preprocessing like <a class="externalLink" href="http://ufldl.stanford.edu/wiki/index.php/Data_Preprocessing";>standardization</a>. The data for preprocessing is loaded by and parsed in a virtual function, which is implemented by its subclasses.</p></div> +<div class="section"> +<h5><a name="RecordInputLayer"></a>RecordInputLayer</h5> +<p>It is a subclass of SingleLabelRecordLayer. It parses the value field from one tuple into a RecordProto, which is generated by Google Protobuf according to common.proto. It can be used to store features for images (e.g., using the pixel field) or other objects (using the data field). The key field is not parsed.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kRecordInput +store_conf { + has_label: # default is true + ... +} +</pre></div></div></div> +<div class="section"> +<h5><a name="CSVInputLayer"></a>CSVInputLayer</h5> +<p>It is a subclass of SingleLabelRecordLayer. The value field from one tuple is parsed as a CSV line (separated by comma). The first number would be parsed as a label if <tt>has_label</tt> is configured in <tt>store_conf</tt>. Otherwise, all numbers would be parsed into one row of the <tt>data_</tt> Blob.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kCSVInput +store_conf { + has_label: # default is true + ... +} +</pre></div></div></div> +<div class="section"> +<h5><a name="ImagePreprocessLayer"></a>ImagePreprocessLayer</h5> +<p>This layer does image preprocessing, e.g., cropping, mirroring and scaling, against the data Blob from its source layer. It deprecates the RGBImageLayer which works on the Record from ShardDataLayer. It still uses the same configuration as RGBImageLayer,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kImagePreprocess +rgbimage_conf { + scale: float + cropsize: int # cropping each image to keep the central part with this size + mirror: bool # mirror the image by set image[i,j]=image[i,len-j] + meanfile: "Image_Mean_File_Path" +} +</pre></div></div></div> +<div class="section"> +<h5><a name="ShardDataLayer_Deprected"></a>ShardDataLayer (Deprected)</h5> +<p>Deprected! Please use ProtoRecordInputLayer or CSVRecordInputLayer.</p> +<p><a href="../api/classsinga_1_1ShardDataLayer.html">ShardDataLayer</a> is a subclass of DataLayer, which reads Records from disk file. The file should be created using <a href="../api/classsinga_1_1DataShard.html">DataShard</a> class. With the data file prepared, users configure the layer as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kShardData +sharddata_conf { + path: "path to data shard folder" + batchsize: int + random_skip: int +} +</pre></div></div> +<p><tt>batchsize</tt> specifies the number of records to be trained for one mini-batch. The first <tt>rand() % random_skip</tt> <tt>Record</tt>s will be skipped at the first iteration. This is to enforce that different workers work on different Records.</p></div> +<div class="section"> +<h5><a name="LMDBDataLayer_Deprected"></a>LMDBDataLayer (Deprected)</h5> +<p>Deprected! Please use ProtoRecordInputLayer or CSVRecordInputLayer.</p> +<p>[LMDBDataLayer] is similar to ShardDataLayer, except that the Records are loaded from LMDB.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kLMDBData +lmdbdata_conf { + path: "path to LMDB folder" + batchsize: int + random_skip: int +} +</pre></div></div></div> +<div class="section"> +<h5><a name="ParserLayer_Deprected"></a>ParserLayer (Deprected)</h5> +<p>Deprected! Please use ProtoRecordInputLayer or CSVRecordInputLayer.</p> +<p>It get a vector of Records from DataLayer and parse features into a Blob.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">virtual void ParseRecords(Phase phase, const vector<Record>& records, Blob<float>* blob) = 0; +</pre></div></div></div> +<div class="section"> +<h5><a name="LabelLayer_Deprected"></a>LabelLayer (Deprected)</h5> +<p>Deprected! Please use ProtoRecordInputLayer or CSVRecordInputLayer.</p> +<p><a href="../api/classsinga_1_1LabelLayer.html">LabelLayer</a> is a subclass of ParserLayer. It parses a single label from each Record. Consequently, it will put $b$ (mini-batch size) values into the Blob. It has no specific configuration fields.</p></div> +<div class="section"> +<h5><a name="MnistImageLayer_Deprected"></a>MnistImageLayer (Deprected)</h5> +<p>Deprected! Please use ProtoRecordInputLayer or CSVRecordInputLayer. [MnistImageLayer] is a subclass of ParserLayer. It parses the pixel values of each image from the MNIST dataset. The pixel values may be normalized as <tt>x/norm_a - norm_b</tt>. For example, if <tt>norm_a</tt> is set to 255 and <tt>norm_b</tt> is set to 0, then every pixel will be normalized into [0, 1].</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kMnistImage +mnistimage_conf { + norm_a: float + norm_b: float +} +</pre></div></div></div> +<div class="section"> +<h5><a name="RGBImageLayer_Deprected"></a>RGBImageLayer (Deprected)</h5> +<p>Deprected! Please use the ImagePreprocessLayer. <a href="../api/classsinga_1_1RGBImageLayer.html">RGBImageLayer</a> is a subclass of ParserLayer. It parses the RGB values of one image from each Record. It may also apply some transformations, e.g., cropping, mirroring operations. If the <tt>meanfile</tt> is specified, it should point to a path that contains one Record for the mean of each pixel over all training images.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kRGBImage +rgbimage_conf { + scale: float + cropsize: int # cropping each image to keep the central part with this size + mirror: bool # mirror the image by set image[i,j]=image[i,len-j] + meanfile: "Image_Mean_File_Path" +} +</pre></div></div></div> +<div class="section"> +<h5><a name="PrefetchLayer"></a>PrefetchLayer</h5> +<p><a href="../api/classsinga_1_1PrefetchLayer.html">PrefetchLayer</a> embeds other input layers to do data prefeching. It will launch a thread to call the embedded layers to load and extract features. It ensures that the I/O task and computation task can work simultaneously. One example PrefetchLayer configuration is,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "prefetch" + type: kPrefetch + sublayers { + name: "data" + type: kShardData + sharddata_conf { } + } + sublayers { + name: "rgb" + type: kRGBImage + srclayers:"data" + rgbimage_conf { } + } + sublayers { + name: "label" + type: kLabel + srclayers: "data" + } + exclude:kTest +} +</pre></div></div> +<p>The layers on top of the PrefetchLayer should use the name of the embedded layers as their source layers. For example, the “rgb” and “label” should be configured to the <tt>srclayers</tt> of other layers.</p></div></div> +<div class="section"> +<h4><a name="Output_Layers"></a>Output Layers</h4> +<p>Output layers get data from their source layers and write them to persistent storage, e.g., disk files or HDFS (to be supported).</p> +<div class="section"> +<h5><a name="RecordOutputLayer"></a>RecordOutputLayer</h5> +<p>This layer gets data (and label if it is available) from its source layer and converts it into records of type RecordProto. Records are written as (key = instance No., value = serialized record) tuples into Store, e.g., KVFile. The configuration of this layer should include the specifics of the Store backend via <tt>store_conf</tt>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "output" + type: kRecordOutput + srclayers: + store_conf { + backend: "kvfile" + path: + } +} +</pre></div></div></div> +<div class="section"> +<h5><a name="CSVOutputLayer"></a>CSVOutputLayer</h5> +<p>This layer gets data (and label if it available) from its source layer and converts it into a string per instance with fields separated by commas (i.e., CSV format). The shape information is not kept in the string. All strings are written into Store, e.g., text file. The configuration of this layer should include the specifics of the Store backend via <tt>store_conf</tt>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "output" + type: kCSVOutput + srclayers: + store_conf { + backend: "textfile" + path: + } +} +</pre></div></div></div></div> +<div class="section"> +<h4><a name="Neuron_Layers"></a>Neuron Layers</h4> +<p>Neuron layers conduct feature transformations.</p> +<div class="section"> +<h5><a name="ConvolutionLayer"></a>ConvolutionLayer</h5> +<p><a href="../api/classsinga_1_1ConvolutionLayer.html">ConvolutionLayer</a> conducts convolution transformation.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kConvolution +convolution_conf { + num_filters: int + kernel: int + stride: int + pad: int +} +param { } # weight/filter matrix +param { } # bias vector +</pre></div></div> +<p>The int value <tt>num_filters</tt> stands for the count of the applied filters; the int value <tt>kernel</tt> stands for the convolution kernel size (equal width and height); the int value <tt>stride</tt> stands for the distance between the successive filters; the int value <tt>pad</tt> pads each with a given int number of pixels border of zeros.</p></div> +<div class="section"> +<h5><a name="InnerProductLayer"></a>InnerProductLayer</h5> +<p><a href="../api/classsinga_1_1InnerProductLayer.html">InnerProductLayer</a> is fully connected with its (single) source layer. Typically, it has two parameter fields, one for weight matrix, and the other for bias vector. It rotates the feature of the source layer (by multiplying with weight matrix) and shifts it (by adding the bias vector).</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kInnerProduct +innerproduct_conf { + num_output: int +} +param { } # weight matrix +param { } # bias vector +</pre></div></div></div> +<div class="section"> +<h5><a name="PoolingLayer"></a>PoolingLayer</h5> +<p><a href="../api/classsinga_1_1PoolingLayer.html">PoolingLayer</a> is used to do a normalization (or averaging or sampling) of the feature vectors from the source layer.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kPooling +pooling_conf { + pool: AVE|MAX // Choose whether use the Average Pooling or Max Pooling + kernel: int // size of the kernel filter + pad: int // the padding size + stride: int // the step length of the filter +} +</pre></div></div> +<p>The pooling layer has two methods: Average Pooling and Max Pooling. Use the enum AVE and MAX to choose the method.</p> + +<ul> + +<li>Max Pooling selects the max value for each filtering area as a point of the result feature blob.</li> + +<li>Average Pooling averages all values for each filtering area at a point of the result feature blob.</li> +</ul></div> +<div class="section"> +<h5><a name="ReLULayer"></a>ReLULayer</h5> +<p><a href="../api/classsinga_1_1ReLULayer.html">ReLuLayer</a> has rectified linear neurons, which conducts the following transformation, <tt>f(x) = Max(0, x)</tt>. It has no specific configuration fields.</p></div> +<div class="section"> +<h5><a name="STanhLayer"></a>STanhLayer</h5> +<p><a href="../api/classsinga_1_1TanhLayer.html">STanhLayer</a> uses the scaled tanh as activation function, i.e., <tt>f(x)=1.7159047* tanh(0.6666667 * x)</tt>. It has no specific configuration fields.</p></div> +<div class="section"> +<h5><a name="SigmoidLayer"></a>SigmoidLayer</h5> +<p>[SigmoidLayer] uses the sigmoid (or logistic) as activation function, i.e., <tt>f(x)=sigmoid(x)</tt>. It has no specific configuration fields.</p></div> +<div class="section"> +<h5><a name="Dropout_Layer"></a>Dropout Layer</h5> +<p><a href="../api/asssinga_1_1DropoutLayer.html">DropoutLayer</a> is a layer that randomly dropouts some inputs. This scheme helps deep learning model away from over-fitting.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kDropout +dropout_conf { + dropout_ratio: float # dropout probability +} +</pre></div></div></div> +<div class="section"> +<h5><a name="LRNLayer"></a>LRNLayer</h5> +<p><a href="../api/classsinga_1_1LRNLayer.html">LRNLayer</a>, (Local Response Normalization), normalizes over the channels.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kLRN +lrn_conf { + local_size: int + alpha: float // scaling parameter + beta: float // exponential number +} +</pre></div></div> +<p><tt>local_size</tt> specifies the quantity of the adjoining channels which will be summed up. For <tt>WITHIN_CHANNEL</tt>, it means the side length of the space region which will be summed up.</p></div></div> +<div class="section"> +<h4><a name="Loss_Layers"></a>Loss Layers</h4> +<p>Loss layers measures the objective training loss.</p> +<div class="section"> +<h5><a name="SoftmaxLossLayer"></a>SoftmaxLossLayer</h5> +<p><a href="../api/classsinga_1_1SoftmaxLossLayer.html">SoftmaxLossLayer</a> is a combination of the Softmax transformation and Cross-Entropy loss. It applies Softmax firstly to get a prediction probability for each output unit (neuron) and compute the cross-entropy against the ground truth. It is generally used as the final layer to generate labels for classification tasks.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kSoftmaxLoss +softmaxloss_conf { + topk: int +} +</pre></div></div> +<p>The configuration field <tt>topk</tt> is for selecting the labels with <tt>topk</tt> probabilities as the prediction results. It is tedious for users to view the prediction probability of every label.</p></div></div> +<div class="section"> +<h4><a name="ConnectionLayer"></a>ConnectionLayer</h4> +<p>Subclasses of ConnectionLayer are utility layers that connects other layers due to neural net partitioning or other cases.</p> +<div class="section"> +<h5><a name="ConcateLayer"></a>ConcateLayer</h5> +<p><a href="../api/classsinga_1_1ConcateLayer.html">ConcateLayer</a> connects more than one source layers to concatenate their feature blob along given dimension.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kConcate +concate_conf { + concate_dim: int // define the dimension +} +</pre></div></div></div> +<div class="section"> +<h5><a name="SliceLayer"></a>SliceLayer</h5> +<p><a href="../api/classsinga_1_1SliceLayer.html">SliceLayer</a> connects to more than one destination layers to slice its feature blob along given dimension.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kSlice +slice_conf { + slice_dim: int +} +</pre></div></div></div> +<div class="section"> +<h5><a name="SplitLayer"></a>SplitLayer</h5> +<p><a href="../api/classsinga_1_1SplitLayer.html">SplitLayer</a> connects to more than one destination layers to replicate its feature blob.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">type: kSplit +split_conf { + num_splits: int +} +</pre></div></div></div> +<div class="section"> +<h5><a name="BridgeSrcLayer__BridgeDstLayer"></a>BridgeSrcLayer & BridgeDstLayer</h5> +<p><a href="../api/classsinga_1_1BridgeSrcLayer.html">BridgeSrcLayer</a> & <a href="../api/classsinga_1_1BridgeDstLayer.html">BridgeDstLayer</a> are utility layers assisting data (e.g., feature or gradient) transferring due to neural net partitioning. These two layers are added implicitly. Users typically do not need to configure them in their neural net configuration.</p></div></div></div> +<div class="section"> +<h3><a name="OutputLayer"></a>OutputLayer</h3> +<p>It write the prediction results or the extracted features into file, HTTP stream or other places. Currently SINGA has not implemented any specific output layer.</p></div></div> +<div class="section"> +<h2><a name="Advanced_user_guide"></a>Advanced user guide</h2> +<p>The base Layer class is introduced in this section, followed by how to implement a new Layer subclass.</p> +<div class="section"> +<h3><a name="Base_Layer_class"></a>Base Layer class</h3> +<div class="section"> +<h4><a name="Members"></a>Members</h4> + +<div class="source"> +<div class="source"><pre class="prettyprint">LayerProto layer_conf_; +Blob<float> data_, grad_; +vector<AuxType> aux_data_; +</pre></div></div> +<p>The base layer class keeps the user configuration in <tt>layer_conf_</tt>. Almost all layers has $b$ (mini-batch size) feature vectors, which are stored in the <tt>data_</tt> <a href="../api/classsinga_1_1Blob.html">Blob</a> (A Blob is a chunk of memory space, proposed in <a class="externalLink" href="http://caffe.berkeleyvision.org/";>Caffe</a>). There are layers without feature vectors; instead, they share the data from source layers. The <tt>grad_</tt> Blob is for storing the gradients of the objective loss w.r.t. the <tt>data_</tt> Blob. It is necessary in <a href="../api/classsinga_1_1BPWorker.html">BP algorithm</a>, hence we put it as a member of the base class. For <a href="../api/classsinga_1_1CDWorker.html">CD algorithm</a>, the <tt>grad_</tt> field is not used; instead, the layers for the RBM model may have a Blob for the positive phase feature and a Blob for the negative phase feature. For a recurrent layer in RNN, one row of the feature blob corresponds to the feature of one internal layer. The <tt>aux_data_</tt> stores the auxiliary data, e.g., image label (set <tt>AuxType</tt> to int). If images have variant number of labels, the AuxType can be defined to <tt>vector<int></tt>. Currently, we hard code <tt>AuxType</tt> to int. It will be added as a template argument of Layer class later.</p> +<p>If a layer has parameters, these parameters are declared using type <a href="param.html">Param</a>. Since some layers do not have parameters, we do not declare any <tt>Param</tt> in the base layer class.</p></div> +<div class="section"> +<h4><a name="Functions"></a>Functions</h4> + +<div class="source"> +<div class="source"><pre class="prettyprint">virtual void Setup(const LayerProto& conf, const vector<Layer*>& srclayers); +virtual void ComputeFeature(int flag, const vector<Layer*>& srclayers) = 0; +virtual void ComputeGradient(int flag, const vector<Layer*>& srclayers) = 0; +</pre></div></div> +<p>The <tt>Setup</tt> function reads user configuration, i.e. <tt>conf</tt>, and information from source layers, e.g., mini-batch size, to set the shape of the <tt>data_</tt> (and <tt>grad_</tt>) field as well as some other layer specific fields. <!-- - +If `npartitions` is larger than 1, then +users need to reduce the sizes of `data_`, `grad_` Blobs or Param objects. For +example, if the `partition_dim=0` and there is no source layer, e.g., this +layer is a (bottom) data layer, then its `data_` and `grad_` Blob should have +`b/npartitions` feature vectors; If the source layer is also partitioned on +dimension 0, then this layer should have the same number of feature vectors as +the source layer. More complex partition cases are discussed in +[Neural net partitioning](neural-net.html#neural-net-partitioning). Typically, the +Setup function just set the shapes of `data_` Blobs and Param objects. --> Memory will not be allocated until computation over the data structure happens.</p> +<p>The <tt>ComputeFeature</tt> function evaluates the feature blob by transforming (e.g. convolution and pooling) features from the source layers. <tt>ComputeGradient</tt> computes the gradients of parameters associated with this layer. These two functions are invoked by the <a href="train-one-batch.html">TrainOneBatch</a> function during training. Hence, they should be consistent with the <tt>TrainOneBatch</tt> function. Particularly, for feed-forward and RNN models, they are trained using <a href="train-one-batch.html#back-propagation">BP algorithm</a>, which requires each layer’s <tt>ComputeFeature</tt> function to compute <tt>data_</tt> based on source layers, and requires each layer’s <tt>ComputeGradient</tt> to compute gradients of parameters and source layers’ <tt>grad_</tt>. For energy models, e.g., RBM, they are trained by <a href="train-one-batch.html#contrastive-divergence">CD algorithm</a>, which requires each layer’s <tt>ComputeFeature</tt> f unction to compute the feature vectors for the positive phase or negative phase depending on the <tt>phase</tt> argument, and requires the <tt>ComputeGradient</tt> function to only compute parameter gradients. For some layers, e.g., loss layer or output layer, they can put the loss or prediction result into the <tt>metric</tt> argument, which will be averaged and displayed periodically.</p></div></div> +<div class="section"> +<h3><a name="Implementing_a_new_Layer_subclass"></a>Implementing a new Layer subclass</h3> +<p>Users can extend the Layer class or other subclasses to implement their own feature transformation logics as long as the two virtual functions are overridden to be consistent with the <tt>TrainOneBatch</tt> function. The <tt>Setup</tt> function may also be overridden to read specific layer configuration.</p> +<p>The <a href="rnn.html">RNNLM</a> provides a couple of user-defined layers. You can refer to them as examples.</p> +<div class="section"> +<h4><a name="Layer_specific_protocol_message"></a>Layer specific protocol message</h4> +<p>To implement a new layer, the first step is to define the layer specific configuration. Suppose the new layer is <tt>FooLayer</tt>, the layer specific google protocol message <tt>FooLayerProto</tt> should be defined as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in user.proto +package singa +import "job.proto" +message FooLayerProto { + optional int32 a = 1; // specific fields to the FooLayer +} +</pre></div></div> +<p>In addition, users need to extend the original <tt>LayerProto</tt> (defined in job.proto of SINGA) to include the <tt>foo_conf</tt> as follows.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">extend LayerProto { + optional FooLayerProto foo_conf = 101; // unique field id, reserved for extensions +} +</pre></div></div> +<p>If there are multiple new layers, then each layer that has specific configurations would have a <tt><type>_conf</tt> field and takes one unique extension number. SINGA has reserved enough extension numbers, e.g., starting from 101 to 1000.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># job.proto of SINGA +LayerProto { + ... + extensions 101 to 1000; +} +</pre></div></div> +<p>With user.proto defined, users can use <a class="externalLink" href="https://developers.google.com/protocol-buffers/";>protoc</a> to generate the <tt>user.pb.cc</tt> and <tt>user.pb.h</tt> files. In users’ code, the extension fields can be accessed via,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">auto conf = layer_proto_.GetExtension(foo_conf); +int a = conf.a(); +</pre></div></div> +<p>When defining configurations of the new layer (in job.conf), users should use <tt>user_type</tt> for its layer type instead of <tt>type</tt>. In addition, <tt>foo_conf</tt> should be enclosed in brackets.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "foo" + user_type: "kFooLayer" # Note user_type of user-defined layers is string + [foo_conf] { # Note there is a pair of [] for extension fields + a: 10 + } +} +</pre></div></div></div> +<div class="section"> +<h4><a name="New_Layer_subclass_declaration"></a>New Layer subclass declaration</h4> +<p>The new layer subclass can be implemented like the built-in layer subclasses.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class FooLayer : public singa::Layer { + public: + void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override; + void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; + void ComputeGradient(int flag, const vector<Layer*>& srclayers) override; + + private: + // members +}; +</pre></div></div> +<p>Users must override the two virtual functions to be called by the <tt>TrainOneBatch</tt> for either BP or CD algorithm. Typically, the <tt>Setup</tt> function will also be overridden to initialize some members. The user configured fields can be accessed through <tt>layer_conf_</tt> as shown in the above paragraphs.</p></div> +<div class="section"> +<h4><a name="New_Layer_subclass_registration"></a>New Layer subclass registration</h4> +<p>The newly defined layer should be registered in <a class="externalLink" href="http://singa.incubator.apache.org/docs/programming-guide";>main.cc</a> by adding</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">driver.RegisterLayer<FooLayer, std::string>("kFooLayer"); // "kFooLayer" should be matched to layer configurations in job.conf. +</pre></div></div> +<p>After that, the <a href="neural-net.html">NeuralNet</a> can create instances of the new Layer subclass.</p></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
Added: websites/staging/singa/trunk/content/v0.2.0/mesos.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/mesos.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/mesos.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,451 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Distributed Training on Mesos</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Distributed Training on Mesos</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Distributed Training on Mesos</h1> +<p>This guide explains how to start SINGA distributed training on a Mesos cluster. It assumes that both Mesos and HDFS are already running, and every node has SINGA installed. We assume the architecture depicted below, in which a cluster nodes are Docker container. Refer to <a href="docker.html">Docker guide</a> for details of how to start individual nodes and set up network connection between them (make sure <a class="externalLink" href="http://weave.works/guides/weave-docker-ubuntu-simple.html";>weave</a> is running at each node, and the cluster’s headnode is running in container <tt>node0</tt>)</p> +<p><img src="http://www.comp.nus.edu.sg/~dinhtta/files/singa_mesos.png"; alt="Nothing" /></p> +<hr /> +<div class="section"> +<h2><a name="Start_HDFS_and_Mesos"></a>Start HDFS and Mesos</h2> +<p>Go inside each container, using: <tt> +docker exec -it nodeX /bin/bash +</tt> and configure it as follows:</p> + +<ul> + +<li> +<p>On container <tt>node0</tt></p> + +<div class="source"> +<div class="source"><pre class="prettyprint">hadoop namenode -format +hadoop-daemon.sh start namenode +/opt/mesos-0.22.0/build/bin/mesos-master.sh --work_dir=/opt --log_dir=/opt --quiet > /dev/null & +zk-service.sh start +</pre></div></div></li> + +<li> +<p>On container <tt>node1, node2, ...</tt></p> + +<div class="source"> +<div class="source"><pre class="prettyprint">hadoop-daemon.sh start datanode +/opt/mesos-0.22.0/build/bin/mesos-slave.sh --master=node0:5050 --log_dir=/opt --quiet > /dev/null & +</pre></div></div></li> +</ul> +<p>To check if the setup has been successful, check that HDFS namenode has registered <tt>N</tt> datanodes, via:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">hadoop dfsadmin -report +</pre></div></div> +<div class="section"> +<div class="section"> +<h4><a name="Mesos_logs"></a>Mesos logs</h4> +<p>Mesos logs are stored at <tt>/opt/lt-mesos-master.INFO</tt> on <tt>node0</tt> and <tt>/opt/lt-mesos-slave.INFO</tt> at other nodes.</p> +<hr /></div></div></div> +<div class="section"> +<h2><a name="Starting_SINGA_training_on_Mesos"></a>Starting SINGA training on Mesos</h2> +<p>Assumed that Mesos and HDFS are already started, SINGA job can be launched at <b>any</b> container.</p> +<div class="section"> +<div class="section"> +<h4><a name="Launching_job"></a>Launching job</h4> + +<ol style="list-style-type: decimal"> + +<li>Log in to any container, then cd incubator-singa/tool/mesos <a name="job_start"></a></li> + +<li>Check that configuration files are correct: + +<ul> + +<li><tt>scheduler.conf</tt> contains information about the master nodes</li> + +<li><tt>singa.conf</tt> contains information about Zookeeper node0</li> + +<li>Job configuration file <tt>job.conf</tt> <b>contains full path to the examples directories (NO RELATIVE PATH!).</b></li> + </ul></li> + +<li> +<p>Start the job:</p> + +<ul> + +<li>If starting for the first time:</li> + </ul> + +<div class="source"> +<div class="source"><pre class="prettyprint"> ./scheduler <job config file> -scheduler_conf <scheduler config file> -singa_conf <SINGA config file> +</pre></div></div> + +<ul> + +<li>If not the first time:</li> + </ul> + +<div class="source"> +<div class="source"><pre class="prettyprint"> ./scheduler <job config file> +</pre></div></div></li> +</ol> +<p><b>Notes.</b> Each running job is given a <tt>frameworkID</tt>. Look for the log message of the form:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> Framework registered with XXX-XXX-XXX-XXX-XXX-XXX +</pre></div></div></div> +<div class="section"> +<h4><a name="Monitoring_and_Debugging"></a>Monitoring and Debugging</h4> +<p>Each Mesos job is given a <tt>frameworkID</tt> and a <i>sandbox</i> directory is created for each job. The directory is in the specified <tt>work_dir</tt> (or <tt>/tmp/mesos</tt>) by default. For example, the error during SINGA execution can be found at:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> /tmp/mesos/slaves/xxxxx-Sx/frameworks/xxxxx/executors/SINGA_x/runs/latest/stderr +</pre></div></div> +<p>Other artifacts, like files downloaded from HDFS (<tt>job.conf</tt>) and <tt>stdout</tt> can be found in the same directory.</p></div> +<div class="section"> +<h4><a name="Stopping"></a>Stopping</h4> +<p>There are two way to kill the running job:</p> + +<ol style="list-style-type: decimal"> + +<li> +<p>If the scheduler is running in the foreground, simply kill it (using <tt>Ctrl-C</tt>, for example).</p></li> + +<li> +<p>If the scheduler is running in the background, kill it using Mesos’s REST API:</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"> curl -d "frameworkId=XXX-XXX-XXX-XXX-XXX-XXX" -X POST http://<master>/master/shutdown +</pre></div></div></li> +</ol></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/v0.2.0/mlp.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/mlp.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/mlp.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,538 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – MLP Example</title> + <link rel="stylesheet" href="../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../css/site.css" /> + <link rel="stylesheet" href="../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../docs/index.html" title="English">English</a> +</li> + <li> <a href="../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">MLP Example</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>MLP Example</h1> +<hr /> +<p>Multilayer perceptron (MLP) is a subclass of feed-forward neural networks. A MLP typically consists of multiple directly connected layers, with each layer fully connected to the next one. In this example, we will use SINGA to train a <a class="externalLink" href="http://arxiv.org/abs/1003.0358";>simple MLP model proposed by Ciresan</a> for classifying handwritten digits from the <a class="externalLink" href="http://yann.lecun.com/exdb/mnist/";>MNIST dataset</a>.</p> +<div class="section"> +<h2><a name="Running_instructions"></a>Running instructions</h2> +<p>Please refer to the <a href="installation.html">installation</a> page for instructions on building SINGA, and the <a href="quick-start.html">quick start</a> for instructions on starting zookeeper.</p> +<p>We have provided scripts for preparing the training and test dataset in <i>examples/cifar10/</i>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in examples/mnist +$ cp Makefile.example Makefile +$ make download +$ make create +</pre></div></div> +<div class="section"> +<h3><a name="Training_on_CPU"></a>Training on CPU</h3> +<p>After the datasets are prepared, we start the training by</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -conf examples/mnist/job.conf +</pre></div></div> +<p>After it is started, you should see output like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">Record job information to /tmp/singa-log/job-info/job-1-20150817-055231 +Executing : ./singa -conf /xxx/incubator-singa/examples/mnist/job.conf -singa_conf /xxx/incubator-singa/conf/singa.conf -singa_job 1 +E0817 07:15:09.211885 34073 cluster.cc:51] proc #0 -> 192.168.5.128:49152 (pid = 34073) +E0817 07:15:14.972231 34114 server.cc:36] Server (group = 0, id = 0) start +E0817 07:15:14.972520 34115 worker.cc:134] Worker (group = 0, id = 0) start +E0817 07:15:24.462602 34073 trainer.cc:373] Test step-0, loss : 2.341021, accuracy : 0.109100 +E0817 07:15:47.341076 34073 trainer.cc:373] Train step-0, loss : 2.357269, accuracy : 0.099000 +E0817 07:16:07.173364 34073 trainer.cc:373] Train step-10, loss : 2.222740, accuracy : 0.201800 +E0817 07:16:26.714855 34073 trainer.cc:373] Train step-20, loss : 2.091030, accuracy : 0.327200 +E0817 07:16:46.590946 34073 trainer.cc:373] Train step-30, loss : 1.969412, accuracy : 0.442100 +E0817 07:17:06.207080 34073 trainer.cc:373] Train step-40, loss : 1.865466, accuracy : 0.514800 +E0817 07:17:25.890033 34073 trainer.cc:373] Train step-50, loss : 1.773849, accuracy : 0.569100 +E0817 07:17:51.208935 34073 trainer.cc:373] Test step-60, loss : 1.613709, accuracy : 0.662100 +E0817 07:17:53.176766 34073 trainer.cc:373] Train step-60, loss : 1.659150, accuracy : 0.652600 +E0817 07:18:12.783370 34073 trainer.cc:373] Train step-70, loss : 1.574024, accuracy : 0.666000 +E0817 07:18:32.904942 34073 trainer.cc:373] Train step-80, loss : 1.529380, accuracy : 0.670500 +E0817 07:18:52.608111 34073 trainer.cc:373] Train step-90, loss : 1.443911, accuracy : 0.703500 +E0817 07:19:12.168465 34073 trainer.cc:373] Train step-100, loss : 1.387759, accuracy : 0.721000 +E0817 07:19:31.855865 34073 trainer.cc:373] Train step-110, loss : 1.335246, accuracy : 0.736500 +E0817 07:19:57.327133 34073 trainer.cc:373] Test step-120, loss : 1.216652, accuracy : 0.769900 +</pre></div></div> +<p>After the training of some steps (depends on the setting) or the job is finished, SINGA will <a href="checkpoint.html">checkpoint</a> the model parameters.</p></div> +<div class="section"> +<h3><a name="Training_on_GPU"></a>Training on GPU</h3> +<p>To train this example model on GPU, just add a field in the configuration file for the GPU device,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># job.conf +gpu: 0 +</pre></div></div></div> +<div class="section"> +<h3><a name="Training_using_Python_script"></a>Training using Python script</h3> +<p>The python helpers come with SINGA 0.2 make it easy to configure the job. For example the job.conf is replaced with a simple python script mnist_mlp.py which has about 30 lines of code following the <a class="externalLink" href="http://keras.io/";>Keras API</a>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -exec tool/python/examples/mnist_mlp.py +</pre></div></div></div></div> +<div class="section"> +<h2><a name="Details"></a>Details</h2> +<p>To train a model in SINGA, you need to prepare the datasets, and a job configuration which specifies the neural net structure, training algorithm (BP or CD), SGD update algorithm (e.g. Adagrad), number of training/test steps, etc.</p> +<div class="section"> +<h3><a name="Data_preparation"></a>Data preparation</h3> +<p>Before using SINGA, you need to write a program to pre-process the dataset you use to a format that SINGA can read. Please refer to the <a href="data.html">Data Preparation</a> to get details about preparing this MNIST dataset.</p></div> +<div class="section"> +<h3><a name="Neural_net"></a>Neural net</h3> + +<div style="text-align: center"> +<img src="../images/example-mlp.png" style="width: 230px" alt="" /> +<br /><b>Figure 1 - Net structure of the MLP example. </b></img> +</div> +<p>Figure 1 shows the structure of the simple MLP model, which is constructed following <a class="externalLink" href="http://arxiv.org/abs/1003.0358";>Ciresan’s paper</a>. The dashed circle contains two layers which represent one feature transformation stage. There are 6 such stages in total. They sizes of the <a href="layer.html#innerproductlayer">InnerProductLayer</a>s in these circles decrease from 2500->2000->1500->1000->500->10.</p> +<p>Next we follow the guide in <a href="neural-net.html">neural net page</a> and <a href="layer.html">layer page</a> to write the neural net configuration.</p> + +<ul> + +<li> +<p>We configure an input layer to read the training/testing records from a disk file.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer { + name: "data" + type: kRecordInput + store_conf { + backend: "kvfile" + path: "examples/mnist/train_data.bin" + random_skip: 5000 + batchsize: 64 + shape: 784 + std_value: 127.5 + mean_value: 127.5 + } + exclude: kTest + } + +layer { + name: "data" + type: kRecordInput + store_conf { + backend: "kvfile" + path: "examples/mnist/test_data.bin" + batchsize: 100 + shape: 784 + std_value: 127.5 + mean_value: 127.5 + } + exclude: kTrain + } +</pre></div></div></li> +</ul> + +<ul> + +<li> +<p>All <a href="layer.html#innerproductlayer">InnerProductLayer</a>s are configured similarly as,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer{ + name: "fc1" + type: kInnerProduct + srclayers:"data" + innerproduct_conf{ + num_output: 2500 + } + param{ + name: "w1" + ... + } + param{ + name: "b1" + .. + } +} +</pre></div></div> +<p>with the <tt>num_output</tt> decreasing from 2500 to 10.</p></li> + +<li> +<p>A <a href="layer.html#stanhlayer">STanhLayer</a> is connected to every InnerProductLayer except the last one. It transforms the feature via scaled tanh function.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer{ + name: "tanh1" + type: kSTanh + srclayers:"fc1" +} +</pre></div></div></li> + +<li> +<p>The final <a href="layer.html#softmaxloss">Softmax loss layer</a> connects to LabelLayer and the last STanhLayer.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">layer{ + name: "loss" + type:kSoftmaxLoss + softmaxloss_conf{ topk:1 } + srclayers:"fc6" + srclayers:"data" +} +</pre></div></div></li> +</ul></div> +<div class="section"> +<h3><a name="Updater"></a>Updater</h3> +<p>The <a href="updater.html#updater">normal SGD updater</a> is selected. The learning rate shrinks by 0.997 every 60 steps (i.e., one epoch).</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">updater{ + type: kSGD + learning_rate{ + base_lr: 0.001 + type : kStep + step_conf{ + change_freq: 60 + gamma: 0.997 + } + } +} +</pre></div></div></div> +<div class="section"> +<h3><a name="TrainOneBatch_algorithm"></a>TrainOneBatch algorithm</h3> +<p>The MLP model is a feed-forward model, hence <a href="train-one-batch#back-propagation">Back-propagation algorithm</a> is selected.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">train_one_batch { + alg: kBP +} +</pre></div></div></div> +<div class="section"> +<h3><a name="Cluster_setting"></a>Cluster setting</h3> +<p>The following configuration set a single worker and server for training. <a href="frameworks.html">Training frameworks</a> page introduces configurations of a couple of distributed training frameworks.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">cluster { + nworker_groups: 1 + nserver_groups: 1 +} +</pre></div></div></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}
![http://www.comp.nus.edu.sg/~dinhtta/files/singa_mesos.png"](http://www.comp.nus.edu.sg/~dinhtta/files/singa_mesos.png"){kind=link}