Added: websites/staging/singa/trunk/content/v0.2.0/zh/overview.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/zh/overview.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/zh/overview.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,371 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – ç®ä»</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../../docs/index.html" title="English">English</a> +</li> + <li> <a href="../../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">ç®ä»</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>简介</h1> +<hr /> +<p>SINGA是一个通用的分布式深度学习平台，面向训练大规模数据集上的大型深度学习模型。其设计基于一种直观的编程模型，即深度学习中层（layer）的抽象。SINGA支持大部分深度学习模型，包括卷积神经网络（CNN）、受限波尔兹曼模型（RBM）和循环神经网络（RNN）等，为用户提供许多可直接使用&# x7684;内建层。SINGA架构灵活，支持同步训练、异步训练和混合式训练。为了并行地训练深度学习模型，SINGA支持不同的神经网络划分机制，即批次维度划分（batch dimension partition），特征维度划分（feature dimension partition）和多维度混合划分（hybrid partition）。</p> +<div class="section"> +<h2><a name="a"></a>目标</h2> +<p>作为一个分布式系统，SINGA的首要目标就是具有良好的可扩展性。换言之，SINGA希望在准确度一定的情况下，通过利用更多的计算资源（即计算机）减少模型的训练时间。</p> +<p>SINGA的另一个目标是易用性。对程序员来说，开发和训练深层的复杂结构的深度学习模型十分困难。分布式训练又进一步增加了程序员的负担，比如：数据和模型划分，网络通信等。因此，提供一个易用的编程模型是十分重要的，可以让程序员在实现自己的深度学习模型和算法时不必€ 03;虑底层的分布式平台。</p></div> +<div class="section"> +<h2><a name="a"></a>原则</h2> +<p>扩展性是分布式深度学习的重要研究问题。SINGA提供了一个利用不同训练框架扩展性的通用平台。同步训练框架可提高每次训练迭代的效率，同时异步训练框架可加快模型收敛。在预算（比如：集群规模）一定的情况下，用户可以运行一个混合训练框架，在效率和收敛速度之间权衡，N e5;取得最大的扩展性。</p> +<p>SINGA的编程模型是基于层的抽象而设计，这对于深度学习模型而言是十分直观的。很多深度学习模型可以直接用这种编程模型来表达和训练。</p></div> +<div class="section"> +<h2><a name="a"></a>系统概览</h2> +<p><img src="../../images/sgd.png" align="center" width="400px" alt="" /> <span><b>图1 - 随机梯度下降流程图</b></span></p> +<p>对于特定的任务，训练一个深度学习模型就是找出能产生良好特征的转换函数中的最优参数。参数的合适程度由损失函数来度量，如交叉熵损失函数<a class="externalLink" href="https://en.wikipedia.org/wiki/Cross_entropy";>（Cross-Entropy Loss）</a>。因为损失函数一般都是非线性和非凸的，难以得到一个封闭形式解。典型的解决方案是使用随机梯度& #x4e0b;降（SGD）算法，首先随机地初始化参数，然后迭代地更新参数值，减小损失函数值，如图1所示。</p> +<p><img src="../../images/overview.png" align="center" width="400px" alt="" /> <span><b>图2 - SINGA 概览</b></span></p> +<p>SINGA使用随机梯度下降来训练深度学习模型中的参数。训练的作业会被分配到作业者（worker）单元和服务器（server）单元，如图2所示。每次迭代中，作业者调用 <i>TrainOneBatch</i> 函数计算参数的梯度。 <i>TainOneBatch</i> 函数以一个神经网络对象 <i>NeuralNet</i> 作为输入，以一定的顺序遍历 <i>NeuralNet</i>。计算得到的梯度将发送给局部&# x7684;根节点（stub），该局部根节点聚合请求并转发给对应的服务器请求更新。服务器给作业者发回更新后的参数，进入下一轮迭代。</p></div> +<div class="section"> +<h2><a name="a"></a>作业提交</h2> +<p>在SINGA中提交一个作业（即训练一个深度学习模型），用户需要将任务配置传给<a href="programming-guide.html">主函数</a>中的SINGA driver。作业配置需要明确图2中的四个主要部分：</p> + +<ul> + +<li><a href="neural-net.html">NeuralNet</a> ：描述神经网络结构，包括每层的具体设置和层与层的连接关系；</li> + +<li><a href="train-one-batch.html">TrainOneBatch</a> ：该算法需要根据不同的模型类别而定制;</li> + +<li><a href="updater.html">Updater</a> ：定义服务器端更新参数的协议；</li> + +<li><a href="distributed-training.html">Cluster Topology</a> ：指定服务器和工作者的分布式拓扑架构。</li> +</ul> +<p>作业提交过程跟Hadoop的作业提交类似，用户在主函数中配置好自己的任务，设置mapper和reducer等。在Hadoop中，用户可用自己实现的或者内建的mapper和reducer来配置他们的作业；类似地，在SINGA中，用户也可以用自己实现的或者内建的layer，updater等来配置他们的作业。</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
Added: websites/staging/singa/trunk/content/v0.2.0/zh/programming-guide.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/zh/programming-guide.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/zh/programming-guide.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,407 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – ç¼ç¨æå</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../../docs/index.html" title="English">English</a> +</li> + <li> <a href="../../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">ç¼ç¨æå</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>编程指南</h1> +<hr /> +<p>要提交一个训练作业，用户需要提供图1中的四个部分的配置：</p> + +<ul> + +<li><a href="neural-net.html">NeuralNet</a> ：描述神经网络结构，包括每层的具体设置和层与层的连接关系；</li> + +<li><a href="train-one-batch.html">TrainOneBatch</a> ：该算法需要根据不同的模型类别而定制;</li> + +<li><a href="updater.html">Updater</a> ：定义服务器端更新参数的协议；</li> + +<li><a href="distributed-training.html">Cluster Topology</a> ：指定服务器和工作者的分布式拓扑架构。</li> +</ul> +<p><i>初级用户指南</i> 将介绍如何利用内建层提交一个训练作业，而 <i>高级用户指南</i> 将详细介绍如何编写用户自己的主函数并注册自己实现的组件。此外，高级用户和初级用户对训练数据集的<a href="data.html">处理</a>方式是相同的。</p> +<p><img src="../../images/overview.png" align="center" width="400px" alt="" /> <span><b>图 1 - SINGA 概览</b></span></p> +<div class="section"> +<h2><a name="a"></a>初级用户指南</h2> +<p>用户可以使用SINGA提供的主函数提交训练作业。对于这种情况，用户必须在命令行中提供根据 <a href="../api/classsinga_1_1JobProto.html">JobProto</a> 设置的作业配置文件，</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -conf <path to job conf> [-resume] +</pre></div></div> +<p><tt>-resume</tt> 表示从上次的<a href="checkpoint.html">检查点（checkpoint）</a>继续训练。 <a href="mlp.html">MLP</a> 模型和 <a href="cnn.html">CNN</a> 模型使用内建层提交训练作业。请阅读相关页面，查看它们的作业配置文件，这些页面会介绍每个组件配置的细节。</p></div> +<div class="section"> +<h2><a name="a"></a>高级用户指南</h2> +<p>如果用户的模型中包含一些自己定义的组件，比如<a href="updater.html">Updater</a>，用户必须自己编写主函数注册这些组件，跟Hadoop的主函数类似。一般地，主函数应该</p> + +<ul> + +<li>初始化SINGA，如：设置日志；</li> + +<li>注册用户自定义组件；</li> + +<li>创建作业配置并传递给SINGA driver。</li> +</ul> +<p>主函数示例</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">#include "singa.h" +#include "user.h" // header for user code + +int main(int argc, char** argv) { + singa::Driver driver; + driver.Init(argc, argv); + bool resume; + // parse resume option from argv. + + // register user defined layers + driver.RegisterLayer<FooLayer>(kFooLayer); + // register user defined updater + driver.RegisterUpdater<FooUpdater>(kFooUpdater); + ... + auto jobConf = driver.job_conf(); + // update jobConf + + driver.Train(resume, jobConf); + return 0; +} +</pre></div></div> +<p>driver 类’ <tt>Init</tt> 方法加载用户在命令行参数中 （<tt>-conf <job conf></tt>）提供的作业配置文件（至少包含集群拓扑结构），并返回<tt>jobConf</tt>给用户，用户可更新和添加神经网络或者Updater的配置。如果定义了Layer、Updater、Worker或者Param的子类，用户需要通过driver为它们注册。最后，作业配置会被提交到driver，由driver启动训练。</p> +<p>将来我们会提供类似<a class="externalLink" href="https://github.com/fchollet/keras";>keras</a> 的帮助工具，使作业配置更加简单。</p> +<p>用户需要使用SINGA库(<i>.libs/libsinga.so</i>)编译和链接自己的代码（如：layer的实现和主函数），得到可执行文件，如名为<i>mysinga</i> 的文件。执行以下命令启动该程序，用户需要将<i>mysinga</i> 和作业配置文件的路径传给 <i>./bin/singa-run.sh</i> 。</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">./bin/singa-run.sh -conf <path to job conf> -exec <path to mysinga> [other arguments] +</pre></div></div> +<p><a href="rnn.html">RNN application</a> 提供了一个完整的实现主函数训练特定RNN模型的例子。</p></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html> Added: websites/staging/singa/trunk/content/v0.2.0/zh/rnn.html ============================================================================== --- websites/staging/singa/trunk/content/v0.2.0/zh/rnn.html (added) +++ websites/staging/singa/trunk/content/v0.2.0/zh/rnn.html Tue Apr 12 06:24:50 2016 @@ -0,0 +1,674 @@ +<!DOCTYPE html> +<!-- + | Generated by Apache Maven Doxia at 2016-04-12 + | Rendered using Apache Maven Fluido Skin 1.4 +--> +<html xmlns="http://www.w3.org/1999/xhtml"; xml:lang="en" lang="en"> + <head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <meta name="Date-Revision-yyyymmdd" content="20160412" /> + <meta http-equiv="Content-Language" content="en" /> + <title>Apache SINGA – Recurrent Neural Networks for Language Modelling</title> + <link rel="stylesheet" href="../../css/apache-maven-fluido-1.4.min.css" /> + <link rel="stylesheet" href="../../css/site.css" /> + <link rel="stylesheet" href="../../css/print.css" media="print" /> + + + + + + <script type="text/javascript" src="../../js/apache-maven-fluido-1.4.min.js"></script> + + + +<script src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; type="text/javascript"></script> + + +<script type="text/x-mathjax-config">MathJax.Hub.Config({tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]}});</script> + + +<meta content="Apache SINGA" name="author"/> + + +<meta content="SINGA, Deep Learning, Distributed training" name="keywords"/> + + </head> + <body class="topBarEnabled"> + + + + + + + <a href="https://github.com/apache/incubator-singa";> + <img style="position: absolute; top: 0; right: 0; border: 0; z-index: 10000;" + src="https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"; + alt="Fork me on GitHub"> + </a> + + + + + + <div id="topbar" class="navbar navbar-fixed-top navbar-inverse"> + <div class="navbar-inner"> + <div class="container-fluid"> + <a data-target=".nav-collapse" data-toggle="collapse" class="btn btn-navbar"> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </a> + + <ul class="nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Apache SINGA <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../docs/overview.html" title="Introduction">Introduction</a> +</li> + + <li> <a href="../../docs/quick-start.html" title="Quick Start">Quick Start</a> +</li> + + <li> <a href="../../downloads.html" title="Downloads">Downloads</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentaion <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li class="dropdown-submenu"> + <a href="../../docs/index.html" title="v0.3.0">v0.3.0</a> + <ul class="dropdown-menu"> + <li> <a href="../../docs/index.html" title="English">English</a> +</li> + <li> <a href="../../docs/zh/index.html" title="ä¸æ">ä¸æ</a> +</li> + <li> <a href="../../docs/jp/index.html" title="æ¥æ¬èª">æ¥æ¬èª</a> +</li> + <li> <a href="../../docs/kr/index.html" title="íêµì´">íêµì´</a> +</li> + </ul> + </li> + + <li> <a href="../../v0.2.0/index.html" title="v0.2.0">v0.2.0</a> +</li> + + <li> <a href="../../v0.1.0/index.html" title="v0.1.0">v0.1.0</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Development <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../develop/schedule.html" title="Schedule">Schedule</a> +</li> + + <li class="dropdown-submenu"> + <a href="../../develop/how-contribute.html" title="How to Contribute">How to Contribute</a> + <ul class="dropdown-menu"> + <li> <a href="../../develop/contribute-code.html" title="Code">Code</a> +</li> + <li> <a href="../../develop/contribute-docs.html" title="Documentation">Documentation</a> +</li> + </ul> + </li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">Community <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="../../community/source-repository.html" title="Source Repository">Source Repository</a> +</li> + + <li> <a href="../../community/mail-lists.html" title="Mailing Lists">Mailing Lists</a> +</li> + + <li> <a href="../../community/issue-tracking.html" title="Issue Tracking">Issue Tracking</a> +</li> + + <li> <a href="../../community/team-list.html" title="SINGA Team">SINGA Team</a> +</li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown">External Links <b class="caret"></b></a> + <ul class="dropdown-menu"> + + <li> <a href="http://www.apache.org/"; title="Apache Software Foundation">Apache Software Foundation</a> +</li> + + <li> <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; title="NUS Site">NUS Site</a> +</li> + </ul> + </li> + </ul> + + + + + </div> + + </div> + </div> + </div> + + <div class="container-fluid"> + <div id="banner"> + <div class="pull-left"> + <a href="../../index.html" id="bannerLeft" title="Apache SINGA"> + <img src="../../images/singa-logo.png" alt="Apache SINGA"/> + </a> + </div> + <div class="pull-right"> <div id="bannerRight"> + <img src="../../images/singa-title.png" alt="Apache SINGA"/> + </div> + </div> + <div class="clear"><hr/></div> + </div> + + <div id="breadcrumbs"> + <ul class="breadcrumb"> + + + <li class=""> + <a href="../../index.html" title="Apache SINGA"> + Apache SINGA</a> + <span class="divider">/</span> + </li> + <li class="active ">Recurrent Neural Networks for Language Modelling</li> + + + + + </ul> + </div> + + + <div class="row-fluid"> + <div id="leftColumn" class="span2"> + <div class="well sidebar-nav"> + + + <ul class="nav nav-list"> + <li class="nav-header">Apache SINGA</li> + + <li> + + <a href="../../docs/overview.html" title="Introduction"> + <span class="none"></span> + Introduction</a> + </li> + + <li> + + <a href="../../docs/quick-start.html" title="Quick Start"> + <span class="none"></span> + Quick Start</a> + </li> + + <li> + + <a href="../../downloads.html" title="Downloads"> + <span class="none"></span> + Downloads</a> + </li> + <li class="nav-header">Documentaion</li> + + <li> + + <a href="../../docs/index.html" title="v0.3.0"> + <span class="icon-chevron-right"></span> + v0.3.0</a> + </li> + + <li> + + <a href="../../v0.2.0/index.html" title="v0.2.0"> + <span class="none"></span> + v0.2.0</a> + </li> + + <li> + + <a href="../../v0.1.0/index.html" title="v0.1.0"> + <span class="none"></span> + v0.1.0</a> + </li> + <li class="nav-header">Development</li> + + <li> + + <a href="../../develop/schedule.html" title="Schedule"> + <span class="none"></span> + Schedule</a> + </li> + + <li> + + <a href="../../develop/how-contribute.html" title="How to Contribute"> + <span class="icon-chevron-right"></span> + How to Contribute</a> + </li> + <li class="nav-header">Community</li> + + <li> + + <a href="../../community/source-repository.html" title="Source Repository"> + <span class="none"></span> + Source Repository</a> + </li> + + <li> + + <a href="../../community/mail-lists.html" title="Mailing Lists"> + <span class="none"></span> + Mailing Lists</a> + </li> + + <li> + + <a href="../../community/issue-tracking.html" title="Issue Tracking"> + <span class="none"></span> + Issue Tracking</a> + </li> + + <li> + + <a href="../../community/team-list.html" title="SINGA Team"> + <span class="none"></span> + SINGA Team</a> + </li> + <li class="nav-header">External Links</li> + + <li> + + <a href="http://www.apache.org/"; class="externalLink" title="Apache Software Foundation"> + <span class="none"></span> + Apache Software Foundation</a> + </li> + + <li> + + <a href="http://www.comp.nus.edu.sg/~dbsystem/singa/"; class="externalLink" title="NUS Site"> + <span class="none"></span> + NUS Site</a> + </li> + </ul> + + + + <hr /> + + <div id="poweredBy"> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <div class="clear"></div> + <a href="http://incubator.apache.org"; title="apache-incubator" class="builtBy"> + <img class="builtBy" alt="Apache Incubator" src="http://incubator.apache.org/images/egg-logo.png"; /> + </a> + </div> + </div> + </div> + + + <div id="bodyColumn" class="span10" > + + <h1>Recurrent Neural Networks for Language Modelling</h1> +<hr /> +<p>Recurrent Neural Networks (RNN) are widely used for modelling sequential data, such as music and sentences. In this example, we use SINGA to train a <a class="externalLink" href="http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf";>RNN model</a> proposed by Tomas Mikolov for <a class="externalLink" href="https://en.wikipedia.org/wiki/Language_model";>language modeling</a>. The training objective (loss) is to minimize the <a class="externalLink" href="https://en.wikipedia.org/wiki/Perplexity";>perplexity per word</a>, which is equivalent to maximize the probability of predicting the next word given the current word in a sentence.</p> +<p>Different to the <a href="cnn.html">CNN</a>, <a href="mlp.html">MLP</a> and <a href="rbm.html">RBM</a> examples which use built-in layers(layer) and records(data), none of the layers in this example are built-in. Hence users would learn to implement their own layers and data records through this example.</p> +<div class="section"> +<h2><a name="Running_instructions"></a>Running instructions</h2> +<p>In <i>SINGA_ROOT/examples/rnnlm/</i>, scripts are provided to run the training job. First, the data is prepared by</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ cp Makefile.example Makefile +$ make download +$ make create +</pre></div></div> +<p>Second, to compile the source code under <i>examples/rnnlm/</i>, run</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">$ make rnnlm +</pre></div></div> +<p>An executable file <i>rnnlm.bin</i> will be generated.</p> +<p>Third, the training is started by passing <i>rnnlm.bin</i> and the job configuration to <i>singa-run.sh</i>,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># at SINGA_ROOT/ +# export LD_LIBRARY_PATH=.libs:$LD_LIBRARY_PATH +$ ./bin/singa-run.sh -exec examples/rnnlm/rnnlm.bin -conf examples/rnnlm/job.conf +</pre></div></div></div> +<div class="section"> +<h2><a name="Implementations"></a>Implementations</h2> +<p><img src="../images/rnnlm.png" align="center" width="400px" alt="" /> <span><b>Figure 1 - Net structure of the RNN model.</b></span></p> +<p>The neural net structure is shown Figure 1. Word records are loaded by <tt>DataLayer</tt>. For every iteration, at most <tt>max_window</tt> word records are processed. If a sentence ending character is read, the <tt>DataLayer</tt> stops loading immediately. <tt>EmbeddingLayer</tt> looks up a word embedding matrix to extract feature vectors for words loaded by the <tt>DataLayer</tt>. These features are transformed by the <tt>HiddenLayer</tt> which propagates the features from left to right. The output feature for word at position k is influenced by words from position 0 to k-1. Finally, <tt>LossLayer</tt> computes the cross-entropy loss (see below) by predicting the next word of each word. The cross-entropy loss is computed as</p> +<p><tt>$$L(w_t)=-log P(w_{t+1}|w_t)$$</tt></p> +<p>Given <tt>$w_t$</tt> the above equation would compute over all words in the vocabulary, which is time consuming. <a class="externalLink" href="https://f25ea9ccb7d3346ce6891573d543960492b92c30.googledrive.com/host/0ByxdPXuxLPS5RFM5dVNvWVhTd0U/rnnlm-0.4b.tgz";>RNNLM Toolkit</a> accelerates the computation as</p> +<p><tt>$$P(w_{t+1}|w_t) = P(C_{w_{t+1}}|w_t) * P(w_{t+1}|C_{w_{t+1}})$$</tt></p> +<p>Words from the vocabulary are partitioned into a user-defined number of classes. The first term on the left side predicts the class of the next word, and then predicts the next word given its class. Both the number of classes and the words from one class are much smaller than the vocabulary size. The probabilities can be calculated much faster.</p> +<p>The perplexity per word is computed by,</p> +<p><tt>$$PPL = 10^{- avg_t log_{10} P(w_{t+1}|w_t)}$$</tt></p> +<div class="section"> +<h3><a name="Data_preparation"></a>Data preparation</h3> +<p>We use a small dataset provided by the <a class="externalLink" href="https://f25ea9ccb7d3346ce6891573d543960492b92c30.googledrive.com/host/0ByxdPXuxLPS5RFM5dVNvWVhTd0U/rnnlm-0.4b.tgz";>RNNLM Toolkit</a>. It has 10,000 training sentences, with 71350 words in total and 3720 unique words. The subsequent steps follow the instructions in <a href="data.html">Data Preparation</a> to convert the raw data into records and insert them into data stores.</p> +<div class="section"> +<h4><a name="Download_source_data"></a>Download source data</h4> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in SINGA_ROOT/examples/rnnlm/ +cp Makefile.example Makefile +make download +</pre></div></div></div> +<div class="section"> +<h4><a name="Define_record_format"></a>Define record format</h4> +<p>We define the word record as follows,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># in SINGA_ROOT/examples/rnnlm/rnnlm.proto +message WordRecord { + optional string word = 1; + optional int32 word_index = 2; + optional int32 class_index = 3; + optional int32 class_start = 4; + optional int32 class_end = 5; +} +</pre></div></div> +<p>It includes the word string and its index in the vocabulary. Words in the vocabulary are sorted based on their frequency in the training dataset. The sorted list is cut into 100 sublists such that each sublist has 1/100 total word frequency. Each sublist is called a class. Hence each word has a <tt>class_index</tt> ([0,100)). The <tt>class_start</tt> is the index of the first word in the same class as <tt>word</tt>. The <tt>class_end</tt> is the index of the first word in the next class.</p></div> +<div class="section"> +<h4><a name="Create_data_stores"></a>Create data stores</h4> +<p>We use code from RNNLM Toolkit to read words, and sort them into classes. The main function in <i>create_store.cc</i> first creates word classes based on the training dataset. Second it calls the following function to create data store for the training, validation and test dataset.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">int create_data(const char *input_file, const char *output_file); +</pre></div></div> +<p><tt>input</tt> is the path to training/validation/testing text file from the RNNLM Toolkit, <tt>output</tt> is output store file. This function starts with</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">singa::io::KVFile store; +store.Open(output, signa::io::kCreate); +</pre></div></div> +<p>Then it reads the words one by one. For each word it creates a <tt>WordRecord</tt> instance, and inserts it into the store,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">int wcnt = 0; // word count +WordRecord wordRecord; +while(1) { + readWord(wordstr, fin); + if (feof(fin)) break; + ...// fill in the wordRecord; + string val; + wordRecord.SerializeToString(&val); + int length = snprintf(key, BUFFER_LEN, "%05d", wcnt++); + store.Write(string(key, length), val); +} +</pre></div></div> +<p>Compilation and running commands are provided in the <i>Makefile.example</i>. After executing</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">make create +</pre></div></div> +<p><i>train_data.bin</i>, <i>test_data.bin</i> and <i>valid_data.bin</i> will be created.</p></div></div> +<div class="section"> +<h3><a name="Layer_implementation"></a>Layer implementation</h3> +<p>4 user-defined layers are implemented for this application. Following the guide for implementing <a href="layer#implementing-a-new-layer-subclass">new Layer subclasses</a>, we extend the <a href="../api/classsinga_1_1LayerProto.html">LayerProto</a> to include the configuration messages of user-defined layers as shown below (3 out of the 7 layers have specific configurations),</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">import "job.proto"; // Layer message for SINGA is defined + +//For implementation of RNNLM application +extend singa.LayerProto { + optional EmbeddingProto embedding_conf = 101; + optional LossProto loss_conf = 102; + optional DataProto data_conf = 103; +} +</pre></div></div> +<p>In the subsequent sections, we describe the implementation of each layer, including its configuration message.</p> +<div class="section"> +<h4><a name="RNNLayer"></a>RNNLayer</h4> +<p>This is the base layer of all other layers for this applications. It is defined as follows,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class RNNLayer : virtual public Layer { +public: + inline int window() { return window_; } +protected: + int window_; +}; +</pre></div></div> +<p>For this application, two iterations may process different number of words. Because sentences have different lengths. The <tt>DataLayer</tt> decides the effective window size. All other layers call its source layers to get the effective window size and resets <tt>window_</tt> in <tt>ComputeFeature</tt> function.</p></div> +<div class="section"> +<h4><a name="DataLayer"></a>DataLayer</h4> +<p>DataLayer is for loading Records.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class DataLayer : public RNNLayer, singa::InputLayer { + public: + void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override; + void ComputeFeature(int flag, const vector<Layer*>& srclayers) override; + int max_window() const { + return max_window_; + } + private: + int max_window_; + singa::io::Store* store_; +}; +</pre></div></div> +<p>The Setup function gets the user configured max window size.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">max_window_ = proto.GetExtension(input_conf).max_window(); +</pre></div></div> +<p>The <tt>ComputeFeature</tt> function loads at most max_window records. It could also stop when the sentence ending character is encountered.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">...// shift the last record to the first +window_ = max_window_; +for (int i = 1; i <= max_window_; i++) { + // load record; break if it is the ending character +} +</pre></div></div> +<p>The configuration of <tt>DataLayer</tt> is like</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">name: "data" +user_type: "kData" +[data_conf] { + path: "examples/rnnlm/train_data.bin" + max_window: 10 +} +</pre></div></div></div> +<div class="section"> +<h4><a name="EmbeddingLayer"></a>EmbeddingLayer</h4> +<p>This layer gets records from <tt>DataLayer</tt>. For each record, the word index is parsed and used to get the corresponding word feature vector from the embedding matrix.</p> +<p>The class is declared as follows,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class EmbeddingLayer : public RNNLayer { + ... + const std::vector<Param*> GetParams() const override { + std::vector<Param*> params{embed_}; + return params; + } + private: + int word_dim_, vocab_size_; + Param* embed_; +} +</pre></div></div> +<p>The <tt>embed_</tt> field is a matrix whose values are parameter to be learned. The matrix size is <tt>vocab_size_</tt> x <tt>word_dim_</tt>.</p> +<p>The Setup function reads configurations for <tt>word_dim_</tt> and <tt>vocab_size_</tt>. Then it allocates feature Blob for <tt>max_window</tt> words and setups <tt>embed_</tt>.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">int max_window = srclayers[0]->data(this).shape()[0]; +word_dim_ = proto.GetExtension(embedding_conf).word_dim(); +data_.Reshape(vector<int>{max_window, word_dim_}); +... +embed_->Setup(vector<int>{vocab_size_, word_dim_}); +</pre></div></div> +<p>The <tt>ComputeFeature</tt> function simply copies the feature vector from the <tt>embed_</tt> matrix into the feature Blob.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># reset effective window size +window_ = datalayer->window(); +auto records = datalayer->records(); +... +for (int t = 0; t < window_; t++) { + int idx <- word index + Copy(words[t], embed[idx]); +} +</pre></div></div> +<p>The <tt>ComputeGradient</tt> function copies back the gradients to the <tt>embed_</tt> matrix.</p> +<p>The configuration for <tt>EmbeddingLayer</tt> is like,</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">user_type: "kEmbedding" +[embedding_conf] { + word_dim: 15 + vocab_size: 3720 +} +srclayers: "data" +param { + name: "w1" + init { + type: kUniform + low:-0.3 + high:0.3 + } +} +</pre></div></div></div> +<div class="section"> +<h4><a name="HiddenLayer"></a>HiddenLayer</h4> +<p>This layer unrolls the recurrent connections for at most max_window times. The feature for position k is computed based on the feature from the embedding layer (position k) and the feature at position k-1 of this layer. The formula is</p> +<p><tt>$$f[k]=\sigma (f[t-1]*W+src[t])$$</tt></p> +<p>where <tt>$W$</tt> is a matrix with <tt>word_dim_</tt> x <tt>word_dim_</tt> parameters.</p> +<p>If you want to implement a recurrent neural network following our design, this layer is of vital importance for you to refer to.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class HiddenLayer : public RNNLayer { + ... + const std::vector<Param*> GetParams() const override { + std::vector<Param*> params{weight_}; + return params; + } +private: + Param* weight_; +}; +</pre></div></div> +<p>The <tt>Setup</tt> function setups the weight matrix as</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">weight_->Setup(std::vector<int>{word_dim, word_dim}); +</pre></div></div> +<p>The <tt>ComputeFeature</tt> function gets the effective window size (<tt>window_</tt>) from its source layer i.e., the embedding layer. Then it propagates the feature from position 0 to position <tt>window_</tt> -1. The detailed descriptions for this process are illustrated as follows.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">void HiddenLayer::ComputeFeature() { + for(int t = 0; t < window_size; t++){ + if(t == 0) + Copy(data[t], src[t]); + else + data[t]=sigmoid(data[t-1]*W + src[t]); + } +} +</pre></div></div> +<p>The <tt>ComputeGradient</tt> function computes the gradient of the loss w.r.t. W and the source layer. Particularly, for each position k, since data[k] contributes to data[k+1] and the feature at position k in its destination layer (the loss layer), grad[k] should contains the gradient from two parts. The destination layer has already computed the gradient from the loss layer into grad[k]; In the <tt>ComputeGradient</tt> function, we need to add the gradient from position k+1.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">void HiddenLayer::ComputeGradient(){ + ... + for (int k = window_ - 1; k >= 0; k--) { + if (k < window_ - 1) { + grad[k] += dot(grad[k + 1], weight.T()); // add gradient from position t+1. + } + grad[k] =... // compute gL/gy[t], y[t]=data[t-1]*W+src[t] + } + gweight = dot(data.Slice(0, window_-1).T(), grad.Slice(1, window_)); + Copy(gsrc, grad); +} +</pre></div></div> +<p>After the loop, we get the gradient of the loss w.r.t y[k], which is used to compute the gradient of W and the src[k].</p></div> +<div class="section"> +<h4><a name="LossLayer"></a>LossLayer</h4> +<p>This layer computes the cross-entropy loss and the <tt>$log_{10}P(w_{t+1}|w_t)$</tt> (which could be averaged over all words by users to get the PPL value).</p> +<p>There are two configuration fields to be specified by users.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">message LossProto { + optional int32 nclass = 1; + optional int32 vocab_size = 2; +} +</pre></div></div> +<p>There are two weight matrices to be learned</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">class LossLayer : public RNNLayer { + ... + private: + Param* word_weight_, *class_weight_; +} +</pre></div></div> +<p>The ComputeFeature function computes the two probabilities respectively.</p> +<p><tt>$$P(C_{w_{t+1}}|w_t) = Softmax(w_t * class\_weight_)$$</tt> <tt>$$P(w_{t+1}|C_{w_{t+1}}) = Softmax(w_t * word\_weight[class\_start:class\_end])$$</tt></p> +<p><tt>$w_t$</tt> is the feature from the hidden layer for the k-th word, its ground truth next word is <tt>$w_{t+1}$</tt>. The first equation computes the probability distribution over all classes for the next word. The second equation computes the probability distribution over the words in the ground truth class for the next word.</p> +<p>The ComputeGradient function computes the gradient of the source layer (i.e., the hidden layer) and the two weight matrices.</p></div></div> +<div class="section"> +<h3><a name="Updater_Configuration"></a>Updater Configuration</h3> +<p>We employ kFixedStep type of the learning rate change method and the configuration is as follows. We decay the learning rate once the performance does not increase on the validation dataset.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint">updater{ + type: kSGD + learning_rate { + type: kFixedStep + fixedstep_conf:{ + step:0 + step:48810 + step:56945 + step:65080 + step:73215 + step_lr:0.1 + step_lr:0.05 + step_lr:0.025 + step_lr:0.0125 + step_lr:0.00625 + } + } +} +</pre></div></div></div> +<div class="section"> +<h3><a name="TrainOneBatch_Function"></a>TrainOneBatch() Function</h3> +<p>We use BP (BackPropagation) algorithm to train the RNN model here. The corresponding configuration can be seen below.</p> + +<div class="source"> +<div class="source"><pre class="prettyprint"># In job.conf file +train_one_batch { + alg: kBackPropagation +} +</pre></div></div></div> +<div class="section"> +<h3><a name="Cluster_Configuration"></a>Cluster Configuration</h3> +<p>The default cluster configuration can be used, i.e., single worker and single server in a single process.</p></div></div> + </div> + </div> + </div> + + <hr/> + + <footer> + <div class="container-fluid"> + <div class="row-fluid"> + +<p>Copyright © 2015 The Apache Software Foundation. All rights reserved. Apache Singa, Apache, the Apache feather logo, and the Apache Singa project logos are trademarks of The Apache Software Foundation. All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> + </div> + + + </div> + </footer> + </body> +</html>
![https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"](https://s3.amazonaws.com/github/ribbons/forkme_right_orange_ff7600.png"){kind=link}
![http://incubator.apache.org/images/egg-logo.png"](http://incubator.apache.org/images/egg-logo.png"){kind=link}