Author: deron
Date: Thu Apr 6 20:19:08 2017
New Revision: 1790452
URL: http://svn.apache.org/viewvc?rev=1790452&view=rev
Log:
Add jekyll syntax highlighting to code blocks
Modified:
incubator/systemml/site/assets/css/main.css
incubator/systemml/site/documentation.html
incubator/systemml/site/download.html
incubator/systemml/site/get-started.html
incubator/systemml/site/privacy-policy.html
Modified: incubator/systemml/site/assets/css/main.css
URL:
http://svn.apache.org/viewvc/incubator/systemml/site/assets/css/main.css?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/assets/css/main.css (original)
+++ incubator/systemml/site/assets/css/main.css Thu Apr 6 20:19:08 2017
@@ -4577,6 +4577,260 @@ input[type=number] {
font-weight: 700;
font-style: italic; }
+.highlight {
+ background: #ffffff; }
+
+.highlight .c {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment */
+.highlight .err {
+ color: #a61717;
+ background-color: #e3d2d2; }
+
+/* Error */
+.highlight .k {
+ font-weight: bold; }
+
+/* Keyword */
+.highlight .o {
+ font-weight: bold; }
+
+/* Operator */
+.highlight .cm {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment.Multiline */
+.highlight .cp {
+ color: #999999;
+ font-weight: bold; }
+
+/* Comment.Preproc */
+.highlight .c1 {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment.Single */
+.highlight .cs {
+ color: #999999;
+ font-weight: bold;
+ font-style: italic; }
+
+/* Comment.Special */
+.highlight .gd {
+ color: #000000;
+ background-color: #ffdddd; }
+
+/* Generic.Deleted */
+.highlight .gd .x {
+ color: #000000;
+ background-color: #ffaaaa; }
+
+/* Generic.Deleted.Specific */
+.highlight .ge {
+ font-style: italic; }
+
+/* Generic.Emph */
+.highlight .gr {
+ color: #aa0000; }
+
+/* Generic.Error */
+.highlight .gh {
+ color: #999999; }
+
+/* Generic.Heading */
+.highlight .gi {
+ color: #000000;
+ background-color: #ddffdd; }
+
+/* Generic.Inserted */
+.highlight .gi .x {
+ color: #000000;
+ background-color: #aaffaa; }
+
+/* Generic.Inserted.Specific */
+.highlight .go {
+ color: #888888; }
+
+/* Generic.Output */
+.highlight .gp {
+ color: #555555; }
+
+/* Generic.Prompt */
+.highlight .gs {
+ font-weight: bold; }
+
+/* Generic.Strong */
+.highlight .gu {
+ color: #aaaaaa; }
+
+/* Generic.Subheading */
+.highlight .gt {
+ color: #aa0000; }
+
+/* Generic.Traceback */
+.highlight .kc {
+ font-weight: bold; }
+
+/* Keyword.Constant */
+.highlight .kd {
+ font-weight: bold; }
+
+/* Keyword.Declaration */
+.highlight .kp {
+ font-weight: bold; }
+
+/* Keyword.Pseudo */
+.highlight .kr {
+ font-weight: bold; }
+
+/* Keyword.Reserved */
+.highlight .kt {
+ color: #445588;
+ font-weight: bold; }
+
+/* Keyword.Type */
+.highlight .m {
+ color: #009999; }
+
+/* Literal.Number */
+.highlight .s {
+ color: #d14; }
+
+/* Literal.String */
+.highlight .na {
+ color: #008080; }
+
+/* Name.Attribute */
+.highlight .nb {
+ color: #0086B3; }
+
+/* Name.Builtin */
+.highlight .nc {
+ color: #445588;
+ font-weight: bold; }
+
+/* Name.Class */
+.highlight .no {
+ color: #008080; }
+
+/* Name.Constant */
+.highlight .ni {
+ color: #800080; }
+
+/* Name.Entity */
+.highlight .ne {
+ color: #990000;
+ font-weight: bold; }
+
+/* Name.Exception */
+.highlight .nf {
+ color: #990000;
+ font-weight: bold; }
+
+/* Name.Function */
+.highlight .nn {
+ color: #555555; }
+
+/* Name.Namespace */
+.highlight .nt {
+ color: #000080; }
+
+/* Name.Tag */
+.highlight .nv {
+ color: #008080; }
+
+/* Name.Variable */
+.highlight .ow {
+ font-weight: bold; }
+
+/* Operator.Word */
+.highlight .w {
+ color: #bbbbbb; }
+
+/* Text.Whitespace */
+.highlight .mf {
+ color: #009999; }
+
+/* Literal.Number.Float */
+.highlight .mh {
+ color: #009999; }
+
+/* Literal.Number.Hex */
+.highlight .mi {
+ color: #009999; }
+
+/* Literal.Number.Integer */
+.highlight .mo {
+ color: #009999; }
+
+/* Literal.Number.Oct */
+.highlight .sb {
+ color: #d14; }
+
+/* Literal.String.Backtick */
+.highlight .sc {
+ color: #d14; }
+
+/* Literal.String.Char */
+.highlight .sd {
+ color: #d14; }
+
+/* Literal.String.Doc */
+.highlight .s2 {
+ color: #d14; }
+
+/* Literal.String.Double */
+.highlight .se {
+ color: #d14; }
+
+/* Literal.String.Escape */
+.highlight .sh {
+ color: #d14; }
+
+/* Literal.String.Heredoc */
+.highlight .si {
+ color: #d14; }
+
+/* Literal.String.Interpol */
+.highlight .sx {
+ color: #d14; }
+
+/* Literal.String.Other */
+.highlight .sr {
+ color: #009926; }
+
+/* Literal.String.Regex */
+.highlight .s1 {
+ color: #d14; }
+
+/* Literal.String.Single */
+.highlight .ss {
+ color: #990073; }
+
+/* Literal.String.Symbol */
+.highlight .bp {
+ color: #999999; }
+
+/* Name.Builtin.Pseudo */
+.highlight .vc {
+ color: #008080; }
+
+/* Name.Variable.Class */
+.highlight .vg {
+ color: #008080; }
+
+/* Name.Variable.Global */
+.highlight .vi {
+ color: #008080; }
+
+/* Name.Variable.Instance */
+.highlight .il {
+ color: #009999; }
+
+/* Literal.Number.Integer.Long */
.main-nav {
float: right; }
.main-nav ul {
@@ -4673,6 +4927,9 @@ pre {
white-space: pre\9;
/* IE7+ */ }
+code {
+ overflow: scroll; }
+
.text-center {
text-align: center; }
@@ -4691,6 +4948,9 @@ hr {
ol {
padding: 0; }
+figure {
+ margin: 0; }
+
.ml-container {
display: -webkit-flex;
display: flex;
@@ -5436,4 +5696,4 @@ ol {
.preview-image img {
width: 500px; }
[... 3 lines stripped ...]
Modified: incubator/systemml/site/documentation.html
URL:
http://svn.apache.org/viewvc/incubator/systemml/site/documentation.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/documentation.html (original)
+++ incubator/systemml/site/documentation.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>
Modified: incubator/systemml/site/download.html
URL:
http://svn.apache.org/viewvc/incubator/systemml/site/download.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/download.html (original)
+++ incubator/systemml/site/download.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>
Modified: incubator/systemml/site/get-started.html
URL:
http://svn.apache.org/viewvc/incubator/systemml/site/get-started.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/get-started.html (original)
+++ incubator/systemml/site/get-started.html Thu Apr 6 20:19:08 2017
@@ -181,9 +181,12 @@
<!-- Step 1 Code -->
<div class="col col-12">
- <pre><code>/usr/bin/ruby -e "$(curl -fsSL
https://raw.githubusercontent.com/Homebrew/install/master/install)"
-# Linux
-ruby -e "$(curl -fsSL
https://raw.githubusercontent.com/Linuxbrew/install/master/install)"</code></pre>
+
+
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">/usr/bin/ruby -e <span class="s2">"</span><span
class="k">$(</span>curl -fsSL
https://raw.githubusercontent.com/Homebrew/install/master/install<span
class="k">)</span><span class="s2">"</span>
+<span class="c"># Linux</span>
+ruby -e <span class="s2">"</span><span class="k">$(</span>curl -fsSL
https://raw.githubusercontent.com/Linuxbrew/install/master/install<span
class="k">)</span><span class="s2">"</span></code></pre></figure>
+
</div>
<!-- Step 2 Instructions -->
@@ -193,11 +196,11 @@ ruby -e "$(curl -fsSL https://raw.github
<!-- Step 2 Code -->
<div class="col col-12">
- <pre><code>brew tap caskroom/cask
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">brew tap caskroom/cask
brew install Caskroom/cask/java
brew install python
-pip install jupyter matplotlib numpy</code></pre>
+pip install jupyter matplotlib numpy</code></pre></figure>
</div>
</div>
@@ -216,8 +219,8 @@ pip install jupyter matplotlib numpy</co
<!-- Step 3 Code -->
<div class="col col-12">
- <pre><code>brew tap homebrew/versions
-brew install apache-spark16</code></pre>
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">brew tap homebrew/versions
+brew install apache-spark16</code></pre></figure>
<p> Alternatively, you can <a
href="http://spark.apache.org/downloads.html";>download Spark</a> directly. </p>
</div>
@@ -230,22 +233,24 @@ brew install apache-spark16</code></pre>
<!-- Step 4 Code -->
<div class="col col-12">
- <p>
- If you are a python user, we recommend that you download and install
Apache SystemML via pip:
- </p>
- <pre><code>Python 2:
+ <p>
+ If you are a python user, we recommend that you download and install
Apache SystemML via pip:
+ </p>
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash"><span class="c"># Python 2</span>
pip install systemml
-# Bleeding edge: pip install
git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python
+<span class="c"># Bleeding edge: pip install
git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</span></code></pre></figure>
-Python 3:
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash"><span class="c"># Python 3:</span>
pip3 install systemml
-# Bleeding edge: pip3 install
git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</code></pre>
+<span class="c"># Bleeding edge: pip3 install
git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</span></code></pre></figure>
+
- <p>
- Alternatively, if you intend to use SystemML via spark-shell (or
spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged
into our official binary release (<a
href="http://www.apache.org/dyn/closer.cgi/pub/apache/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating.zip";
target="_blank">systemml-0.12.0-incubating.zip</a>).
- Note: If you have installed SystemML via pip, you can get the location
of this jar by executing following command:
- </p>
- <pre><code>python -c 'import imp; import os; print
os.path.join(imp.find_module("systemml")[1], "systemml-java")'</code></pre>
+
+ <p>
+ Alternatively, if you intend to use SystemML via spark-shell (or
spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged
into our official binary release (<a
href="http://www.apache.org/dyn/closer.cgi/pub/apache/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating.zip";
target="_blank">systemml-0.12.0-incubating.zip</a>).
+ Note: If you have installed SystemML via pip, you can get the location
of this jar by executing following command:
+ </p>
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">python -c <span class="s1">'import imp; import os; print
os.path.join(imp.find_module("systemml")[1],
"systemml-java")'</span></code></pre></figure>
</div>
@@ -254,31 +259,31 @@ pip3 install systemml
<!-- Section Header -->
<div class="col col-12 content-group--medium-bottom-margin">
<h2>Ways to Use</h2>
- <p>You can use SystemML in one of the following ways:
- <ol>
- <li>On Cluster (using our programmatic APIs):
- <ul>
- <li>Using pyspark: Please see our <a
href="http://apache.github.io/incubator-systemml/beginners-guide-python";>beginner's
guide for python users</a>.</li>
- <li>Using Jupyter: Described below in step
5.</li>
- <li>Using spark-shell: Described below in step
6.</li>
- </ul>
- </li>
-
- <li>On Cluster (command-line batch mode):
- <ul>
- <li>Using spark-submit: Please see our <a
href="http://apache.github.io/incubator-systemml/spark-batch-mode";>spark batch
mode tutorial</a>.</li>
- <li>Using hadoop: Please see our <a
href="http://apache.github.io/incubator-systemml/hadoop-batch-mode";>hadoop
batch model tutorial</a>.</li>
- </ul>
- </li>
-
- <li>On laptop (command-line batch mode) without installing
Spark or Hadoop: Please see our <a
href="http://apache.github.io/incubator-systemml/standalone-guide";>standalone
mode tutorial</a>.</li>
-
- <li>In-memory mode (as part of another Java application for
scoring): Please see our <a
href="http://apache.github.io/incubator-systemml/jmlc";>JMLC tutorial</a>.</li>
- </ol>
-
- <p>
- Note that you can also run pyspark, spark-shell, spark-submit on you
laptop using "--master local[*]" parameter.
- </p>
+ <p>You can use SystemML in one of the following ways:</p>
+ <ol>
+ <li>On Cluster (using our programmatic APIs):
+ <ul>
+ <li>Using pyspark: Please see our <a
href="http://apache.github.io/incubator-systemml/beginners-guide-python";>beginner's
guide for python users</a>.</li>
+ <li>Using Jupyter: Described below in step
5.</li>
+ <li>Using spark-shell: Described below in step
6.</li>
+ </ul>
+ </li>
+
+ <li>On Cluster (command-line batch mode):
+ <ul>
+ <li>Using spark-submit: Please see our <a
href="http://apache.github.io/incubator-systemml/spark-batch-mode";>spark batch
mode tutorial</a>.</li>
+ <li>Using hadoop: Please see our <a
href="http://apache.github.io/incubator-systemml/hadoop-batch-mode";>hadoop
batch model tutorial</a>.</li>
+ </ul>
+ </li>
+
+ <li>On laptop (command-line batch mode) without installing
Spark or Hadoop: Please see our <a
href="http://apache.github.io/incubator-systemml/standalone-guide";>standalone
mode tutorial</a>.</li>
+
+ <li>In-memory mode (as part of another Java application for
scoring): Please see our <a
href="http://apache.github.io/incubator-systemml/jmlc";>JMLC tutorial</a>.</li>
+ </ol>
+
+ <p>
+ Note that you can also run pyspark, spark-shell, spark-submit on you
laptop using "--master local[*]" parameter.
+ </p>
</div>
<!-- Step 5 Instructions -->
@@ -290,11 +295,10 @@ pip3 install systemml
<div class="col col-12">
<h4>Get Started</h4>
<p>Start up your Jupyter notebook by moving to the folder where you
saved the notebook. Then copy and paste the line below:</p>
- <pre><code>Python 2:
-PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark
--master local[*] --driver-class-path SystemML.jar --jars SystemML.jar--conf
"spark.driver.memory=12g" --conf spark.driver.maxResultSize=0 --conf
spark.akka.frameSize=128 --conf spark.default.parallelism=100
-
-Python 3:
-PYSPARK_PYTHON=python3 PYSPARK_DRIVER_PYTHON=jupyter
PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark --master local[*]
--driver-class-path SystemML.jar --jars SystemML.jar --conf
"spark.driver.memory=12g" --conf spark.driver.maxResultSize=0 --conf
spark.akka.frameSize=128 --conf spark.default.parallelism=100</code></pre>
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="c"># Python 2:</span>
+<span class="n">PYSPARK_DRIVER_PYTHON</span><span class="o">=</span><span
class="n">jupyter</span> <span class="n">PYSPARK_DRIVER_PYTHON_OPTS</span><span
class="o">=</span><span class="s">"notebook"</span> <span
class="n">pyspark</span> <span class="o">--</span><span class="n">master</span>
<span class="n">local</span><span class="p">[</span><span
class="o">*</span><span class="p">]</span> <span class="o">--</span><span
class="n">driver</span><span class="o">-</span><span
class="n">class</span><span class="o">-</span><span class="n">path</span> <span
class="n">SystemML</span><span class="o">.</span><span class="n">jar</span>
<span class="o">--</span><span class="n">jars</span> <span
class="n">SystemML</span><span class="o">.</span><span
class="n">jar</span><span class="o">--</span><span class="n">conf</span> <span
class="s">"spark.driver.memory=12g"</span> <span class="o">--</span><span
class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span
class="n">driver<
/span><span class="o">.</span><span class="n">maxResultSize</span><span
class="o">=</span><span class="mi">0</span> <span class="o">--</span><span
class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span
class="n">akka</span><span class="o">.</span><span
class="n">frameSize</span><span class="o">=</span><span class="mi">128</span>
<span class="o">--</span><span class="n">conf</span> <span
class="n">spark</span><span class="o">.</span><span
class="n">default</span><span class="o">.</span><span
class="n">parallelism</span><span class="o">=</span><span
class="mi">100</span></code></pre></figure>
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="c"># Python 3:</span>
+<span class="n">PYSPARK_PYTHON</span><span class="o">=</span><span
class="n">python3</span> <span class="n">PYSPARK_DRIVER_PYTHON</span><span
class="o">=</span><span class="n">jupyter</span> <span
class="n">PYSPARK_DRIVER_PYTHON_OPTS</span><span class="o">=</span><span
class="s">"notebook"</span> <span class="n">pyspark</span> <span
class="o">--</span><span class="n">master</span> <span
class="n">local</span><span class="p">[</span><span class="o">*</span><span
class="p">]</span> <span class="o">--</span><span class="n">driver</span><span
class="o">-</span><span class="n">class</span><span class="o">-</span><span
class="n">path</span> <span class="n">SystemML</span><span
class="o">.</span><span class="n">jar</span> <span class="o">--</span><span
class="n">jars</span> <span class="n">SystemML</span><span
class="o">.</span><span class="n">jar</span> <span class="o">--</span><span
class="n">conf</span> <span class="s">"spark.driver.memory=12g"</span> <span
class="o">--</span><span clas
s="n">conf</span> <span class="n">spark</span><span class="o">.</span><span
class="n">driver</span><span class="o">.</span><span
class="n">maxResultSize</span><span class="o">=</span><span class="mi">0</span>
<span class="o">--</span><span class="n">conf</span> <span
class="n">spark</span><span class="o">.</span><span class="n">akka</span><span
class="o">.</span><span class="n">frameSize</span><span class="o">=</span><span
class="mi">128</span> <span class="o">--</span><span class="n">conf</span>
<span class="n">spark</span><span class="o">.</span><span
class="n">default</span><span class="o">.</span><span
class="n">parallelism</span><span class="o">=</span><span
class="mi">100</span></code></pre></figure>
</div>
@@ -307,64 +311,67 @@ PYSPARK_PYTHON=python3 PYSPARK_DRIVER_PY
<div class="col col-12">
<h4>Start Spark Shell with SystemML</h4>
<p> To use SystemML with Spark Shell, the SystemML jar can be
referenced using Spark Shellâs --jars option. Start the Spark Shell with
SystemML with the following line of code in your terminal:</p>
- <pre><code>spark-shell --executor-memory 4G --driver-memory 4G --jars
SystemML.jar</code></pre>
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">spark-shell --executor-memory 4G --driver-memory 4G --jars
SystemML.jar</code></pre></figure>
+ <!-- <pre><code>spark-shell --executor-memory 4G --driver-memory 4G
--jars SystemML.jar</code></pre> -->
<h4>Create the MLContext</h4>
<p>To begin, start an MLContext by typing the code below. Once
successful, you should see a âWelcome to Apache SystemML!â message.</p>
- <pre><code>import org.apache.sysml.api.mlcontext._
+ <figure class="highlight"><pre><code class="language-bash"
data-lang="bash">import org.apache.sysml.api.mlcontext._
import org.apache.sysml.api.mlcontext.ScriptFactory._
-val ml = new MLContext(sc)</code></pre>
+val ml <span class="o">=</span> new MLContext<span class="o">(</span>sc<span
class="o">)</span></code></pre></figure>
+
<h4>Hello World</h4>
<p>The ScriptFactory class allows DML and PYDML scripts to be created
from Strings, Files, URLs, and InputStreams. Here, weâll use the dmlmethod to
create a DML âhello worldâ script based on a String. We execute the script
using MLContextâs execute method, which displays âhello worldâ to the
console. The execute method returns an MLResults object, which contains no
results since the script has no outputs.</p>
- <pre><code>val helloScript = dml("print('hello world')")
-ml.execute(helloScript)</code></pre>
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="n">val</span> <span
class="n">helloScript</span> <span class="o">=</span> <span
class="n">dml</span><span class="p">(</span><span class="s">"print('hello
world')"</span><span class="p">)</span>
+<span class="n">ml</span><span class="o">.</span><span
class="n">execute</span><span class="p">(</span><span
class="n">helloScript</span><span class="p">)</span></code></pre></figure>
<h4>DataFrame Example</h4>
<p>As an example of how to use SystemML, weâll first use Spark to
create a DataFrame called df of random doubles from 0 to 1 consisting of 10,000
rows and 1,000 columns.</p>
- <pre><code>import org.apache.spark.sql._
-import org.apache.spark.sql.types.{StructType,StructField,DoubleType}
-import scala.util.Random
-val numRows = 10000
-val numCols = 1000
-val data = sc.parallelize(0 to numRows-1).map { _ =>
Row.fromSeq(Seq.fill(numCols)(Random.nextDouble)) }
-val schema = StructType((0 to numCols-1).map { i => StructField("C" + i,
DoubleType, true) } )
-val df = sqlContext.createDataFrame(data, schema)</code></pre>
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="kn">import</span> <span
class="nn">org.apache.spark.sql._</span>
+<span class="kn">import</span> <span
class="nn">org.apache.spark.sql.types.</span><span class="p">{</span><span
class="n">StructType</span><span class="p">,</span><span
class="n">StructField</span><span class="p">,</span><span
class="n">DoubleType</span><span class="p">}</span>
+<span class="kn">import</span> <span class="nn">scala.util.Random</span>
+<span class="n">val</span> <span class="n">numRows</span> <span
class="o">=</span> <span class="mi">10000</span>
+<span class="n">val</span> <span class="n">numCols</span> <span
class="o">=</span> <span class="mi">1000</span>
+<span class="n">val</span> <span class="n">data</span> <span
class="o">=</span> <span class="n">sc</span><span class="o">.</span><span
class="n">parallelize</span><span class="p">(</span><span class="mi">0</span>
<span class="n">to</span> <span class="n">numRows</span><span
class="o">-</span><span class="mi">1</span><span class="p">)</span><span
class="o">.</span><span class="nb">map</span> <span class="p">{</span> <span
class="n">_</span> <span class="o">=></span> <span class="n">Row</span><span
class="o">.</span><span class="n">fromSeq</span><span class="p">(</span><span
class="n">Seq</span><span class="o">.</span><span class="n">fill</span><span
class="p">(</span><span class="n">numCols</span><span class="p">)(</span><span
class="n">Random</span><span class="o">.</span><span
class="n">nextDouble</span><span class="p">))</span> <span class="p">}</span>
+<span class="n">val</span> <span class="n">schema</span> <span
class="o">=</span> <span class="n">StructType</span><span
class="p">((</span><span class="mi">0</span> <span class="n">to</span> <span
class="n">numCols</span><span class="o">-</span><span class="mi">1</span><span
class="p">)</span><span class="o">.</span><span class="nb">map</span> <span
class="p">{</span> <span class="n">i</span> <span class="o">=></span> <span
class="n">StructField</span><span class="p">(</span><span class="s">"C"</span>
<span class="o">+</span> <span class="n">i</span><span class="p">,</span> <span
class="n">DoubleType</span><span class="p">,</span> <span
class="n">true</span><span class="p">)</span> <span class="p">}</span> <span
class="p">)</span>
+<span class="n">val</span> <span class="n">df</span> <span class="o">=</span>
<span class="n">sqlContext</span><span class="o">.</span><span
class="n">createDataFrame</span><span class="p">(</span><span
class="n">data</span><span class="p">,</span> <span
class="n">schema</span><span class="p">)</span></code></pre></figure>
+
<p>Weâll create a DML script using the ScriptFactory dml method to
find the minimum, maximum, and mean values in a matrix. This script has one
input variable, matrix Xin, and three output variables, minOut, maxOut, and
meanOut.
For performance, weâll specify metadata indicating that the matrix has
10,000 rows and 1,000 columns.
We execute the script and obtain the results as a Tuple by calling getTuple on
the results, specifying the types and names of the output variables.</p>
- <pre><code>val minMaxMean =
-"""
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="n">val</span> <span class="n">minMaxMean</span>
<span class="o">=</span>
+<span class="s">"""
minOut = min(Xin)
maxOut = max(Xin)
meanOut = mean(Xin)
-"""
-val mm = new MatrixMetadata(numRows, numCols)
-val minMaxMeanScript = dml(minMaxMean).in("Xin", df, mm).out("minOut",
"maxOut", "meanOut")
-val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Double,
Double]("minOut", "maxOut", "meanOut")</code></pre>
+"""</span>
+<span class="n">val</span> <span class="n">mm</span> <span class="o">=</span>
<span class="n">new</span> <span class="n">MatrixMetadata</span><span
class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span
class="n">numCols</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">minMaxMeanScript</span> <span
class="o">=</span> <span class="n">dml</span><span class="p">(</span><span
class="n">minMaxMean</span><span class="p">)</span><span
class="o">.</span><span class="ow">in</span><span class="p">(</span><span
class="s">"Xin"</span><span class="p">,</span> <span class="n">df</span><span
class="p">,</span> <span class="n">mm</span><span class="p">)</span><span
class="o">.</span><span class="n">out</span><span class="p">(</span><span
class="s">"minOut"</span><span class="p">,</span> <span
class="s">"maxOut"</span><span class="p">,</span> <span
class="s">"meanOut"</span><span class="p">)</span>
+<span class="n">val</span> <span class="p">(</span><span
class="nb">min</span><span class="p">,</span> <span class="nb">max</span><span
class="p">,</span> <span class="n">mean</span><span class="p">)</span> <span
class="o">=</span> <span class="n">ml</span><span class="o">.</span><span
class="n">execute</span><span class="p">(</span><span
class="n">minMaxMeanScript</span><span class="p">)</span><span
class="o">.</span><span class="n">getTuple</span><span class="p">[</span><span
class="n">Double</span><span class="p">,</span> <span
class="n">Double</span><span class="p">,</span> <span
class="n">Double</span><span class="p">](</span><span
class="s">"minOut"</span><span class="p">,</span> <span
class="s">"maxOut"</span><span class="p">,</span> <span
class="s">"meanOut"</span><span class="p">)</span></code></pre></figure>
<p>Many different types of input and output variables are
automatically allowed. These types include Boolean, Long, Double, String,
Array[Array[Double]], RDD<String> and JavaRDD<String> in CSV (dense) and IJV
(sparse) formats, DataFrame, BinaryBlockMatrix,Matrix, and Frame. RDDs and
JavaRDDs are assumed to be CSV format unless MatrixMetadata is supplied
indicating IJV format.</p>
<h4>RDD Example:</h4>
<p>Letâs take a look at an example of input matrices as RDDs in CSV
format. Weâll create two 2x2 matrices and input these into a DML script. This
script will sum each matrix and create a message based on which sum is greater.
We will output the sums and the message.</p>
-<pre><code>val rdd1 = sc.parallelize(Array("1.0,2.0", "3.0,4.0"))
-val rdd2 = sc.parallelize(Array("5.0,6.0", "7.0,8.0"))
-val sums = """
+ <figure class="highlight"><pre><code class="language-python"
data-lang="python"><span class="n">val</span> <span class="n">rdd1</span> <span
class="o">=</span> <span class="n">sc</span><span class="o">.</span><span
class="n">parallelize</span><span class="p">(</span><span
class="n">Array</span><span class="p">(</span><span
class="s">"1.0,2.0"</span><span class="p">,</span> <span
class="s">"3.0,4.0"</span><span class="p">))</span>
+<span class="n">val</span> <span class="n">rdd2</span> <span
class="o">=</span> <span class="n">sc</span><span class="o">.</span><span
class="n">parallelize</span><span class="p">(</span><span
class="n">Array</span><span class="p">(</span><span
class="s">"5.0,6.0"</span><span class="p">,</span> <span
class="s">"7.0,8.0"</span><span class="p">))</span>
+<span class="n">val</span> <span class="n">sums</span> <span
class="o">=</span> <span class="s">"""
s1 = sum(m1);
s2 = sum(m2);
-if (s1 > s2) {
+if (s1 > s2) {
message = "s1 is greater"
-} else if (s2 > s1) {
+} else if (s2 > s1) {
message = "s2 is greater"
} else {
message = "s1 and s2 are equal"
}
-"""
-scala.tools.nsc.io.File("sums.dml").writeAll(sums)
-val sumScript = dmlFromFile("sums.dml").in(Map("m1"-> rdd1, "m2"->
rdd2)).out("s1", "s2", "message")
-val sumResults = ml.execute(sumScript)
-val s1 = sumResults.getDouble("s1")
-val s2 = sumResults.getDouble("s2")
-val message = sumResults.getString("message")
-val rdd1Metadata = new MatrixMetadata(2, 2)
-val rdd2Metadata = new MatrixMetadata(2, 2)
-val sumScript = dmlFromFile("sums.dml").in(Seq(("m1", rdd1, rdd1Metadata),
("m2", rdd2, rdd2Metadata))).out("s1", "s2", "message")
-val (firstSum, secondSum, sumMessage) = ml.execute(sumScript).getTuple[Double,
Double, String]("s1", "s2", "message")</code></pre>
+"""</span>
+<span class="n">scala</span><span class="o">.</span><span
class="n">tools</span><span class="o">.</span><span class="n">nsc</span><span
class="o">.</span><span class="n">io</span><span class="o">.</span><span
class="n">File</span><span class="p">(</span><span
class="s">"sums.dml"</span><span class="p">)</span><span
class="o">.</span><span class="n">writeAll</span><span class="p">(</span><span
class="n">sums</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">sumScript</span> <span
class="o">=</span> <span class="n">dmlFromFile</span><span
class="p">(</span><span class="s">"sums.dml"</span><span
class="p">)</span><span class="o">.</span><span class="ow">in</span><span
class="p">(</span><span class="n">Map</span><span class="p">(</span><span
class="s">"m1"</span><span class="o">-></span> <span
class="n">rdd1</span><span class="p">,</span> <span class="s">"m2"</span><span
class="o">-></span> <span class="n">rdd2</span><span
class="p">))</span><span class="o">.</span><span class="n">out</span><span
class="p">(</span><span class="s">"s1"</span><span class="p">,</span> <span
class="s">"s2"</span><span class="p">,</span> <span
class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">sumResults</span> <span
class="o">=</span> <span class="n">ml</span><span class="o">.</span><span
class="n">execute</span><span class="p">(</span><span
class="n">sumScript</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">s1</span> <span class="o">=</span>
<span class="n">sumResults</span><span class="o">.</span><span
class="n">getDouble</span><span class="p">(</span><span
class="s">"s1"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">s2</span> <span class="o">=</span>
<span class="n">sumResults</span><span class="o">.</span><span
class="n">getDouble</span><span class="p">(</span><span
class="s">"s2"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">message</span> <span
class="o">=</span> <span class="n">sumResults</span><span
class="o">.</span><span class="n">getString</span><span class="p">(</span><span
class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">rdd1Metadata</span> <span
class="o">=</span> <span class="n">new</span> <span
class="n">MatrixMetadata</span><span class="p">(</span><span
class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span
class="p">)</span>
+<span class="n">val</span> <span class="n">rdd2Metadata</span> <span
class="o">=</span> <span class="n">new</span> <span
class="n">MatrixMetadata</span><span class="p">(</span><span
class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span
class="p">)</span>
+<span class="n">val</span> <span class="n">sumScript</span> <span
class="o">=</span> <span class="n">dmlFromFile</span><span
class="p">(</span><span class="s">"sums.dml"</span><span
class="p">)</span><span class="o">.</span><span class="ow">in</span><span
class="p">(</span><span class="n">Seq</span><span class="p">((</span><span
class="s">"m1"</span><span class="p">,</span> <span class="n">rdd1</span><span
class="p">,</span> <span class="n">rdd1Metadata</span><span class="p">),</span>
<span class="p">(</span><span class="s">"m2"</span><span class="p">,</span>
<span class="n">rdd2</span><span class="p">,</span> <span
class="n">rdd2Metadata</span><span class="p">)))</span><span
class="o">.</span><span class="n">out</span><span class="p">(</span><span
class="s">"s1"</span><span class="p">,</span> <span class="s">"s2"</span><span
class="p">,</span> <span class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="p">(</span><span
class="n">firstSum</span><span class="p">,</span> <span
class="n">secondSum</span><span class="p">,</span> <span
class="n">sumMessage</span><span class="p">)</span> <span class="o">=</span>
<span class="n">ml</span><span class="o">.</span><span
class="n">execute</span><span class="p">(</span><span
class="n">sumScript</span><span class="p">)</span><span class="o">.</span><span
class="n">getTuple</span><span class="p">[</span><span
class="n">Double</span><span class="p">,</span> <span
class="n">Double</span><span class="p">,</span> <span
class="n">String</span><span class="p">](</span><span
class="s">"s1"</span><span class="p">,</span> <span class="s">"s2"</span><span
class="p">,</span> <span class="s">"message"</span><span
class="p">)</span></code></pre></figure>
<p>Congratulations! Youâve now run examples in Apache SystemML!</p>
</div>
</div>
Modified: incubator/systemml/site/privacy-policy.html
URL:
http://svn.apache.org/viewvc/incubator/systemml/site/privacy-policy.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/privacy-policy.html (original)
+++ incubator/systemml/site/privacy-policy.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>
