http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/quickstart/java_api_quickstart.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/quickstart/java_api_quickstart.html b/content/docs/0.9/quickstart/java_api_quickstart.html deleted file mode 100644 index fbe07db..0000000 --- a/content/docs/0.9/quickstart/java_api_quickstart.html +++ /dev/null @@ -1,330 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Quickstart: Java API</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Quickstart: Java API</h1> - - - -<ul id="markdown-toc"> - <li><a href="#requirements" id="markdown-toc-requirements">Requirements</a></li> - <li><a href="#create-project" id="markdown-toc-create-project">Create Project</a></li> - <li><a href="#inspect-project" id="markdown-toc-inspect-project">Inspect Project</a></li> - <li><a href="#build-project" id="markdown-toc-build-project">Build Project</a></li> - <li><a href="#next-steps" id="markdown-toc-next-steps">Next Steps</a></li> -</ul> - -<p>Start working on your Flink Java program in a few simple steps.</p> - -<h2 id="requirements">Requirements</h2> - -<p>The only requirements are working <strong>Maven 3.0.4</strong> (or higher) and <strong>Java 6.x</strong> (or higher) installations.</p> - -<h2 id="create-project">Create Project</h2> - -<p>Use one of the following commands to <strong>create a project</strong>:</p> - -<ul class="nav nav-tabs" style="border-bottom: none;"> - <li class="active"><a href="#quickstart-script" data-toggle="tab">Run the <strong>quickstart script</strong></a></li> - <li><a href="#maven-archetype" data-toggle="tab">Use <strong>Maven archetypes</strong></a></li> -</ul> -<div class="tab-content"> - <div class="tab-pane active" id="quickstart-script"> - -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>curl http://flink.apache.org/q/quickstart.sh <span class="p">|</span> bash</code></pre></div> - - </div> - <div class="tab-pane" id="maven-archetype"> - -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>mvn archetype:generate <span class="se">\</span> - -DarchetypeGroupId<span class="o">=</span>org.apache.flink <span class="se">\</span> - -DarchetypeArtifactId<span class="o">=</span>flink-quickstart-java <span class="se">\</span> - -DarchetypeVersion<span class="o">=</span>0.9.0</code></pre></div> - - This allows you to <strong>name your newly created project</strong>. It will interactively ask you for the groupId, artifactId, and package name. - </div> -</div> - -<h2 id="inspect-project">Inspect Project</h2> - -<p>There will be a new directory in your working directory. If youâve used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p> - -<p>The sample project is a <strong>Maven project</strong>, which contains two classes. <em>Job</em> is a basic skeleton program and <em>WordCountJob</em> a working example. Please note that the <em>main</em> method of both classes allow you to start Flink in a development/testing mode.</p> - -<p>We recommend to <strong>import this project into your IDE</strong> to develop and test it. If you use Eclipse, the <a href="http://www.eclipse.org/m2e/";>m2e plugin</a> allows to <a href="http://books.sonatype.com/m2eclipse-book/reference/creating-sect-importing-projects.html#fig-creating-import";>import Maven projects</a>. Some Eclipse bundles include that plugin by default, other require you to install it manually. The IntelliJ IDE also supports Maven projects out of the box.</p> - -<p>A note to Mac OS X users: The default JVM heapsize for Java is too small for Flink. You have to manually increase it. Choose âRun Configurationsâ -> Arguments and write into the âVM Argumentsâ box: â-Xmx800mâ in Eclipse.</p> - -<h2 id="build-project">Build Project</h2> - -<p>If you want to <strong>build your project</strong>, go to your project directory and issue the<code>mvn clean install -Pbuild-jar</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/your-artifact-id-1.0-SNAPSHOT.jar</strong>. There is also a fat-jar, <strong>target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar</strong>. This -also contains all dependencies that get added to the maven project.</p> - -<h2 id="next-steps">Next Steps</h2> - -<p>Write your application!</p> - -<p>The quickstart project contains a WordCount implementation, the âHello Worldâ of Big Data processing systems. The goal of WordCount is to determine the frequencies of words in a text, e.g., how often do the terms âtheâ or âhouseâ occurs in all Wikipedia texts.</p> - -<p><strong>Sample Input</strong>:</p> - -<div class="highlight"><pre><code class="language-bash">big data is big</code></pre></div> - -<p><strong>Sample Output</strong>:</p> - -<div class="highlight"><pre><code class="language-bash">big 2 -data 1 -is 1</code></pre></div> - -<p>The following code shows the WordCount implementation from the Quickstart which processes some text lines with two operators (FlatMap and Reduce), and writes the prints the resulting words and counts to std-out.</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">WordCount</span> <span class="o">{</span> - - <span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> - - <span class="c1">// set up the execution environment</span> - <span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> - - <span class="c1">// get input data</span> - <span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">text</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span> - <span class="s">"To be, or not to be,--that is the question:--"</span><span class="o">,</span> - <span class="s">"Whether 'tis nobler in the mind to suffer"</span><span class="o">,</span> - <span class="s">"The slings and arrows of outrageous fortune"</span><span class="o">,</span> - <span class="s">"Or to take arms against a sea of troubles,"</span> - <span class="o">);</span> - - <span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">counts</span> <span class="o">=</span> - <span class="c1">// split up the lines in pairs (2-tuples) containing: (word,1)</span> - <span class="n">text</span><span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="nf">LineSplitter</span><span class="o">())</span> - <span class="c1">// group by the tuple field "0" and sum up tuple field "1"</span> - <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="o">.</span><span class="na">aggregate</span><span class="o">(</span><span class="n">Aggregations</span><span class="o">.</span><span class="na">SUM</span><span class="o">,</span> <span class="mi">1</span><span class="o">);</span> - - <span class="c1">// emit result</span> - <span class="n">counts</span><span class="o">.</span><span class="na">print</span><span class="o">();</span> - - <span class="c1">// execute program</span> - <span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"WordCount Example"</span><span class="o">);</span> - <span class="o">}</span> -<span class="o">}</span></code></pre></div> - -<p>The operations are defined by specialized classes, here the LineSplitter class.</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">LineSplitter</span> <span class="kd">implements</span> <span class="n">FlatMapFunction</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="o">{</span> - - <span class="nd">@Override</span> - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">flatMap</span><span class="o">(</span><span class="n">String</span> <span class="n">value</span><span class="o">,</span> <span class="n">Collector</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">out</span><span class="o">)</span> <span class="o">{</span> - <span class="c1">// normalize and split the line into words</span> - <span class="n">String</span><span class="o">[]</span> <span class="n">tokens</span> <span class="o">=</span> <span class="n">value</span><span class="o">.</span><span class="na">toLowerCase</span><span class="o">().</span><span class="na">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">);</span> - - <span class="c1">// emit the pairs</span> - <span class="k">for</span> <span class="o">(</span><span class="n">String</span> <span class="n">token</span> <span class="o">:</span> <span class="n">tokens</span><span class="o">)</span> <span class="o">{</span> - <span class="k">if</span> <span class="o">(</span><span class="n">token</span><span class="o">.</span><span class="na">length</span><span class="o">()</span> <span class="o">></span> <span class="mi">0</span><span class="o">)</span> <span class="o">{</span> - <span class="n">out</span><span class="o">.</span><span class="na">collect</span><span class="o">(</span><span class="k">new</span> <span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>(</span><span class="n">token</span><span class="o">,</span> <span class="mi">1</span><span class="o">));</span> - <span class="o">}</span> - <span class="o">}</span> - <span class="o">}</span> -<span class="o">}</span></code></pre></div> - -<p><a href="https://github.com/apache/flink/blob/master//flink-examples/flink-java-examples/src/main/java/org/apache/flink/examples/java/wordcount/WordCount.java";>Check GitHub</a> for the full example code.</p> - -<p>For a complete overview over our API, have a look at the <a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";>Programming Guide</a> and <a href="examples.html">further example programs</a>. If you have any trouble, ask on our <a href="http://mail-archives.apache.org/mod_mbox/flink-dev/";>Mailing List</a>. We are happy to provide help.</p> - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/quickstart/plotPoints.py ---------------------------------------------------------------------- diff --git a/content/docs/0.9/quickstart/plotPoints.py b/content/docs/0.9/quickstart/plotPoints.py deleted file mode 100755 index 9ddf5dd..0000000 --- a/content/docs/0.9/quickstart/plotPoints.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/python - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys -import matplotlib.pyplot as plt -import csv -import os - -if len(sys.argv) < 4 or not sys.argv[1] in ['points', 'result']: - print "Usage: plot-clusters.py (points|result) <src-file> <pdf-file-prefix>" - sys.exit(1) - -inFile = sys.argv[1] -inFile = sys.argv[2] -outFilePx = sys.argv[3] - -inFileName = os.path.splitext(os.path.basename(inFile))[0] -outFile = os.path.join(".", outFilePx+"-plot.pdf") - -########### READ DATA - -cs = [] -xs = [] -ys = [] - -minX = None -maxX = None -minY = None -maxY = None - -if sys.argv[1] == 'points': - - with open(inFile, 'rb') as file: - for line in file: - # parse data - csvData = line.strip().split(' ') - - x = float(csvData[0]) - y = float(csvData[1]) - - if not minX or minX > x: - minX = x - if not maxX or maxX < x: - maxX = x - if not minY or minY > y: - minY = y - if not maxY or maxY < y: - maxY = y - - xs.append(x) - ys.append(y) - - # plot data - plt.clf() - plt.scatter(xs, ys, s=25, c="#999999", edgecolors='None', alpha=1.0) - plt.ylim([minY,maxY]) - plt.xlim([minX,maxX]) - -elif sys.argv[1] == 'result': - - with open(inFile, 'rb') as file: - for line in file: - # parse data - csvData = line.strip().split(' ') - - c = int(csvData[0]) - x = float(csvData[1]) - y = float(csvData[2]) - - cs.append(c) - xs.append(x) - ys.append(y) - - # plot data - plt.clf() - plt.scatter(xs, ys, s=25, c=cs, edgecolors='None', alpha=1.0) - plt.ylim([minY,maxY]) - plt.xlim([minX,maxX]) - - -plt.savefig(outFile, dpi=600) -print "\nPlotted file: %s" % outFile - -sys.exit(0) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/quickstart/run_example_quickstart.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/quickstart/run_example_quickstart.html b/content/docs/0.9/quickstart/run_example_quickstart.html deleted file mode 100644 index de636ca..0000000 --- a/content/docs/0.9/quickstart/run_example_quickstart.html +++ /dev/null @@ -1,359 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Quick Start: Run K-Means Example</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Quick Start: Run K-Means Example</h1> - - - -<ul id="markdown-toc"> - <li><a href="#setup-flink" id="markdown-toc-setup-flink">Setup Flink</a></li> - <li><a href="#generate-input-data" id="markdown-toc-generate-input-data">Generate Input Data</a></li> - <li><a href="#inspect-the-input-data" id="markdown-toc-inspect-the-input-data">Inspect the Input Data</a></li> - <li><a href="#start-flink" id="markdown-toc-start-flink">Start Flink</a></li> - <li><a href="#inspect-and-run-the-k-means-example-program" id="markdown-toc-inspect-and-run-the-k-means-example-program">Inspect and Run the K-Means Example Program</a></li> - <li><a href="#shutdown-flink" id="markdown-toc-shutdown-flink">Shutdown Flink</a></li> - <li><a href="#analyze-the-result" id="markdown-toc-analyze-the-result">Analyze the Result</a></li> -</ul> - -<p>This guide walks you through the steps of executing an example program (<a href="http://en.wikipedia.org/wiki/K-means_clustering";>K-Means clustering</a>) on Flink. On the way, you will see the a visualization of the program, the optimized execution plan, and track the progress of its execution.</p> - -<h2 id="setup-flink">Setup Flink</h2> -<p>Follow the <a href="setup_quickstart.html">instructions</a> to setup Flink and enter the root directory of your Flink setup.</p> - -<h2 id="generate-input-data">Generate Input Data</h2> -<p>Flink contains a data generator for K-Means.</p> - -<div class="highlight"><pre><code class="language-bash"><span class="c"># Assuming you are in the root directory of your Flink setup</span> -mkdir kmeans -<span class="nb">cd </span>kmeans -<span class="c"># Run data generator</span> -java -cp ../examples/flink-java-examples-0.9.0-KMeans.jar:../lib/flink-dist-0.9.0.jar <span class="se">\</span> - org.apache.flink.examples.java.clustering.util.KMeansDataGenerator <span class="se">\</span> - -points <span class="m">500</span> -k <span class="m">10</span> -stddev 0.08 -output <span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span></code></pre></div> - -<p>The generator has the following arguments (arguments in <code>[]</code> are optional):</p> - -<div class="highlight"><pre><code class="language-bash">-points <num> -k <num clusters> <span class="o">[</span>-output <output-path><span class="o">]</span> <span class="o">[</span>-stddev <relative stddev><span class="o">]</span> <span class="o">[</span>-range <centroid range><span class="o">]</span> <span class="o">[</span>-seed <seed><span class="o">]</span></code></pre></div> - -<p>The <em>relative standard deviation</em> is an interesting tuning parameter. It determines the closeness of the points to randomly generated centers.</p> - -<p>The <code>kmeans/</code> directory should now contain two files: <code>centers</code> and <code>points</code>. The <code>points</code> file contains the points to cluster and the <code>centers</code> file contains initial cluster centers.</p> - -<h2 id="inspect-the-input-data">Inspect the Input Data</h2> -<p>Use the <code>plotPoints.py</code> tool to review the generated data points. <a href="quickstart/plotPoints.py">Download Python Script</a></p> - -<div class="highlight"><pre><code class="language-bash">python plotPoints.py points ./points input</code></pre></div> - -<p>Note: You might have to install <a href="http://matplotlib.org/";>matplotlib</a> (<code>python-matplotlib</code> package on Ubuntu) to use the Python script.</p> - -<p>You can review the input data stored in the <code>input-plot.pdf</code>, for example with Evince (<code>evince input-plot.pdf</code>).</p> - -<p>The following overview presents the impact of the different standard deviations on the input data.</p> - -<table> - <thead> - <tr> - <th style="text-align: center">relative stddev = 0.03</th> - <th style="text-align: center">relative stddev = 0.08</th> - <th style="text-align: center">relative stddev = 0.15</th> - </tr> - </thead> - <tbody> - <tr> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans003.png"; alt="example1" style="width: 275px;" /></td> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans008.png"; alt="example2" style="width: 275px;" /></td> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans015.png"; alt="example3" style="width: 275px;" /></td> - </tr> - </tbody> -</table> - -<h2 id="start-flink">Start Flink</h2> -<p>Start Flink and the web job submission client on your local machine.</p> - -<div class="highlight"><pre><code class="language-bash"><span class="c"># return to the Flink root directory</span> -<span class="nb">cd</span> .. -<span class="c"># start Flink</span> -./bin/start-local.sh -<span class="c"># Start the web client</span> -./bin/start-webclient.sh</code></pre></div> - -<h2 id="inspect-and-run-the-k-means-example-program">Inspect and Run the K-Means Example Program</h2> -<p>The Flink web client allows to submit Flink programs using a graphical user interface.</p> - -<div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="example-1" href="http://flink.apache.org/docs/0.9/page/img/quickstart-example/run-webclient.png";><img class="img-responsive" src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/run-webclient.png"; /></a> - </div> - <div class="col-md-6"> - 1. Open web client on <a href="http://localhost:8080/launch.html";>localhost:8080</a> <br /> - 2. Upload the K-Mean job JAR file. - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">./examples/flink-java-examples-*-KMeans.jar</code></pre></div> - - 3. Select it in the left box to see how the operators in the plan are connected to each other. <br /> - 4. Enter the arguments in the lower left box: - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">file://<pathToFlink>/kmeans/points file://<pathToFlink>/kmeans/centers file://<pathToFlink>/kmeans/result 10</code></pre></div> - - For example: - -<div class="highlight"><pre><code class="language-bash" data-lang="bash">file:///tmp/flink/kmeans/points file:///tmp/flink/kmeans/centers file:///tmp/flink/kmeans/result 10</code></pre></div> - - </div> -</div> -<hr /> - -<div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="example-1" href="http://flink.apache.org/docs/0.9/page/img/quickstart-example/compiler-webclient-new.png";><img class="img-responsive" src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/compiler-webclient-new.png"; /></a> - </div> - - <div class="col-md-6"> - 1. Press the <b>RunJob</b> to see the optimizer plan. <br /> - 2. Inspect the operators and see the properties (input sizes, cost estimation) determined by the optimizer. - </div> -</div> -<hr /> - -<div class="row" style="padding-top:15px"> - <div class="col-md-6"> - <a data-lightbox="example-1" href="http://flink.apache.org/docs/0.9/page/img/quickstart-example/jobmanager-running-new.png";><img class="img-responsive" src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/jobmanager-running-new.png"; /></a> - </div> - <div class="col-md-6"> - 1. Press the <b>Continue</b> button to start executing the job. <br /> - 2. <a href="http://localhost:8080/launch.html";>Open Flink's monitoring interface</a> to see the job's progress. (Due to the small input data, the job will finish really quick!)<br /> - 3. Once the job has finished, you can analyze the runtime of the individual operators. - </div> -</div> - -<h2 id="shutdown-flink">Shutdown Flink</h2> -<p>Stop Flink when you are done.</p> - -<div class="highlight"><pre><code class="language-bash"><span class="c"># stop Flink</span> -./bin/stop-local.sh -<span class="c"># Stop the Flink web client</span> -./bin/stop-webclient.sh</code></pre></div> - -<h2 id="analyze-the-result">Analyze the Result</h2> -<p>Use the <a href="quickstart/plotPoints.py">Python Script</a> again to visualize the result.</p> - -<div class="highlight"><pre><code class="language-bash"><span class="nb">cd </span>kmeans -python plotPoints.py result ./result clusters</code></pre></div> - -<p>The following three pictures show the results for the sample input above. Play around with the parameters (number of iterations, number of clusters) to see how they affect the result.</p> - -<table> - <thead> - <tr> - <th style="text-align: center">relative stddev = 0.03</th> - <th style="text-align: center">relative stddev = 0.08</th> - <th style="text-align: center">relative stddev = 0.15</th> - </tr> - </thead> - <tbody> - <tr> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/result003.png"; alt="example1" style="width: 275px;" /></td> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/result008.png"; alt="example2" style="width: 275px;" /></td> - <td style="text-align: center"><img src="http://flink.apache.org/docs/0.9/page/img/quickstart-example/result015.png"; alt="example3" style="width: 275px;" /></td> - </tr> - </tbody> -</table> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/quickstart/scala_api_quickstart.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/quickstart/scala_api_quickstart.html b/content/docs/0.9/quickstart/scala_api_quickstart.html deleted file mode 100644 index ffe7e1a..0000000 --- a/content/docs/0.9/quickstart/scala_api_quickstart.html +++ /dev/null @@ -1,334 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Quickstart: Scala API</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Quickstart: Scala API</h1> - - - -<ul id="markdown-toc"> - <li><a href="#requirements" id="markdown-toc-requirements">Requirements</a></li> - <li><a href="#create-project" id="markdown-toc-create-project">Create Project</a></li> - <li><a href="#inspect-project" id="markdown-toc-inspect-project">Inspect Project</a></li> - <li><a href="#build-project" id="markdown-toc-build-project">Build Project</a></li> - <li><a href="#next-steps" id="markdown-toc-next-steps">Next Steps</a></li> - <li><a href="#alternative-build-tools-sbt" id="markdown-toc-alternative-build-tools-sbt">Alternative Build Tools: SBT</a></li> -</ul> - -<p>Start working on your Flink Scala program in a few simple steps.</p> - -<h2 id="requirements">Requirements</h2> - -<p>The only requirements are working <strong>Maven 3.0.4</strong> (or higher) and <strong>Java 6.x</strong> (or higher) installations.</p> - -<h2 id="create-project">Create Project</h2> - -<p>Use one of the following commands to <strong>create a project</strong>:</p> - -<ul class="nav nav-tabs" style="border-bottom: none;"> - <li class="active"><a href="#quickstart-script" data-toggle="tab">Run the <strong>quickstart script</strong></a></li> - <li><a href="#maven-archetype" data-toggle="tab">Use <strong>Maven archetypes</strong></a></li> -</ul> -<div class="tab-content"> - <div class="tab-pane active" id="quickstart-script"> - -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>curl http://flink.apache.org/q/quickstart-scala.sh <span class="p">|</span> bash</code></pre></div> - - </div> - <div class="tab-pane" id="maven-archetype"> - -<div class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="nv">$ </span>mvn archetype:generate <span class="se">\</span> - -DarchetypeGroupId<span class="o">=</span>org.apache.flink <span class="se">\</span> - -DarchetypeArtifactId<span class="o">=</span>flink-quickstart-scala <span class="se">\</span> - -DarchetypeVersion<span class="o">=</span>0.9.0</code></pre></div> - - This allows you to <strong>name your newly created project</strong>. It will interactively ask you for the groupId, artifactId, and package name. - </div> -</div> - -<h2 id="inspect-project">Inspect Project</h2> - -<p>There will be a new directory in your working directory. If youâve used the <em>curl</em> approach, the directory is called <code>quickstart</code>. Otherwise, it has the name of your artifactId.</p> - -<p>The sample project is a <strong>Maven project</strong>, which contains two classes. <em>Job</em> is a basic skeleton program and <em>WordCountJob</em> a working example. Please note that the <em>main</em> method of both classes allow you to start Flink in a development/testing mode.</p> - -<p>We recommend to <strong>import this project into your IDE</strong>. For Eclipse, you need the following plugins, which you can install from the provided Eclipse Update Sites:</p> - -<ul> - <li><em>Eclipse 4.x</em> - <ul> - <li><a href="http://download.scala-ide.org/sdk/e38/scala210/stable/site";>Scala IDE</a></li> - <li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> - <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.15.0/N/0.15.0.201206251206/";>Build Helper Maven Plugin</a></li> - </ul> - </li> - <li><em>Eclipse 3.7</em> - <ul> - <li><a href="http://download.scala-ide.org/sdk/e37/scala210/stable/site";>Scala IDE</a></li> - <li><a href="http://alchim31.free.fr/m2e-scala/update-site";>m2eclipse-scala</a></li> - <li><a href="https://repository.sonatype.org/content/repositories/forge-sites/m2e-extras/0.14.0/N/0.14.0.201109282148/";>Build Helper Maven Plugin</a></li> - </ul> - </li> -</ul> - -<p>The IntelliJ IDE also supports Maven and offers a plugin for Scala development.</p> - -<h2 id="build-project">Build Project</h2> - -<p>If you want to <strong>build your project</strong>, go to your project directory and issue the <code>mvn clean package -Pbuild-jar</code> command. You will <strong>find a jar</strong> that runs on every Flink cluster in <strong>target/your-artifact-id-1.0-SNAPSHOT.jar</strong>. There is also a fat-jar, <strong>target/your-artifact-id-1.0-SNAPSHOT-flink-fat-jar.jar</strong>. This -also contains all dependencies that get added to the maven project.</p> - -<h2 id="next-steps">Next Steps</h2> - -<p>Write your application!</p> - -<p>The quickstart project contains a WordCount implementation, the âHello Worldâ of Big Data processing systems. The goal of WordCount is to determine the frequencies of words in a text, e.g., how often do the terms âtheâ or âhouseâ occurs in all Wikipedia texts.</p> - -<p><strong>Sample Input</strong>:</p> - -<div class="highlight"><pre><code class="language-bash">big data is big</code></pre></div> - -<p><strong>Sample Output</strong>:</p> - -<div class="highlight"><pre><code class="language-bash">big 2 -data 1 -is 1</code></pre></div> - -<p>The following code shows the WordCount implementation from the Quickstart which processes some text lines with two operators (FlatMap and Reduce), and writes the prints the resulting words and counts to std-out.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="k">object</span> <span class="nc">WordCountJob</span> <span class="o">{</span> - <span class="k">def</span> <span class="n">main</span><span class="o">(</span><span class="n">args</span><span class="k">:</span> <span class="kt">Array</span><span class="o">[</span><span class="kt">String</span><span class="o">])</span> <span class="o">{</span> - - <span class="c1">// set up the execution environment</span> - <span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span> - - <span class="c1">// get input data</span> - <span class="k">val</span> <span class="n">text</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">fromElements</span><span class="o">(</span><span class="s">"To be, or not to be,--that is the question:--"</span><span class="o">,</span> - <span class="s">"Whether 'tis nobler in the mind to suffer"</span><span class="o">,</span> <span class="s">"The slings and arrows of outrageous fortune"</span><span class="o">,</span> - <span class="s">"Or to take arms against a sea of troubles,"</span><span class="o">)</span> - - <span class="k">val</span> <span class="n">counts</span> <span class="k">=</span> <span class="n">text</span><span class="o">.</span><span class="n">flatMap</span> <span class="o">{</span> <span class="k">_</span><span class="o">.</span><span class="n">toLowerCase</span><span class="o">.</span><span class="n">split</span><span class="o">(</span><span class="s">"\\W+"</span><span class="o">)</span> <span class="o">}</span> - <span class="o">.</span><span class="n">map</span> <span class="o">{</span> <span class="o">(</span><span class="k">_</span><span class="o">,</span> <span class="mi">1</span><span class="o">)</span> <span class="o">}</span> - <span class="o">.</span><span class="n">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> - - <span class="c1">// emit result</span> - <span class="n">counts</span><span class="o">.</span><span class="n">print</span><span class="o">()</span> - - <span class="c1">// execute program</span> - <span class="n">env</span><span class="o">.</span><span class="n">execute</span><span class="o">(</span><span class="s">"WordCount Example"</span><span class="o">)</span> - <span class="o">}</span> -<span class="o">}</span></code></pre></div> - -<p><a href="https://github.com/apache/flink/blob/master//flink-examples/flink-scala-examples/src/main/scala/org/apache/flink/examples/scala/wordcount/WordCount.scala";>Check GitHub</a> for the full example code.</p> - -<p>For a complete overview over our API, have a look at the <a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";>Programming Guide</a> and <a href="http://flink.apache.org/docs/0.9/apis/examples.html";>further example programs</a>. If you have any trouble, ask on our <a href="http://mail-archives.apache.org/mod_mbox/flink-dev/";>Mailing List</a>. We are happy to provide help.</p> - -<h2 id="alternative-build-tools-sbt">Alternative Build Tools: SBT</h2> - -<p>To build and run applications with SBT instead of Maven is pretty straight forward. After creating the standard sbt <a href="http://www.scala-sbt.org/0.13/tutorial/Directories.html";>directory layout</a> itâs enough to add the Flink dependencies to the <code>build.sbt</code> file:</p> - -<div class="highlight"><pre><code class="language-scala"><span class="n">libraryDependencies</span> <span class="o">++=</span> <span class="nc">Seq</span><span class="o">(</span><span class="s">"org.apache.flink"</span> <span class="o">%</span> <span class="s">"flink-scala"</span> <span class="o">%</span> <span class="s">"0.9.0"</span><span class="o">,</span> <span class="s">"org.apache.flink"</span> <span class="o">%</span> <span class="s">"flink-clients"</span> <span class="o">%</span> <span class="s">"0.9.0"</span><span class="o">)</span></code></pre></div> - -<p>Now the application can be executed by <code>sbt run</code>. By default SBT runs an application in the same JVM itself is running in. This can lead to lass loading issues with Flink. To avoid these, append the following line to <code>build.sbt</code>:</p> - -<div class="highlight"><pre><code class="language-scala"><span class="n">fork</span> <span class="n">in</span> <span class="n">run</span> <span class="o">:=</span> <span class="kc">true</span></code></pre></div> - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/quickstart/setup_quickstart.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/quickstart/setup_quickstart.html b/content/docs/0.9/quickstart/setup_quickstart.html deleted file mode 100644 index 82f98fa..0000000 --- a/content/docs/0.9/quickstart/setup_quickstart.html +++ /dev/null @@ -1,350 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Quickstart: Setup</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Quickstart: Setup</h1> - - - -<ul id="markdown-toc"> - <li><a href="#requirements" id="markdown-toc-requirements">Requirements</a></li> - <li><a href="#download" id="markdown-toc-download">Download</a></li> - <li><a href="#start" id="markdown-toc-start">Start</a></li> - <li><a href="#run-example" id="markdown-toc-run-example">Run Example</a></li> - <li><a href="#cluster-setup" id="markdown-toc-cluster-setup">Cluster Setup</a></li> - <li><a href="#flink-on-yarn" id="markdown-toc-flink-on-yarn">Flink on YARN</a></li> -</ul> - -<p>Get Flink up and running in a few simple steps.</p> - -<h2 id="requirements">Requirements</h2> - -<p>Flink runs on <strong>Linux, Mac OS X, and Windows</strong>. To be able to run Flink, the -only requirement is to have a working <strong>Java 6.x</strong> (or higher) -installation. Windows users, please take a look at the -<a href="local_setup.html#flink-on-windows">Flink on Windows</a> guide which describes -how to run Flink on Windows for local setups.</p> - -<h2 id="download">Download</h2> -<p>Download the ready to run binary package. Choose the Flink distribution that <strong>matches your Hadoop version</strong>. If you are unsure which version to choose or you just want to run locally, pick the package for Hadoop 1.2.</p> - -<ul class="nav nav-tabs"> - <li class="active"><a href="#bin-hadoop1" data-toggle="tab">Hadoop 1.2</a></li> - <li><a href="#bin-hadoop2" data-toggle="tab">Hadoop 2 (YARN)</a></li> -</ul> -<p> -<div class="tab-content text-center"> - <div class="tab-pane active" id="bin-hadoop1"> - <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-1',this.href]);" href="http://www.apache.org/dyn/closer.cgi/flink/flink-0.9.0/flink-0.9.0-bin-hadoop1.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 1.2</a> - </div> - <div class="tab-pane" id="bin-hadoop2"> - <a class="btn btn-info btn-lg" onclick="_gaq.push(['_trackEvent','Action','download-quickstart-setup-2',this.href]);" href="http://www.apache.org/dyn/closer.cgi/flink/flink-0.9.0/flink-0.9.0-bin-hadoop2.tgz";><i class="icon-download"> </i> Download Flink for Hadoop 2</a> - </div> -</div> -</p> - -<h2 id="start">Start</h2> - -<ol> - <li>Go to the download directory.</li> - <li>Unpack the downloaded archive.</li> - <li>Start Flink.</li> -</ol> - -<div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span><span class="nb">cd</span> ~/Downloads <span class="c"># Go to download directory</span> -<span class="nv">$ </span>tar xzf flink-*.tgz <span class="c"># Unpack the downloaded archive</span> -<span class="nv">$ </span><span class="nb">cd </span>flink-0.9.0 -<span class="nv">$ </span>bin/start-local.sh <span class="c"># Start Flink</span></code></pre></div> - -<p>Check the <strong>JobManagerâs web frontend</strong> at <a href="http://localhost:8081";>http://localhost:8081</a> and make -sure everything is up and running.</p> - -<p>Instead of starting Flink with <code>bin/start-local.sh</code> you can also start Flink in an streaming optimized -mode, using <code>bin/start-local-streaming.sh</code>.</p> - -<h2 id="run-example">Run Example</h2> - -<p>Run the <strong>Word Count example</strong> to see Flink at work.</p> - -<ul> - <li> - <p><strong>Download test data</strong>:</p> - - <div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>wget -O hamlet.txt http://www.gutenberg.org/cache/epub/1787/pg1787.txt</code></pre></div> - </li> - <li>You now have a text file called <em>hamlet.txt</em> in your working directory.</li> - <li> - <p><strong>Start the example program</strong>:</p> - - <div class="highlight"><pre><code class="language-bash"><span class="nv">$ </span>bin/flink run ./examples/flink-java-examples-0.9.0-WordCount.jar file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/hamlet.txt file://<span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/wordcount-result.txt</code></pre></div> - </li> - <li>You will find a file called <strong>wordcount-result.txt</strong> in your current directory.</li> -</ul> - -<h2 id="cluster-setup">Cluster Setup</h2> - -<p><strong>Running Flink on a cluster</strong> is as easy as running it locally. Having <strong>passwordless SSH</strong> and -<strong>the same directory structure</strong> on all your cluster nodes lets you use our scripts to control -everything.</p> - -<ol> - <li>Copy the unpacked <strong>flink</strong> directory from the downloaded archive to the same file system path -on each node of your setup.</li> - <li>Choose a <strong>master node</strong> (JobManager) and set the <code>jobmanager.rpc.address</code> key in -<code>conf/flink-conf.yaml</code> to its IP or hostname. Make sure that all nodes in your cluster have the same -<code>jobmanager.rpc.address</code> configured.</li> - <li>Add the IPs or hostnames (one per line) of all <strong>worker nodes</strong> (TaskManager) to the slaves files -in <code>conf/slaves</code>.</li> -</ol> - -<p>You can now <strong>start the cluster</strong> at your master node with <code>bin/start-cluster.sh</code>. If you are planning -to run only streaming jobs with Flink, you can also an optimized streaming mode: <code>start-cluster-streaming.sh</code>.</p> - -<p>The following <strong>example</strong> illustrates the setup with three nodes (with IP addresses from <em>10.0.0.1</em> -to <em>10.0.0.3</em> and hostnames <em>master</em>, <em>worker1</em>, <em>worker2</em>) and shows the contents of the -configuration files, which need to be accessible at the same path on all machines:</p> - -<div class="row"> - <div class="col-md-6 text-center"> - <img src="http://flink.apache.org/docs/0.9/page/img/quickstart_cluster.png"; style="width: 85%" /> - </div> -<div class="col-md-6"> - <div class="row"> - <p class="lead text-center"> - /path/to/<strong>flink/conf/<br />flink-conf.yaml</strong> - <pre>jobmanager.rpc.address: 10.0.0.1</pre> - </p> - </div> -<div class="row" style="margin-top: 1em;"> - <p class="lead text-center"> - /path/to/<strong>flink/<br />conf/slaves</strong> - <pre> -10.0.0.2 -10.0.0.3</pre> - </p> -</div> -</div> -</div> - -<p>Have a look at the <a href="config.html">Configuration</a> section of the documentation to see other available configuration options. -For Flink to run efficiently, a few configuration values need to be set.</p> - -<p>In particular,</p> - -<ul> - <li>the amount of available memory per TaskManager (<code>taskmanager.heap.mb</code>),</li> - <li>the number of available CPUs per machine (<code>taskmanager.numberOfTaskSlots</code>),</li> - <li>the total number of CPUs in the cluster (<code>parallelism.default</code>) and</li> - <li>the temporary directories (<code>taskmanager.tmp.dirs</code>)</li> -</ul> - -<p>are very important configuration values.</p> - -<h2 id="flink-on-yarn">Flink on YARN</h2> -<p>You can easily deploy Flink on your existing <strong>YARN cluster</strong>.</p> - -<ol> - <li>Download the <strong>Flink Hadoop2 package</strong>: <a href="http://www.apache.org/dyn/closer.cgi/flink/flink-0.9.0/flink-0.9.0-bin-hadoop2.tgz";>Flink with Hadoop 2</a></li> - <li>Make sure your <strong>HADOOP_HOME</strong> (or <em>YARN_CONF_DIR</em> or <em>HADOOP_CONF_DIR</em>) <strong>environment variable</strong> is set to read your YARN and HDFS configuration.</li> - <li>Run the <strong>YARN client</strong> with: <code>./bin/yarn-session.sh</code>. You can run the client with options <code>-n 10 -tm 8192</code> to allocate 10 TaskManagers with 8GB of memory each.</li> -</ol> - -<p>For <strong>more detailed instructions</strong>, check out the programming Guides and examples.</p> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
![http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans003.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans003.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans008.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans008.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans015.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/kmeans015.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/run-webclient.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/run-webclient.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/compiler-webclient-new.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/compiler-webclient-new.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/jobmanager-running-new.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/jobmanager-running-new.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/result003.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/result003.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/result008.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/result008.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart-example/result015.png"](http://flink.apache.org/docs/0.9/page/img/quickstart-example/result015.png"){kind=link}
![http://flink.apache.org/docs/0.9/page/img/quickstart_cluster.png"](http://flink.apache.org/docs/0.9/page/img/quickstart_cluster.png"){kind=link}