http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/apis/java8.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/apis/java8.html b/content/docs/0.9/apis/java8.html deleted file mode 100644 index 215eadd..0000000 --- a/content/docs/0.9/apis/java8.html +++ /dev/null @@ -1,360 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Java 8 Programming Guide</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Java 8 Programming Guide</h1> - - - -<p>Java 8 introduces several new language features designed for faster and clearer coding. With the most important feature, -the so-called âLambda Expressionsâ, Java 8 opens the door to functional programming. Lambda Expressions allow for implementing and -passing functions in a straightforward way without having to declare additional (anonymous) classes.</p> - -<p>The newest version of Flink supports the usage of Lambda Expressions for all operators of the Java API. -This document shows how to use Lambda Expressions and describes current limitations. For a general introduction to the -Flink API, please refer to the <a href="programming_guide.html">Programming Guide</a></p> - -<ul id="markdown-toc"> - <li><a href="#examples" id="markdown-toc-examples">Examples</a></li> - <li><a href="#compiler-limitations" id="markdown-toc-compiler-limitations">Compiler Limitations</a></li> -</ul> - -<h3 id="examples">Examples</h3> - -<p>The following example illustrates how to implement a simple, inline <code>map()</code> function that squares its input using a Lambda Expression. -The types of input <code>i</code> and output parameters of the <code>map()</code> function need not to be declared as they are inferred by the Java 8 compiler.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">)</span> -<span class="c1">// returns the squared i</span> -<span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="n">i</span> <span class="o">-></span> <span class="n">i</span><span class="o">*</span><span class="n">i</span><span class="o">)</span> -<span class="o">.</span><span class="na">print</span><span class="o">();</span></code></pre></div> - -<p>The next two examples show different implementations of a function that uses a <code>Collector</code> for output. -Functions, such as <code>flatMap()</code>, require a output type (in this case <code>String</code>) to be defined for the <code>Collector</code> in order to be type-safe. -If the <code>Collector</code> type can not be inferred from the surrounding context, it need to be declared in the Lambda Expressionâs parameter list manually. -Otherwise the output will be treated as type <code>Object</code> which can lead to undesired behaviour.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">input</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">);</span> - -<span class="c1">// collector type must be declared</span> -<span class="n">input</span><span class="o">.</span><span class="na">flatMap</span><span class="o">((</span><span class="n">Integer</span> <span class="n">number</span><span class="o">,</span> <span class="n">Collector</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">out</span><span class="o">)</span> <span class="o">-></span> <span class="o">{</span> - <span class="k">for</span><span class="o">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span> <span class="n">i</span> <span class="o"><</span> <span class="n">number</span><span class="o">;</span> <span class="n">i</span><span class="o">++)</span> <span class="o">{</span> - <span class="n">out</span><span class="o">.</span><span class="na">collect</span><span class="o">(</span><span class="s">"a"</span><span class="o">);</span> - <span class="o">}</span> -<span class="o">})</span> -<span class="c1">// returns "a", "a", "aa", "a", "aa" , "aaa"</span> -<span class="o">.</span><span class="na">print</span><span class="o">();</span></code></pre></div> - -<div class="highlight"><pre><code class="language-java"><span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">input</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">);</span> - -<span class="c1">// collector type must not be declared, it is inferred from the type of the dataset</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">manyALetters</span> <span class="o">=</span> <span class="n">input</span><span class="o">.</span><span class="na">flatMap</span><span class="o">((</span><span class="n">number</span><span class="o">,</span> <span class="n">out</span><span class="o">)</span> <span class="o">-></span> <span class="o">{</span> - <span class="k">for</span><span class="o">(</span><span class="kt">int</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="o">;</span> <span class="n">i</span> <span class="o"><</span> <span class="n">number</span><span class="o">;</span> <span class="n">i</span><span class="o">++)</span> <span class="o">{</span> - <span class="n">out</span><span class="o">.</span><span class="na">collect</span><span class="o">(</span><span class="s">"a"</span><span class="o">);</span> - <span class="o">}</span> -<span class="o">});</span> - -<span class="c1">// returns "a", "a", "aa", "a", "aa" , "aaa"</span> -<span class="n">manyALetters</span><span class="o">.</span><span class="na">print</span><span class="o">();</span></code></pre></div> - -<p>The following code demonstrates a word count which makes extensive use of Lambda Expressions.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">input</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span><span class="s">"Please count"</span><span class="o">,</span> <span class="s">"the words"</span><span class="o">,</span> <span class="s">"but not this"</span><span class="o">);</span> - -<span class="c1">// filter out strings that contain "not"</span> -<span class="n">input</span><span class="o">.</span><span class="na">filter</span><span class="o">(</span><span class="n">line</span> <span class="o">-></span> <span class="o">!</span><span class="n">line</span><span class="o">.</span><span class="na">contains</span><span class="o">(</span><span class="s">"not"</span><span class="o">))</span> -<span class="c1">// split each line by space</span> -<span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="n">line</span> <span class="o">-></span> <span class="n">line</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> -<span class="c1">// emit a pair <word,1> for each array element</span> -<span class="o">.</span><span class="na">flatMap</span><span class="o">((</span><span class="n">String</span><span class="o">[]</span> <span class="n">wordArray</span><span class="o">,</span> <span class="n">Collector</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Integer</span><span class="o">>></span> <span class="n">out</span><span class="o">)</span> - <span class="o">-></span> <span class="n">Arrays</span><span class="o">.</span><span class="na">stream</span><span class="o">(</span><span class="n">wordArray</span><span class="o">).</span><span class="na">forEach</span><span class="o">(</span><span class="n">t</span> <span class="o">-></span> <span class="n">out</span><span class="o">.</span><span class="na">collect</span><span class="o">(</span><span class="k">new</span> <span class="n">Tuple2</span><span class="o"><>(</span><span class="n">t</span><span class="o">,</span> <span class="mi">1</span><span class="o">)))</span> - <span class="o">)</span> -<span class="c1">// group and sum up</span> -<span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">).</span><span class="na">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> -<span class="c1">// print</span> -<span class="o">.</span><span class="na">print</span><span class="o">();</span></code></pre></div> - -<h3 id="compiler-limitations">Compiler Limitations</h3> -<p>Currently, Flink only supports jobs containing Lambda Expressions completely if they are <strong>compiled with the Eclipse JDT compiler contained in Eclipse Luna 4.4.2 (and above)</strong>.</p> - -<p>Only the Eclipse JDT compiler preserves the generic type information necessary to use the entire Lambda Expressions feature type-safely. -Other compilers such as the OpenJDKâs and Oracle JDKâs <code>javac</code> throw away all generic parameters related to Lambda Expressions. This means that types such as <code>Tuple2<String,Integer</code> or <code>Collector<String></code> declared as a Lambda function input or output parameter will be pruned to <code>Tuple2</code> or <code>Collector</code> in the compiled <code>.class</code> files, which is too little information for the Flink Compiler.</p> - -<p>How to compile a Flink job that contains Lambda Expressions with the JDT compiler will be covered in the next section.</p> - -<p>However, it is possible to implement functions such as <code>map()</code> or <code>filter()</code> with Lambda Expressions in Java 8 compilers other than the Eclipse JDT compiler as long as the function has no <code>Collector</code>s or <code>Iterable</code>s <em>and</em> only if the function handles unparameterized types such as <code>Integer</code>, <code>Long</code>, <code>String</code>, <code>MyOwnClass</code> (types without Generics!).</p> - -<h4 id="compile-flink-jobs-with-the-eclipse-jdt-compiler-and-maven">Compile Flink jobs with the Eclipse JDT compiler and Maven</h4> - -<p>If you are using the Eclipse IDE, you can run and debug your Flink code within the IDE without any problems after some configuration steps. The Eclipse IDE by default compiles its Java sources with the Eclipse JDT compiler. The next section describes how to configure the Eclipse IDE.</p> - -<p>If you are using a different IDE such as IntelliJ IDEA or you want to package your Jar-File with Maven to run your job on a cluster, you need to modify your projectâs <code>pom.xml</code> file and build your program with Maven. The <a href="http://flink.apache.org/docs/0.9/quickstart/setup_quickstart.html";>quickstart</a> contains preconfigured Maven projects which can be used for new projects or as a reference. Uncomment the mentioned lines in your generated quickstart <code>pom.xml</code> file if you want to use Java 8 with Lambda Expressions.</p> - -<p>Alternatively, you can manually insert the following lines to your Maven <code>pom.xml</code> file. Maven will then use the Eclipse JDT compiler for compilation.</p> - -<div class="highlight"><pre><code class="language-xml"><span class="c"><!-- put these lines under "project/build/pluginManagement/plugins" of your pom.xml --></span> - -<span class="nt"><plugin></span> - <span class="c"><!-- Use compiler plugin with tycho as the adapter to the JDT compiler. --></span> - <span class="nt"><artifactId></span>maven-compiler-plugin<span class="nt"></artifactId></span> - <span class="nt"><configuration></span> - <span class="nt"><source></span>1.8<span class="nt"></source></span> - <span class="nt"><target></span>1.8<span class="nt"></target></span> - <span class="nt"><compilerId></span>jdt<span class="nt"></compilerId></span> - <span class="nt"></configuration></span> - <span class="nt"><dependencies></span> - <span class="c"><!-- This dependency provides the implementation of compiler "jdt": --></span> - <span class="nt"><dependency></span> - <span class="nt"><groupId></span>org.eclipse.tycho<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>tycho-compiler-jdt<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.21.0<span class="nt"></version></span> - <span class="nt"></dependency></span> - <span class="nt"></dependencies></span> -<span class="nt"></plugin></span></code></pre></div> - -<p>If you are using Eclipse for development, the m2e plugin might complain about the inserted lines above and marks your <code>pom.xml</code> as invalid. If so, insert the following lines to your <code>pom.xml</code>.</p> - -<div class="highlight"><pre><code class="language-xml"><span class="c"><!-- put these lines under "project/build/pluginManagement/plugins/plugin[groupId="org.eclipse.m2e", artifactId="lifecycle-mapping"]/configuration/lifecycleMappingMetadata/pluginExecutions" of your pom.xml --></span> - -<span class="nt"><pluginExecution></span> - <span class="nt"><pluginExecutionFilter></span> - <span class="nt"><groupId></span>org.apache.maven.plugins<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>maven-compiler-plugin<span class="nt"></artifactId></span> - <span class="nt"><versionRange></span>[3.1,)<span class="nt"></versionRange></span> - <span class="nt"><goals></span> - <span class="nt"><goal></span>testCompile<span class="nt"></goal></span> - <span class="nt"><goal></span>compile<span class="nt"></goal></span> - <span class="nt"></goals></span> - <span class="nt"></pluginExecutionFilter></span> - <span class="nt"><action></span> - <span class="nt"><ignore></ignore></span> - <span class="nt"></action></span> -<span class="nt"></pluginExecution></span></code></pre></div> - -<h4 id="run-and-debug-flink-jobs-within-the-eclipse-ide">Run and debug Flink jobs within the Eclipse IDE</h4> - -<p>First of all, make sure you are running a current version of Eclipse IDE (4.4.2 or later). Also make sure that you have a Java 8 Runtime Environment (JRE) installed in Eclipse IDE (<code>Window</code> -> <code>Preferences</code> -> <code>Java</code> -> <code>Installed JREs</code>).</p> - -<p>Create/Import your Eclipse project.</p> - -<p>If you are using Maven, you also need to change the Java version in your <code>pom.xml</code> for the <code>maven-compiler-plugin</code>. Otherwise right click the <code>JRE System Library</code> section of your project and open the <code>Properties</code> window in order to switch to a Java 8 JRE (or above) that supports Lambda Expressions.</p> - -<p>The Eclipse JDT compiler needs a special compiler flag in order to store type information in <code>.class</code> files. Open the JDT configuration file at <code>{project directoy}/.settings/org.eclipse.jdt.core.prefs</code> with your favorite text editor and add the following line:</p> - -<div class="highlight"><pre><code>org.eclipse.jdt.core.compiler.codegen.lambda.genericSignature=generate -</code></pre></div> - -<p>If not already done, also modify the Java versions of the following properties to <code>1.8</code> (or above):</p> - -<div class="highlight"><pre><code>org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.source=1.8 -</code></pre></div> - -<p>After you have saved the file, perform a complete project refresh in Eclipse IDE.</p> - -<p>If you are using Maven, right click your Eclipse project and select <code>Maven</code> -> <code>Update Project...</code>.</p> - -<p>You have configured everything correctly, if the following Flink program runs without exceptions:</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> -<span class="n">env</span><span class="o">.</span><span class="na">fromElements</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">).</span><span class="na">map</span><span class="o">((</span><span class="n">in</span><span class="o">)</span> <span class="o">-></span> <span class="k">new</span> <span class="n">Tuple1</span><span class="o"><</span><span class="n">String</span><span class="o">>(</span><span class="s">" "</span> <span class="o">+</span> <span class="n">in</span><span class="o">)).</span><span class="na">print</span><span class="o">();</span> -<span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">();</span></code></pre></div> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/apis/local_execution.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/apis/local_execution.html b/content/docs/0.9/apis/local_execution.html deleted file mode 100644 index 25cac1f..0000000 --- a/content/docs/0.9/apis/local_execution.html +++ /dev/null @@ -1,302 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Local Execution</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Local Execution</h1> - - - -<p>Flink can run on a single machine, even in a single Java Virtual Machine. This allows users to test and debug Flink programs locally. This section gives an overview of the local execution mechanisms.</p> - -<p>The local environments and executors allow you to run Flink programs in a local Java Virtual Machine, or with within any JVM as part of existing programs. Most examples can be launched locally by simply hitting the âRunâ button of your IDE.</p> - -<p>There are two different kinds of local execution supported in Flink. The <code>LocalExecutionEnvironment</code> is starting the full Flink runtime, including a JobManager and a TaskManager. These include memory management and all the internal algorithms that are executed in the cluster mode.</p> - -<p>The <code>CollectionEnvironment</code> is executing the Flink program on Java collections. This mode will not start the full Flink runtime, so the execution is very low-overhead and lightweight. For example a <code>DataSet.map()</code>-transformation will be executed by applying the <code>map()</code> function to all elements in a Java list.</p> - -<ul id="markdown-toc"> - <li><a href="#debugging" id="markdown-toc-debugging">Debugging</a></li> - <li><a href="#maven-dependency" id="markdown-toc-maven-dependency">Maven Dependency</a></li> - <li><a href="#local-environment" id="markdown-toc-local-environment">Local Environment</a></li> - <li><a href="#collection-environment" id="markdown-toc-collection-environment">Collection Environment</a></li> -</ul> - -<h2 id="debugging">Debugging</h2> - -<p>If you are running Flink programs locally, you can also debug your program like any other Java program. You can either use <code>System.out.println()</code> to write out some internal variables or you can use the debugger. It is possible to set breakpoints within <code>map()</code>, <code>reduce()</code> and all the other methods. -Please also refer to the <a href="programming_guide.html#debugging">debugging section</a> in the Java API documentation for a guide to testing and local debugging utilities in the Java API.</p> - -<h2 id="maven-dependency">Maven Dependency</h2> - -<p>If you are developing your program in a Maven project, you have to add the <code>flink-clients</code> module using this dependency:</p> - -<div class="highlight"><pre><code class="language-xml"><span class="nt"><dependency></span> - <span class="nt"><groupId></span>org.apache.flink<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>flink-clients<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.9.0<span class="nt"></version></span> -<span class="nt"></dependency></span></code></pre></div> - -<h2 id="local-environment">Local Environment</h2> - -<p>The <code>LocalEnvironment</code> is a handle to local execution for Flink programs. Use it to run a program within a local JVM - standalone or embedded in other programs.</p> - -<p>The local environment is instantiated via the method <code>ExecutionEnvironment.createLocalEnvironment()</code>. By default, it will use as many local threads for execution as your machine has CPU cores (hardware contexts). You can alternatively specify the desired parallelism. The local environment can be configured to log to the console using <code>enableLogging()</code>/<code>disableLogging()</code>.</p> - -<p>In most cases, calling <code>ExecutionEnvironment.getExecutionEnvironment()</code> is the even better way to go. That method returns a <code>LocalEnvironment</code> when the program is started locally (outside the command line interface), and it returns a pre-configured environment for cluster execution, when the program is invoked by the <a href="cli.html">command line interface</a>.</p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> - <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">createLocalEnvironment</span><span class="o">();</span> - - <span class="n">DataSet</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">data</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">readTextFile</span><span class="o">(</span><span class="s">"file:///path/to/file"</span><span class="o">);</span> - - <span class="n">data</span> - <span class="o">.</span><span class="na">filter</span><span class="o">(</span><span class="k">new</span> <span class="n">FilterFunction</span><span class="o"><</span><span class="n">String</span><span class="o">>()</span> <span class="o">{</span> - <span class="kd">public</span> <span class="kt">boolean</span> <span class="nf">filter</span><span class="o">(</span><span class="n">String</span> <span class="n">value</span><span class="o">)</span> <span class="o">{</span> - <span class="k">return</span> <span class="n">value</span><span class="o">.</span><span class="na">startsWith</span><span class="o">(</span><span class="s">"http://"</span><span class="o">);</span> - <span class="o">}</span> - <span class="o">})</span> - <span class="o">.</span><span class="na">writeAsText</span><span class="o">(</span><span class="s">"file:///path/to/result"</span><span class="o">);</span> - - <span class="n">JobExecutionResult</span> <span class="n">res</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">();</span> -<span class="o">}</span></code></pre></div> - -<p>The <code>JobExecutionResult</code> object, which is returned after the execution finished, contains the program runtime and the accumulator results.</p> - -<p>The <code>LocalEnvironment</code> allows also to pass custom configuration values to Flink.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">Configuration</span> <span class="n">conf</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">Configuration</span><span class="o">();</span> -<span class="n">conf</span><span class="o">.</span><span class="na">setFloat</span><span class="o">(</span><span class="n">ConfigConstants</span><span class="o">.</span><span class="na">TASK_MANAGER_MEMORY_FRACTION_KEY</span><span class="o">,</span> <span class="mf">0.5f</span><span class="o">);</span> -<span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">createLocalEnvironment</span><span class="o">(</span><span class="n">conf</span><span class="o">);</span></code></pre></div> - -<p><em>Note:</em> The local execution environments do not start any web frontend to monitor the execution.</p> - -<h2 id="collection-environment">Collection Environment</h2> - -<p>The execution on Java Collections using the <code>CollectionEnvironment</code> is a low-overhead approach for executing Flink programs. Typical use-cases for this mode are automated tests, debugging and code re-use.</p> - -<p>Users can use algorithms implemented for batch processing also for cases that are more interactive. A slightly changed variant of a Flink program could be used in a Java Application Server for processing incoming requests.</p> - -<p><strong>Skeleton for Collection-based execution</strong></p> - -<div class="highlight"><pre><code class="language-java"><span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="kd">throws</span> <span class="n">Exception</span> <span class="o">{</span> - <span class="c1">// initialize a new Collection-based execution environment</span> - <span class="kd">final</span> <span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="k">new</span> <span class="nf">CollectionEnvironment</span><span class="o">();</span> - - <span class="n">DataSet</span><span class="o"><</span><span class="n">User</span><span class="o">></span> <span class="n">users</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">fromCollection</span><span class="o">(</span> <span class="cm">/* get elements from a Java Collection */</span><span class="o">);</span> - - <span class="cm">/* Data Set transformations ... */</span> - - <span class="c1">// retrieve the resulting Tuple2 elements into a ArrayList.</span> - <span class="n">Collection</span><span class="o"><...></span> <span class="n">result</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><...>();</span> - <span class="n">resultDataSet</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="k">new</span> <span class="n">LocalCollectionOutputFormat</span><span class="o"><...>(</span><span class="n">result</span><span class="o">));</span> - - <span class="c1">// kick off execution.</span> - <span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">();</span> - - <span class="c1">// Do some work with the resulting ArrayList (=Collection).</span> - <span class="k">for</span><span class="o">(...</span> <span class="n">t</span> <span class="o">:</span> <span class="n">result</span><span class="o">)</span> <span class="o">{</span> - <span class="n">System</span><span class="o">.</span><span class="na">err</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"Result = "</span><span class="o">+</span><span class="n">t</span><span class="o">);</span> - <span class="o">}</span> -<span class="o">}</span></code></pre></div> - -<p>The <code>flink-java-examples</code> module contains a full example, called <code>CollectionExecutionExample</code>.</p> - -<p>Please note that the execution of the collection-based Flink programs is only possible on small data, which fits into the JVM heap. The execution on collections is not multi-threaded, only one thread is used.</p> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
![http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"){kind=link}