http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/apis/hadoop_compatibility.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/apis/hadoop_compatibility.html b/content/docs/0.9/apis/hadoop_compatibility.html deleted file mode 100644 index cbbafe7..0000000 --- a/content/docs/0.9/apis/hadoop_compatibility.html +++ /dev/null @@ -1,423 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Hadoop Compatibility</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Hadoop Compatibility <span class="badge">Beta</span></h1> - - - -<p>Flink is compatible with Apache Hadoop MapReduce interfaces and therefore allows -reusing code that was implemented for Hadoop MapReduce.</p> - -<p>You can:</p> - -<ul> - <li>use Hadoopâs <code>Writable</code> <a href="programming_guide.html#data-types">data types</a> in Flink programs.</li> - <li>use any Hadoop <code>InputFormat</code> as a <a href="programming_guide.html#data-sources">DataSource</a>.</li> - <li>use any Hadoop <code>OutputFormat</code> as a <a href="programming_guide.html#data-sinks">DataSink</a>.</li> - <li>use a Hadoop <code>Mapper</code> as <a href="dataset_transformations.html#flatmap">FlatMapFunction</a>.</li> - <li>use a Hadoop <code>Reducer</code> as <a href="dataset_transformations.html#groupreduce-on-grouped-dataset">GroupReduceFunction</a>.</li> -</ul> - -<p>This document shows how to use existing Hadoop MapReduce code with Flink. Please refer to the -<a href="example_connectors.html">Connecting to other systems</a> guide for reading from Hadoop supported file systems.</p> - -<ul id="markdown-toc"> - <li><a href="#project-configuration" id="markdown-toc-project-configuration">Project Configuration</a></li> - <li><a href="#using-hadoop-data-types" id="markdown-toc-using-hadoop-data-types">Using Hadoop Data Types</a></li> - <li><a href="#using-hadoop-inputformats" id="markdown-toc-using-hadoop-inputformats">Using Hadoop InputFormats</a></li> - <li><a href="#using-hadoop-outputformats" id="markdown-toc-using-hadoop-outputformats">Using Hadoop OutputFormats</a></li> - <li><a href="#using-hadoop-mappers-and-reducers" id="markdown-toc-using-hadoop-mappers-and-reducers">Using Hadoop Mappers and Reducers</a></li> - <li><a href="#complete-hadoop-wordcount-example" id="markdown-toc-complete-hadoop-wordcount-example">Complete Hadoop WordCount Example</a></li> -</ul> - -<h3 id="project-configuration">Project Configuration</h3> - -<p>Support for Haddop input/output formats is part of the <code>flink-java</code> and -<code>flink-scala</code> Maven modules that are always required when writing Flink jobs. -The code is located in <code>org.apache.flink.api.java.hadoop</code> and -<code>org.apache.flink.api.scala.hadoop</code> in an additional sub-package for the -<code>mapred</code> and <code>mapreduce</code> API.</p> - -<p>Support for Hadoop Mappers and Reducers is contained in the <code>flink-staging</code> -Maven module. -This code resides in the <code>org.apache.flink.hadoopcompatibility</code> -package.</p> - -<p>Add the following dependency to your <code>pom.xml</code> if you want to reuse Mappers -and Reducers.</p> - -<div class="highlight"><pre><code class="language-xml"><span class="nt"><dependency></span> - <span class="nt"><groupId></span>org.apache.flink<span class="nt"></groupId></span> - <span class="nt"><artifactId></span>flink-hadoop-compatibility<span class="nt"></artifactId></span> - <span class="nt"><version></span>0.9.0<span class="nt"></version></span> -<span class="nt"></dependency></span></code></pre></div> - -<h3 id="using-hadoop-data-types">Using Hadoop Data Types</h3> - -<p>Flink supports all Hadoop <code>Writable</code> and <code>WritableComparable</code> data types -out-of-the-box. You do not need to include the Hadoop Compatibility dependency, -if you only want to use your Hadoop data types. See the -<a href="programming_guide.html#data-types">Programming Guide</a> for more details.</p> - -<h3 id="using-hadoop-inputformats">Using Hadoop InputFormats</h3> - -<p>Hadoop input formats can be used to create a data source by using -one of the methods <code>readHadoopFile</code> or <code>createHadoopInput</code> of the -<code>ExecutionEnvironment</code>. The former is used for input formats derived -from <code>FileInputFormat</code> while the latter has to be used for general purpose -input formats.</p> - -<p>The resulting <code>DataSet</code> contains 2-tuples where the first field -is the key and the second field is the value retrieved from the Hadoop -InputFormat.</p> - -<p>The following example shows how to use Hadoopâs <code>TextInputFormat</code>.</p> - -<div class="codetabs"> - <div data-lang="java"> - - <div class="highlight"><pre><code class="language-java"><span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> - -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">>></span> <span class="n">input</span> <span class="o">=</span> - <span class="n">env</span><span class="o">.</span><span class="na">readHadoopFile</span><span class="o">(</span><span class="k">new</span> <span class="nf">TextInputFormat</span><span class="o">(),</span> <span class="n">LongWritable</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">Text</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">textPath</span><span class="o">);</span> - -<span class="c1">// Do something with the data.</span> -<span class="o">[...]</span></code></pre></div> - - </div> - <div data-lang="scala"> - - <div class="highlight"><pre><code class="language-scala"><span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span> - -<span class="k">val</span> <span class="n">input</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[(</span><span class="kt">LongWritable</span>, <span class="kt">Text</span><span class="o">)]</span> <span class="k">=</span> - <span class="n">env</span><span class="o">.</span><span class="n">readHadoopFile</span><span class="o">(</span><span class="k">new</span> <span class="nc">TextInputFormat</span><span class="o">,</span> <span class="n">classOf</span><span class="o">[</span><span class="kt">LongWritable</span><span class="o">],</span> <span class="n">classOf</span><span class="o">[</span><span class="kt">Text</span><span class="o">],</span> <span class="n">textPath</span><span class="o">)</span> - -<span class="c1">// Do something with the data.</span> -<span class="o">[</span><span class="kt">...</span><span class="o">]</span></code></pre></div> - - </div> - -</div> - -<h3 id="using-hadoop-outputformats">Using Hadoop OutputFormats</h3> - -<p>Flink provides a compatibility wrapper for Hadoop <code>OutputFormats</code>. Any class -that implements <code>org.apache.hadoop.mapred.OutputFormat</code> or extends -<code>org.apache.hadoop.mapreduce.OutputFormat</code> is supported. -The OutputFormat wrapper expects its input data to be a DataSet containing -2-tuples of key and value. These are to be processed by the Hadoop OutputFormat.</p> - -<p>The following example shows how to use Hadoopâs <code>TextOutputFormat</code>.</p> - -<div class="codetabs"> - <div data-lang="java"> - - <div class="highlight"><pre><code class="language-java"><span class="c1">// Obtain the result we want to emit</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">>></span> <span class="n">hadoopResult</span> <span class="o">=</span> <span class="o">[...]</span> - -<span class="c1">// Set up the Hadoop TextOutputFormat.</span> -<span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">></span> <span class="n">hadoopOF</span> <span class="o">=</span> - <span class="c1">// create the Flink wrapper.</span> - <span class="k">new</span> <span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">>(</span> - <span class="c1">// set the Hadoop OutputFormat and specify the job.</span> - <span class="k">new</span> <span class="n">TextOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">>(),</span> <span class="n">job</span> - <span class="o">);</span> -<span class="n">hadoopOF</span><span class="o">.</span><span class="na">getConfiguration</span><span class="o">().</span><span class="na">set</span><span class="o">(</span><span class="s">"mapreduce.output.textoutputformat.separator"</span><span class="o">,</span> <span class="s">" "</span><span class="o">);</span> -<span class="n">TextOutputFormat</span><span class="o">.</span><span class="na">setOutputPath</span><span class="o">(</span><span class="n">job</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">outputPath</span><span class="o">));</span> - -<span class="c1">// Emit data using the Hadoop TextOutputFormat.</span> -<span class="n">hadoopResult</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="n">hadoopOF</span><span class="o">);</span></code></pre></div> - - </div> - <div data-lang="scala"> - - <div class="highlight"><pre><code class="language-scala"><span class="c1">// Obtain your result to emit.</span> -<span class="k">val</span> <span class="n">hadoopResult</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[(</span><span class="kt">Text</span>, <span class="kt">IntWritable</span><span class="o">)]</span> <span class="k">=</span> <span class="o">[</span><span class="kt">...</span><span class="o">]</span> - -<span class="k">val</span> <span class="n">hadoopOF</span> <span class="k">=</span> <span class="k">new</span> <span class="nc">HadoopOutputFormat</span><span class="o">[</span><span class="kt">Text</span>,<span class="kt">IntWritable</span><span class="o">](</span> - <span class="k">new</span> <span class="nc">TextOutputFormat</span><span class="o">[</span><span class="kt">Text</span>, <span class="kt">IntWritable</span><span class="o">],</span> - <span class="k">new</span> <span class="nc">JobConf</span><span class="o">)</span> - -<span class="n">hadoopOF</span><span class="o">.</span><span class="n">getJobConf</span><span class="o">.</span><span class="n">set</span><span class="o">(</span><span class="s">"mapred.textoutputformat.separator"</span><span class="o">,</span> <span class="s">" "</span><span class="o">)</span> -<span class="nc">FileOutputFormat</span><span class="o">.</span><span class="n">setOutputPath</span><span class="o">(</span><span class="n">hadoopOF</span><span class="o">.</span><span class="n">getJobConf</span><span class="o">,</span> <span class="k">new</span> <span class="nc">Path</span><span class="o">(</span><span class="n">resultPath</span><span class="o">))</span> - -<span class="n">hadoopResult</span><span class="o">.</span><span class="n">output</span><span class="o">(</span><span class="n">hadoopOF</span><span class="o">)</span></code></pre></div> - - </div> - -</div> - -<h3 id="using-hadoop-mappers-and-reducers">Using Hadoop Mappers and Reducers</h3> - -<p>Hadoop Mappers are semantically equivalent to Flinkâs <a href="dataset_transformations.html#flatmap">FlatMapFunctions</a> and Hadoop Reducers are equivalent to Flinkâs <a href="dataset_transformations.html#groupreduce-on-grouped-dataset">GroupReduceFunctions</a>. Flink provides wrappers for implementations of Hadoop MapReduceâs <code>Mapper</code> and <code>Reducer</code> interfaces, i.e., you can reuse your Hadoop Mappers and Reducers in regular Flink programs. At the moment, only the Mapper and Reduce interfaces of Hadoopâs mapred API (<code>org.apache.hadoop.mapred</code>) are supported.</p> - -<p>The wrappers take a <code>DataSet<Tuple2<KEYIN,VALUEIN>></code> as input and produce a <code>DataSet<Tuple2<KEYOUT,VALUEOUT>></code> as output where <code>KEYIN</code> and <code>KEYOUT</code> are the keys and <code>VALUEIN</code> and <code>VALUEOUT</code> are the values of the Hadoop key-value pairs that are processed by the Hadoop functions. For Reducers, Flink offers a wrapper for a GroupReduceFunction with (<code>HadoopReduceCombineFunction</code>) and without a Combiner (<code>HadoopReduceFunction</code>). The wrappers accept an optional <code>JobConf</code> object to configure the Hadoop Mapper or Reducer.</p> - -<p>Flinkâs function wrappers are</p> - -<ul> - <li><code>org.apache.flink.hadoopcompatibility.mapred.HadoopMapFunction</code>,</li> - <li><code>org.apache.flink.hadoopcompatibility.mapred.HadoopReduceFunction</code>, and</li> - <li><code>org.apache.flink.hadoopcompatibility.mapred.HadoopReduceCombineFunction</code>.</li> -</ul> - -<p>and can be used as regular Flink <a href="dataset_transformations.html#flatmap">FlatMapFunctions</a> or <a href="dataset_transformations.html#groupreduce-on-grouped-dataset">GroupReduceFunctions</a>.</p> - -<p>The following example shows how to use Hadoop <code>Mapper</code> and <code>Reducer</code> functions.</p> - -<div class="highlight"><pre><code class="language-java"><span class="c1">// Obtain data to process somehow.</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>></span> <span class="n">text</span> <span class="o">=</span> <span class="o">[...]</span> - -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>></span> <span class="n">result</span> <span class="o">=</span> <span class="n">text</span> - <span class="c1">// use Hadoop Mapper (Tokenizer) as MapFunction</span> - <span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopMapFunction</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> - <span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()</span> - <span class="o">))</span> - <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// use Hadoop Reducer (Counter) as Reduce- and CombineFunction</span> - <span class="o">.</span><span class="na">reduceGroup</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopReduceCombineFunction</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> - <span class="k">new</span> <span class="nf">Counter</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Counter</span><span class="o">()</span> - <span class="o">));</span></code></pre></div> - -<p><strong>Please note:</strong> The Reducer wrapper works on groups as defined by Flinkâs <a href="dataset_transformations.html#transformations-on-grouped-dataset">groupBy()</a> operation. It does not consider any custom partitioners, sort or grouping comparators you might have set in the <code>JobConf</code>.</p> - -<h3 id="complete-hadoop-wordcount-example">Complete Hadoop WordCount Example</h3> - -<p>The following example shows a complete WordCount implementation using Hadoop data types, Input- and OutputFormats, and Mapper and Reducer implementations.</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">ExecutionEnvironment</span> <span class="n">env</span> <span class="o">=</span> <span class="n">ExecutionEnvironment</span><span class="o">.</span><span class="na">getExecutionEnvironment</span><span class="o">();</span> - -<span class="c1">// Set up the Hadoop TextInputFormat.</span> -<span class="n">Job</span> <span class="n">job</span> <span class="o">=</span> <span class="n">Job</span><span class="o">.</span><span class="na">getInstance</span><span class="o">();</span> -<span class="n">HadoopInputFormat</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">></span> <span class="n">hadoopIF</span> <span class="o">=</span> - <span class="k">new</span> <span class="n">HadoopInputFormat</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">>(</span> - <span class="k">new</span> <span class="nf">TextInputFormat</span><span class="o">(),</span> <span class="n">LongWritable</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">Text</span><span class="o">.</span><span class="na">class</span><span class="o">,</span> <span class="n">job</span> - <span class="o">);</span> -<span class="n">TextInputFormat</span><span class="o">.</span><span class="na">addInputPath</span><span class="o">(</span><span class="n">job</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">inputPath</span><span class="o">));</span> - -<span class="c1">// Read data using the Hadoop TextInputFormat.</span> -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">>></span> <span class="n">text</span> <span class="o">=</span> <span class="n">env</span><span class="o">.</span><span class="na">createInput</span><span class="o">(</span><span class="n">hadoopIF</span><span class="o">);</span> - -<span class="n">DataSet</span><span class="o"><</span><span class="n">Tuple2</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>></span> <span class="n">result</span> <span class="o">=</span> <span class="n">text</span> - <span class="c1">// use Hadoop Mapper (Tokenizer) as MapFunction</span> - <span class="o">.</span><span class="na">flatMap</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopMapFunction</span><span class="o"><</span><span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> - <span class="k">new</span> <span class="nf">Tokenizer</span><span class="o">()</span> - <span class="o">))</span> - <span class="o">.</span><span class="na">groupBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// use Hadoop Reducer (Counter) as Reduce- and CombineFunction</span> - <span class="o">.</span><span class="na">reduceGroup</span><span class="o">(</span><span class="k">new</span> <span class="n">HadoopReduceCombineFunction</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">,</span> <span class="n">Text</span><span class="o">,</span> <span class="n">LongWritable</span><span class="o">>(</span> - <span class="k">new</span> <span class="nf">Counter</span><span class="o">(),</span> <span class="k">new</span> <span class="nf">Counter</span><span class="o">()</span> - <span class="o">));</span> - -<span class="c1">// Set up the Hadoop TextOutputFormat.</span> -<span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">></span> <span class="n">hadoopOF</span> <span class="o">=</span> - <span class="k">new</span> <span class="n">HadoopOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">>(</span> - <span class="k">new</span> <span class="n">TextOutputFormat</span><span class="o"><</span><span class="n">Text</span><span class="o">,</span> <span class="n">IntWritable</span><span class="o">>(),</span> <span class="n">job</span> - <span class="o">);</span> -<span class="n">hadoopOF</span><span class="o">.</span><span class="na">getConfiguration</span><span class="o">().</span><span class="na">set</span><span class="o">(</span><span class="s">"mapreduce.output.textoutputformat.separator"</span><span class="o">,</span> <span class="s">" "</span><span class="o">);</span> -<span class="n">TextOutputFormat</span><span class="o">.</span><span class="na">setOutputPath</span><span class="o">(</span><span class="n">job</span><span class="o">,</span> <span class="k">new</span> <span class="nf">Path</span><span class="o">(</span><span class="n">outputPath</span><span class="o">));</span> - -<span class="c1">// Emit data using the Hadoop TextOutputFormat.</span> -<span class="n">result</span><span class="o">.</span><span class="na">output</span><span class="o">(</span><span class="n">hadoopOF</span><span class="o">);</span> - -<span class="c1">// Execute Program</span> -<span class="n">env</span><span class="o">.</span><span class="na">execute</span><span class="o">(</span><span class="s">"Hadoop WordCount"</span><span class="o">);</span></code></pre></div> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/apis/index.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/apis/index.html b/content/docs/0.9/apis/index.html deleted file mode 100644 index d68e2e7..0000000 --- a/content/docs/0.9/apis/index.html +++ /dev/null @@ -1,205 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Programming Guides</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Programming Guides</h1> - - - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/f0ac0cdb/content/docs/0.9/apis/iterations.html ---------------------------------------------------------------------- diff --git a/content/docs/0.9/apis/iterations.html b/content/docs/0.9/apis/iterations.html deleted file mode 100644 index e5065fe..0000000 --- a/content/docs/0.9/apis/iterations.html +++ /dev/null @@ -1,399 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<!DOCTYPE html> - -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - - <title>Apache Flink 0.9.0 Documentation: Iterations</title> - - <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico"; type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css";> - <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css";> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - - - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="http://flink.apache.org";><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg";></a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li><a href="http://flink.apache.org/docs/0.9/index.html";>Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li> - - <!-- Setup --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/setup"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/setup/building.html";>Get Flink 0.9-SNAPSHOT</a></li> - - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html"; class="active">Local</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html";>Cluster (Standalone)</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html";>YARN</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html";>GCloud</a></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html";>Flink on Tez <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="http://flink.apache.org/docs/0.9/setup/config.html";>Configuration</a></li> - </ul> - </li> - - <!-- Programming Guides --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/apis"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html";><strong>Batch: DataSet API</strong></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html";><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/python.html";>Python API <span class="badge">Beta</span></a></li> - - <li class="divider"></li> - <li><a href="scala_shell.html">Interactive Scala Shell</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html";>Dataset Transformations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html";>Best Practices</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html";>Connectors</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html";>Examples</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html";>Local Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html";>Cluster Execution</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html";>Command Line Interface</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html";>Web Client</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html";>Iterations</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html";>Java 8</a></li> - <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html";>Hadoop Compatability <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Libraries --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/libs"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html";>Graphs: Spargel</a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Graphs: Gelly <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/ml/";>Machine Learning <span class="badge">Beta</span></a></li> - <li><a href="http://flink.apache.org/docs/0.9/libs/table.html";>Relational: Table <span class="badge">Beta</span></a></li> - </ul> - </li> - - <!-- Internals --> - <li class="dropdown"> - <a href="http://flink.apache.org/docs/0.9/internals"; class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html";>How to Contribute</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html";>Coding Guidelines</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html";>IDE Setup</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html";>Logging</a></li> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Internals</strong></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html";>Architecture & Process Model</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html";>Type Extraction & Serialization</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html";>Jobs & Scheduling</a></li> - <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html";>How-To: Add an Operator</a></li> - </ul> - </li> - </ul> - <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html";> - <div class="form-group"> - <input type="text" class="form-control" name="q" placeholder="Search all pages"> - </div> - <button type="submit" class="btn btn-default">Search</button> - </form> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-10 col-sm-offset-1"> - <h1>Iterations</h1> - - - -<p>Iterative algorithms occur in many domains of data analysis, such as <em>machine learning</em> or <em>graph analysis</em>. Such algorithms are crucial in order to realize the promise of Big Data to extract meaningful information out of your data. With increasing interest to run these kinds of algorithms on very large data sets, there is a need to execute iterations in a massively parallel fashion.</p> - -<p>Flink programs implement iterative algorithms by defining a <strong>step function</strong> and embedding it into a special iteration operator. There are two variants of this operator: <strong>Iterate</strong> and <strong>Delta Iterate</strong>. Both operators repeatedly invoke the step function on the current iteration state until a certain termination condition is reached.</p> - -<p>Here, we provide background on both operator variants and outline their usage. The <a href="programming_guide.html">programming guide</a> explains how to implement the operators in both Scala and Java. We also support both <strong>vertex-centric and gather-sum-apply iterations</strong> through Flinkâs graph processing API, <a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html";>Gelly</a>.</p> - -<p>The following table provides an overview of both operators:</p> - -<table class="table table-striped table-hover table-bordered"> - <thead> - <th></th> - <th class="text-center">Iterate</th> - <th class="text-center">Delta Iterate</th> - </thead> - <tr> - <td class="text-center" width="20%"><strong>Iteration Input</strong></td> - <td class="text-center" width="40%"><strong>Partial Solution</strong></td> - <td class="text-center" width="40%"><strong>Workset</strong> and <strong>Solution Set</strong></td> - </tr> - <tr> - <td class="text-center"><strong>Step Function</strong></td> - <td colspan="2" class="text-center">Arbitrary Data Flows</td> - </tr> - <tr> - <td class="text-center"><strong>State Update</strong></td> - <td class="text-center">Next <strong>partial solution</strong></td> - <td> - <ul> - <li>Next workset</li> - <li><strong>Changes to solution set</strong></li> - </ul> - </td> - </tr> - <tr> - <td class="text-center"><strong>Iteration Result</strong></td> - <td class="text-center">Last partial solution</td> - <td class="text-center">Solution set state after last iteration</td> - </tr> - <tr> - <td class="text-center"><strong>Termination</strong></td> - <td> - <ul> - <li><strong>Maximum number of iterations</strong> (default)</li> - <li>Custom aggregator convergence</li> - </ul> - </td> - <td> - <ul> - <li><strong>Maximum number of iterations or empty workset</strong> (default)</li> - <li>Custom aggregator convergence</li> - </ul> - </td> - </tr> -</table> - -<ul id="markdown-toc"> - <li><a href="#iterate-operator" id="markdown-toc-iterate-operator">Iterate Operator</a> <ul> - <li><a href="#example-incrementing-numbers" id="markdown-toc-example-incrementing-numbers">Example: Incrementing Numbers</a></li> - </ul> - </li> - <li><a href="#delta-iterate-operator" id="markdown-toc-delta-iterate-operator">Delta Iterate Operator</a> <ul> - <li><a href="#example-propagate-minimum-in-graph" id="markdown-toc-example-propagate-minimum-in-graph">Example: Propagate Minimum in Graph</a></li> - </ul> - </li> - <li><a href="#superstep-synchronization" id="markdown-toc-superstep-synchronization">Superstep Synchronization</a></li> -</ul> - -<h2 id="iterate-operator">Iterate Operator</h2> - -<p>The <strong>iterate operator</strong> covers the <em>simple form of iterations</em>: in each iteration, the <strong>step function</strong> consumes the <strong>entire input</strong> (the <em>result of the previous iteration</em>, or the <em>initial data set</em>), and computes the <strong>next version of the partial solution</strong> (e.g. <code>map</code>, <code>reduce</code>, <code>join</code>, etc.).</p> - -<p class="text-center"> - <img alt="Iterate Operator" width="60%" src="fig/iterations_iterate_operator.png" /> -</p> - -<ol> - <li><strong>Iteration Input</strong>: Initial input for the <em>first iteration</em> from a <em>data source</em> or <em>previous operators</em>.</li> - <li><strong>Step Function</strong>: The step function will be executed in each iteration. It is an arbitrary data flow consisting of operators like <code>map</code>, <code>reduce</code>, <code>join</code>, etc. and depends on your specific task at hand.</li> - <li><strong>Next Partial Solution</strong>: In each iteration, the output of the step function will be fed back into the <em>next iteration</em>.</li> - <li><strong>Iteration Result</strong>: Output of the <em>last iteration</em> is written to a <em>data sink</em> or used as input to the <em>following operators</em>.</li> -</ol> - -<p>There are multiple options to specify <strong>termination conditions</strong> for an iteration:</p> - -<ul> - <li><strong>Maximum number of iterations</strong>: Without any further conditions, the iteration will be executed this many times.</li> - <li><strong>Custom aggregator convergence</strong>: Iterations allow to specify <em>custom aggregators</em> and <em>convergence criteria</em> like sum aggregate the number of emitted records (aggregator) and terminate if this number is zero (convergence criterion).</li> -</ul> - -<p>You can also think about the iterate operator in pseudo-code:</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">IterationState</span> <span class="n">state</span> <span class="o">=</span> <span class="n">getInitialState</span><span class="o">();</span> - -<span class="k">while</span> <span class="o">(!</span><span class="n">terminationCriterion</span><span class="o">())</span> <span class="o">{</span> - <span class="n">state</span> <span class="o">=</span> <span class="n">step</span><span class="o">(</span><span class="n">state</span><span class="o">);</span> -<span class="o">}</span> - -<span class="n">setFinalState</span><span class="o">(</span><span class="n">state</span><span class="o">);</span></code></pre></div> - -<div class="panel panel-default"> - <div class="panel-body"> - See the <strong><a href="programming_guide.html">Programming Guide</a> </strong> for details and code examples.</div> -</div> - -<h3 id="example-incrementing-numbers">Example: Incrementing Numbers</h3> - -<p>In the following example, we <strong>iteratively incremenet a set numbers</strong>:</p> - -<p class="text-center"> - <img alt="Iterate Operator Example" width="60%" src="fig/iterations_iterate_operator_example.png" /> -</p> - -<ol> - <li><strong>Iteration Input</strong>: The inital input is read from a data source and consists of five single-field records (integers <code>1</code> to <code>5</code>).</li> - <li><strong>Step function</strong>: The step function is a single <code>map</code> operator, which increments the integer field from <code>i</code> to <code>i+1</code>. It will be applied to every record of the input.</li> - <li><strong>Next Partial Solution</strong>: The output of the step function will be the output of the map operator, i.e. records with incremented integers.</li> - <li><strong>Iteration Result</strong>: After ten iterations, the initial numbers will have been incremented ten times, resulting in integers <code>11</code> to <code>15</code>.</li> -</ol> - -<div class="highlight"><pre><code>// 1st 2nd 10th -map(1) -> 2 map(2) -> 3 ... map(10) -> 11 -map(2) -> 3 map(3) -> 4 ... map(11) -> 12 -map(3) -> 4 map(4) -> 5 ... map(12) -> 13 -map(4) -> 5 map(5) -> 6 ... map(13) -> 14 -map(5) -> 6 map(6) -> 7 ... map(14) -> 15 -</code></pre></div> - -<p>Note that <strong>1</strong>, <strong>2</strong>, and <strong>4</strong> can be arbitrary data flows.</p> - -<h2 id="delta-iterate-operator">Delta Iterate Operator</h2> - -<p>The <strong>delta iterate operator</strong> covers the case of <strong>incremental iterations</strong>. Incremental iterations <strong>selectively modify elements</strong> of their <strong>solution</strong> and evolve the solution rather than fully recompute it.</p> - -<p>Where applicable, this leads to <strong>more efficient algorithms</strong>, because not every element in the solution set changes in each iteration. This allows to <strong>focus on the hot parts</strong> of the solution and leave the <strong>cold parts untouched</strong>. Frequently, the majority of the solution cools down comparatively fast and the later iterations operate only on a small subset of the data.</p> - -<p class="text-center"> - <img alt="Delta Iterate Operator" width="60%" src="fig/iterations_delta_iterate_operator.png" /> -</p> - -<ol> - <li><strong>Iteration Input</strong>: The initial workset and solution set are read from <em>data sources</em> or <em>previous operators</em> as input to the first iteration.</li> - <li><strong>Step Function</strong>: The step function will be executed in each iteration. It is an arbitrary data flow consisting of operators like <code>map</code>, <code>reduce</code>, <code>join</code>, etc. and depends on your specific task at hand.</li> - <li><strong>Next Workset/Update Solution Set</strong>: The <em>next workset</em> drives the iterative computation and will be fed back into the <em>next iteration</em>. Furthermore, the solution set will be updated and implicitly forwarded (it is not required to be rebuild). Both data sets can be updated by different operators of the step function.</li> - <li><strong>Iteration Result</strong>: After the <em>last iteration</em>, the <em>solution set</em> is written to a <em>data sink</em> or used as input to the <em>following operators</em>.</li> -</ol> - -<p>The default <strong>termination condition</strong> for delta iterations is specified by the <strong>empty workset convergence criterion</strong> and a <strong>maximum number of iterations</strong>. The iteration will terminate when a produced <em>next workset</em> is empty or when the maximum number of iterations is reached. It is also possible to specify a <strong>custom aggregator</strong> and <strong>convergence criterion</strong>.</p> - -<p>You can also think about the iterate operator in pseudo-code:</p> - -<div class="highlight"><pre><code class="language-java"><span class="n">IterationState</span> <span class="n">workset</span> <span class="o">=</span> <span class="n">getInitialState</span><span class="o">();</span> -<span class="n">IterationState</span> <span class="n">solution</span> <span class="o">=</span> <span class="n">getInitialSolution</span><span class="o">();</span> - -<span class="k">while</span> <span class="o">(!</span><span class="n">terminationCriterion</span><span class="o">())</span> <span class="o">{</span> - <span class="o">(</span><span class="n">delta</span><span class="o">,</span> <span class="n">workset</span><span class="o">)</span> <span class="o">=</span> <span class="n">step</span><span class="o">(</span><span class="n">workset</span><span class="o">,</span> <span class="n">solution</span><span class="o">);</span> - - <span class="n">solution</span><span class="o">.</span><span class="na">update</span><span class="o">(</span><span class="n">delta</span><span class="o">)</span> -<span class="o">}</span> - -<span class="n">setFinalState</span><span class="o">(</span><span class="n">solution</span><span class="o">);</span></code></pre></div> - -<div class="panel panel-default"> - <div class="panel-body"> - See the <strong><a href="programming_guide.html">programming guide</a></strong> for details and code examples.</div> -</div> - -<h3 id="example-propagate-minimum-in-graph">Example: Propagate Minimum in Graph</h3> - -<p>In the following example, every vertex has an <strong>ID</strong> and a <strong>coloring</strong>. Each vertex will propagate its vertex ID to neighboring vertices. The <strong>goal</strong> is to <em>assign the minimum ID to every vertex in a subgraph</em>. If a received ID is smaller then the current one, it changes to the color of the vertex with the received ID. One application of this can be found in <em>community analysis</em> or <em>connected components</em> computation.</p> - -<p class="text-center"> - <img alt="Delta Iterate Operator Example" width="100%" src="fig/iterations_delta_iterate_operator_example.png" /> -</p> - -<p>The <strong>intial input</strong> is set as <strong>both workset and solution set.</strong> In the above figure, the colors visualize the <strong>evolution of the solution set</strong>. With each iteration, the color of the minimum ID is spreading in the respective subgraph. At the same time, the amount of work (exchanged and compared vertex IDs) decreases with each iteration. This corresponds to the <strong>decreasing size of the workset</strong>, which goes from all seven vertices to zero after three iterations, at which time the iteration terminates. The <strong>important observation</strong> is that <em>the lower subgraph converges before the upper half</em> does and the delta iteration is able to capture this with the workset abstraction.</p> - -<p>In the upper subgraph <strong>ID 1</strong> (<em>orange</em>) is the <strong>minimum ID</strong>. In the <strong>first iteration</strong>, it will get propagated to vertex 2, which will subsequently change its color to orange. Vertices 3 and 4 will receive <strong>ID 2</strong> (in <em>yellow</em>) as their current minimum ID and change to yellow. Because the color of <em>vertex 1</em> didnât change in the first iteration, it can be skipped it in the next workset.</p> - -<p>In the lower subgraph <strong>ID 5</strong> (<em>cyan</em>) is the <strong>minimum ID</strong>. All vertices of the lower subgraph will receive it in the first iteration. Again, we can skip the unchanged vertices (<em>vertex 5</em>) for the next workset.</p> - -<p>In the <strong>2nd iteration</strong>, the workset size has already decreased from seven to five elements (vertices 2, 3, 4, 6, and 7). These are part of the iteration and further propagate their current minimum IDs. After this iteration, the lower subgraph has already converged (<strong>cold part</strong> of the graph), as it has no elements in the workset, whereas the upper half needs a further iteration (<strong>hot part</strong> of the graph) for the two remaining workset elements (vertices 3 and 4).</p> - -<p>The iteration <strong>terminates</strong>, when the workset is empty after the <strong>3rd iteration</strong>.</p> - -<p><a href="#supersteps"></a></p> - -<h2 id="superstep-synchronization">Superstep Synchronization</h2> - -<p>We referred to each execution of the step function of an iteration operator as <em>a single iteration</em>. In parallel setups, <strong>multiple instances of the step function are evaluated in parallel</strong> on different partitions of the iteration state. In many settings, one evaluation of the step function on all parallel instances forms a so called <strong>superstep</strong>, which is also the granularity of synchronization. Therefore, <em>all</em> parallel tasks of an iteration need to complete the superstep, before a next superstep will be initialized. <strong>Termination criteria</strong> will also be evaluated at superstep barriers.</p> - -<p class="text-center"> - <img alt="Supersteps" width="50%" src="fig/iterations_supersteps.png" /> -</p> - - </div> - - <div class="col-sm-10 col-sm-offset-1"> - <!-- Disqus thread and some vertical offset --> - <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div> - </div> -</div> - - </div><!-- /.container --> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js";></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - - <!-- Disqus --> - <script type="text/javascript"> - var disqus_shortname = 'stratosphere-eu'; - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); -</script> - </body> -</html>
![http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"](http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"){kind=link}