http://git-wip-us.apache.org/repos/asf/flink-web/blob/d8883b04/content/news/2015/11/16/release-0.10.0.html ---------------------------------------------------------------------- diff --git a/content/news/2015/11/16/release-0.10.0.html b/content/news/2015/11/16/release-0.10.0.html deleted file mode 100644 index 149c2c3..0000000 --- a/content/news/2015/11/16/release-0.10.0.html +++ /dev/null @@ -1,376 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <title>Apache Flink: Announcing Apache Flink 0.10.0</title> - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="/css/flink.css"> - <link rel="stylesheet" href="/css/syntax.css"> - - <!-- Blog RSS feed --> - <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <!-- We need to load Jquery in the header for custom google analytics event tracking--> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="/"> - <img alt="Apache Flink" src="/img/navbar-brand-logo.jpg" width="78px" height="40px"> - </a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <!-- Overview --> - <li><a href="/index.html">Overview</a></li> - - <!-- Features --> - <li><a href="/features.html">Features</a></li> - - <!-- Downloads --> - <li><a href="/downloads.html">Downloads</a></li> - - <!-- FAQ --> - <li><a href="/faq.html">FAQ</a></li> - - - <!-- Quickstart --> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Quickstart <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html";>Setup</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/run_example_quickstart.html";>Example: Wikipedia Edit Stream</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/java_api_quickstart.html";>Java API</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/scala_api_quickstart.html";>Scala API</a></li> - </ul> - </li> - - <!-- Documentation --> - <li class="dropdown"> - <a href="" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Documentation <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Latest stable release --> - <li role="presentation" class="dropdown-header"><strong>Latest Release</strong> (Stable)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1";>1.1 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/java"; class="active">1.1 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/scala/index.html"; class="active">1.1 ScalaDocs</a></li> --> - - <!-- Snapshot docs --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Snapshot</strong> (Development)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2";>1.2 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/java"; class="active">1.2 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/scala/index.html"; class="active">1.2 ScalaDocs</a></li> --> - - <!-- Wiki --> - <li class="divider"></li> - <li><a href="/visualizer/"><small><span class="glyphicon glyphicon-new-window"></span></small> Plan Visualizer</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - - </ul> - - <ul class="nav navbar-nav navbar-right"> - <!-- Blog --> - <li class=" active hidden-md hidden-sm"><a href="/blog/">Blog</a></li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Community <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Community --> - <li role="presentation" class="dropdown-header"><strong>Community</strong></li> - <li><a href="/community.html#mailing-lists">Mailing Lists</a></li> - <li><a href="/community.html#irc">IRC</a></li> - <li><a href="/community.html#stack-overflow">Stack Overflow</a></li> - <li><a href="/community.html#issue-tracker">Issue Tracker</a></li> - <li><a href="/community.html#third-party-packages">Third Party Packages</a></li> - <li><a href="/community.html#source-code">Source Code</a></li> - <li><a href="/community.html#people">People</a></li> - <li><a href="/poweredby.html">Powered by Flink</a></li> - - <!-- Contribute --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="/how-to-contribute.html">How to Contribute</a></li> - <li><a href="/contribute-code.html">Contribute Code</a></li> - <li><a href="/contribute-documentation.html">Contribute Documentation</a></li> - <li><a href="/improve-website.html">Improve the Website</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Improvement+Proposals";><small><span class="glyphicon glyphicon-new-window"></span></small> Flink Improvement Proposals (Design Docs)</a></li> - </ul> - </li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Project <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Project --> - <li role="presentation" class="dropdown-header"><strong>Project</strong></li> - <li><a href="/slides.html">Slides</a></li> - <li><a href="/material.html">Material</a></li> - <li><a href="https://twitter.com/apacheflink";><small><span class="glyphicon glyphicon-new-window"></span></small> Twitter</a></li> - <li><a href="https://github.com/apache/flink";><small><span class="glyphicon glyphicon-new-window"></span></small> GitHub</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - </ul> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-8 col-sm-offset-2"> - <div class="row"> - <h1>Announcing Apache Flink 0.10.0</h1> - - <article> - <p>16 Nov 2015</p> - -<p>The Apache Flink community is pleased to announce the availability of the 0.10.0 release. The community put significant effort into improving and extending Apache Flink since the last release, focusing on data stream processing and operational features. About 80 contributors provided bug fixes, improvements, and new features such that in total more than 400 JIRA issues could be resolved.</p> - -<p>For Flink 0.10.0, the focus of the community was to graduate the DataStream API from beta and to evolve Apache Flink into a production-ready stream data processor with a competitive feature set. These efforts resulted in support for event-time and out-of-order streams, exactly-once guarantees in the case of failures, a very flexible windowing mechanism, sophisticated operator state management, and a highly-available cluster operation mode. Flink 0.10.0 also brings a new monitoring dashboard with real-time system and job monitoring capabilities. Both batch and streaming modes of Flink benefit from the new high availability and improved monitoring features. Needless to say that Flink 0.10.0 includes many more features, improvements, and bug fixes.</p> - -<p>We encourage everyone to <a href="/downloads.html">download the release</a> and <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/";>check out the documentation</a>. Feedback through the Flink <a href="/community.html#mailing-lists">mailing lists</a> is, as always, very welcome!</p> - -<h2 id="new-features">New Features</h2> - -<h3 id="event-time-stream-processing">Event-time Stream Processing</h3> - -<p>Many stream processing applications consume data from sources that produce events with associated timestamps such as sensor or user-interaction events. Very often, events have to be collected from several sources such that it is usually not guaranteed that events arrive in the exact order of their timestamps at the stream processor. Consequently, stream processors must take out-of-order elements into account in order to produce results which are correct and consistent with respect to the timestamps of the events. With release 0.10.0, Apache Flink supports event-time processing as well as ingestion-time and processing-time processing. See <a href="https://issues.apache.org/jira/browse/FLINK-2674";>FLINK-2674</a> for details.</p> - -<h3 id="stateful-stream-processing">Stateful Stream Processing</h3> - -<p>Operators that maintain and update state are a common pattern in many stream processing applications. Since streaming applications tend to run for a very long time, operator state can become very valuable and impossible to recompute. In order to enable fault-tolerance, operator state must be backed up to persistent storage in regular intervals. Flink 0.10.0 offers flexible interfaces to define, update, and query operator state and hooks to connect various state backends.</p> - -<h3 id="highly-available-cluster-operations">Highly-available Cluster Operations</h3> - -<p>Stream processing applications may be live for months. Therefore, a production-ready stream processor must be highly-available and continue to process data even in the face of failures. With release 0.10.0, Flink supports high availability modes for standalone cluster and <a href="https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html";>YARN</a> setups, eliminating any single point of failure. In this mode, Flink relies on <a href="https://zookeeper.apache.org";>Apache Zookeeper</a> for leader election and persisting small sized meta-data of running jobs. You can <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/setup/jobmanager_high_availability.html";>check out the documentation</a> to see how to enable high availability. See <a href="https://issues.apache.org/jira/browse/FLINK-2287";>FLINK-2287</a> for details.</p> - -<h3 id="graduated-datastream-api">Graduated DataStream API</h3> - -<p>The DataStream API was revised based on user feedback and with foresight for upcoming features and graduated from beta status to fully supported. The most obvious changes are related to the methods for stream partitioning and window operations. The new windowing system is based on the concepts of window assigners, triggers, and evictors, inspired by the <a href="http://www.vldb.org/pvldb/vol8/p1792-Akidau.pdf";>Dataflow Model</a>. The new API is fully described in the <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/apis/streaming_guide.html";>DataStream API documentation</a>. This <a href="https://cwiki.apache.org/confluence/display/FLINK/Migration+Guide%3A+0.9.x+to+0.10.x";>migration guide</a> will help to port your Flink 0.9 DataStream programs to the revised API of Flink 0.10.0. See <a href="https://issues.apache.org/jira/browse/FLINK-2674";>FLINK-2674</a> and <a href="https://issues.apache.org/jira/browse/FLINK-2877";>FLINK-2877</a> for details.</p> - -<h3 id="new-connectors-for-data-streams">New Connectors for Data Streams</h3> - -<p>Apache Flink 0.10.0 features DataStream sources and sinks for many common data producers and stores. This includes an exactly-once rolling file sink which supports any file system, including HDFS, local FS, and S3. We also updated the <a href="https://kafka.apache.org";>Apache Kafka</a> producer to use the new producer API, and added a connectors for <a href="https://github.com/elastic/elasticsearch";>ElasticSearch</a> and <a href="https://nifi.apache.org";>Apache Nifi</a>. More connectors for DataStream programs will be added by the community in the future. See the following JIRA issues for details <a href="https://issues.apache.org/jira/browse/FLINK-2583";>FLINK-2583</a>, <a href="https://issues.apache.org/jira/browse/FLINK-2386";>FLINK-2386</a>, <a href="https://issues.apache.org/jira/browse/FLINK-2372";>FLINK-2372</a>, <a href="https://issues.apache.org/jira/browse/FLINK-2740";>FLINK-2740</a>, and <a href="https://issues.apache.org/jira/browse/FLINK-2558";>FLINK-2558</a>.</p> - -<h3 id="new-web-dashboard--real-time-monitoring">New Web Dashboard & Real-time Monitoring</h3> - -<p>The 0.10.0 release features a newly designed and significantly improved monitoring dashboard for Apache Flink. The new dashboard visualizes the progress of running jobs and shows real-time statistics of processed data volumes and record counts. Moreover, it gives access to resource usage and JVM statistics of TaskManagers including JVM heap usage and garbage collection details. The following screenshot shows the job view of the new dashboard.</p> - -<center> -<img src="/img/blog/new-dashboard-screenshot.png" style="width:90%;margin:15px" /> -</center> - -<p>The web server that provides all monitoring statistics has been designed with a REST interface allowing other systems to also access the internal system metrics. See <a href="https://issues.apache.org/jira/browse/FLINK-2357";>FLINK-2357</a> for details.</p> - -<h3 id="off-heap-managed-memory">Off-heap Managed Memory</h3> - -<p>Flinkâs internal operators (such as its sort algorithm and hash tables) write data to and read data from managed memory to achieve memory-safe operations and reduce garbage collection overhead. Until version 0.10.0, managed memory was allocated only from JVM heap memory. With this release, managed memory can also be allocated from off-heap memory. This will facilitate shorter TaskManager start-up times as well as reduce garbage collection pressure. See <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/setup/config.html#managed-memory";>the documentation</a> to learn how to configure managed memory on off-heap memory. JIRA issue <a href="https://issues.apache.org/jira/browse/FLINK-1320";>FLINK-1320</a> contains further details.</p> - -<h3 id="outer-joins">Outer Joins</h3> - -<p>Outer joins have been one of the most frequently requested features for Flinkâs <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/apis/programming_guide.html";>DataSet API</a>. Although there was a workaround to implement outer joins as CoGroup function, it had significant drawbacks including added code complexity and not being fully memory-safe. With release 0.10.0, Flink adds native support for <a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/apis/dataset_transformations.html#outerjoin";>left, right, and full outer joins</a> to the DataSet API. All outer joins are backed by a memory-safe operator implementation that leverages Flinkâs managed memory. See <a href="https://issues.apache.org/jira/browse/FLINK-687";>FLINK-687</a> and <a href="https://issues.apache.org/jira/browse/FLINK-2107";>FLINK-2107</a> for details.</p> - -<h3 id="gelly-major-improvements-and-scala-api">Gelly: Major Improvements and Scala API</h3> - -<p><a href="https://ci.apache.org/projects/flink/flink-docs-release-0.10/libs/gelly_guide.html";>Gelly</a> is Flinkâs API and library for processing and analyzing large-scale graphs. Gelly was introduced with release 0.9.0 and has been very well received by users and contributors. Based on user feedback, Gelly has been improved since then. In addition, Flink 0.10.0 introduces a Scala API for Gelly. See <a href="https://issues.apache.org/jira/browse/FLINK-2857";>FLINK-2857</a> and <a href="https://issues.apache.org/jira/browse/FLINK-1962";>FLINK-1962</a> for details.</p> - -<h2 id="more-improvements-and-fixes">More Improvements and Fixes</h2> - -<p>The Flink community resolved more than 400 issues. The following list is a selection of new features and fixed bugs.</p> - -<ul> - <li><a href="https://issues.apache.org/jira/browse/FLINK-1851";>FLINK-1851</a> Java Table API does not support Casting</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2152";>FLINK-2152</a> Provide zipWithIndex utility in flink-contrib</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2158";>FLINK-2158</a> NullPointerException in DateSerializer.</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2240";>FLINK-2240</a> Use BloomFilter to minimize probe side records which are spilled to disk in Hybrid-Hash-Join</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2533";>FLINK-2533</a> Gap based random sample optimization</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2555";>FLINK-2555</a> Hadoop Input/Output Formats are unable to access secured HDFS clusters</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2565";>FLINK-2565</a> Support primitive arrays as keys</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2582";>FLINK-2582</a> Document how to build Flink with other Scala versions</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2584";>FLINK-2584</a> ASM dependency is not shaded away</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2689";>FLINK-2689</a> Reusing null object for joins with SolutionSet</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2703";>FLINK-2703</a> Remove log4j classes from fat jar / document how to use Flink with logback</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2763";>FLINK-2763</a> Bug in Hybrid Hash Join: Request to spill a partition with less than two buffers.</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2767";>FLINK-2767</a> Add support Scala 2.11 to Scala shell</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2774";>FLINK-2774</a> Import Java API classes automatically in Flinkâs Scala shell</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2782";>FLINK-2782</a> Remove deprecated features for 0.10</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2800";>FLINK-2800</a> kryo serialization problem</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2834";>FLINK-2834</a> Global round-robin for temporary directories</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2842";>FLINK-2842</a> S3FileSystem is broken</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2874";>FLINK-2874</a> Certain Avro generated getters/setters not recognized</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2895";>FLINK-2895</a> Duplicate immutable object creation</li> - <li><a href="https://issues.apache.org/jira/browse/FLINK-2964";>FLINK-2964</a> MutableHashTable fails when spilling partitions without overflow segments</li> -</ul> - -<h2 id="notice">Notice</h2> - -<p>As previously announced, Flink 0.10.0 no longer supports Java 6. If you are still using Java 6, please consider upgrading to Java 8 (Java 7 ended its free support in April 2015). -Also note that some methods in the DataStream API had to be renamed as part of the API rework. For example the <code>groupBy</code> method has been renamed to <code>keyBy</code> and the windowing API changed. This <a href="https://cwiki.apache.org/confluence/display/FLINK/Migration+Guide%3A+0.9.x+to+0.10.x";>migration guide</a> will help to port your Flink 0.9 DataStream programs to the revised API of Flink 0.10.0.</p> - -<h2 id="contributors">Contributors</h2> - -<ul> - <li>Alexander Alexandrov</li> - <li>Marton Balassi</li> - <li>Enrique Bautista</li> - <li>Faye Beligianni</li> - <li>Bryan Bende</li> - <li>Ajay Bhat</li> - <li>Chris Brinkman</li> - <li>Dmitry Buzdin</li> - <li>Kun Cao</li> - <li>Paris Carbone</li> - <li>Ufuk Celebi</li> - <li>Shivani Chandna</li> - <li>Liang Chen</li> - <li>Felix Cheung</li> - <li>Hubert Czerpak</li> - <li>Vimal Das</li> - <li>Behrouz Derakhshan</li> - <li>Suminda Dharmasena</li> - <li>Stephan Ewen</li> - <li>Fengbin Fang</li> - <li>Gyula Fora</li> - <li>Lun Gao</li> - <li>Gabor Gevay</li> - <li>Piotr Godek</li> - <li>Sachin Goel</li> - <li>Anton Haglund</li> - <li>Gábor Hermann</li> - <li>Greg Hogan</li> - <li>Fabian Hueske</li> - <li>Martin Junghanns</li> - <li>Vasia Kalavri</li> - <li>Ulf Karlsson</li> - <li>Frederick F. Kautz</li> - <li>Samia Khalid</li> - <li>Johannes Kirschnick</li> - <li>Kostas Kloudas</li> - <li>Alexander Kolb</li> - <li>Johann Kovacs</li> - <li>Aljoscha Krettek</li> - <li>Sebastian Kruse</li> - <li>Andreas Kunft</li> - <li>Chengxiang Li</li> - <li>Chen Liang</li> - <li>Andra Lungu</li> - <li>Suneel Marthi</li> - <li>Tamara Mendt</li> - <li>Robert Metzger</li> - <li>Maximilian Michels</li> - <li>Chiwan Park</li> - <li>Sahitya Pavurala</li> - <li>Pietro Pinoli</li> - <li>Ricky Pogalz</li> - <li>Niraj Rai</li> - <li>Lokesh Rajaram</li> - <li>Johannes Reifferscheid</li> - <li>Till Rohrmann</li> - <li>Henry Saputra</li> - <li>Matthias Sax</li> - <li>Shiti Saxena</li> - <li>Chesnay Schepler</li> - <li>Peter Schrott</li> - <li>Saumitra Shahapure</li> - <li>Nikolaas Steenbergen</li> - <li>Thomas Sun</li> - <li>Peter Szabo</li> - <li>Viktor Taranenko</li> - <li>Kostas Tzoumas</li> - <li>Pieter-Jan Van Aeken</li> - <li>Theodore Vasiloudis</li> - <li>Timo Walther</li> - <li>Chengxuan Wang</li> - <li>Huang Wei</li> - <li>Dawid Wysakowicz</li> - <li>Rerngvit Yanggratoke</li> - <li>Nezih Yigitbasi</li> - <li>Ted Yu</li> - <li>Rucong Zhang</li> - <li>Vyacheslav Zholudev</li> - <li>Zoltán Zvara</li> -</ul> - - - </article> - </div> - - <div class="row"> - <div id="disqus_thread"></div> - <script type="text/javascript"> - /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ - var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname - - /* * * DON'T EDIT BELOW THIS LINE * * */ - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); - </script> - </div> - </div> -</div> - - <hr /> - <div class="footer text-center"> - <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> - <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> - <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> - </div> - - </div><!-- /.container --> - - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="/js/codetabs.js"></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - </body> -</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/d8883b04/content/news/2015/11/27/release-0.10.1.html ---------------------------------------------------------------------- diff --git a/content/news/2015/11/27/release-0.10.1.html b/content/news/2015/11/27/release-0.10.1.html deleted file mode 100644 index 3004786..0000000 --- a/content/news/2015/11/27/release-0.10.1.html +++ /dev/null @@ -1,260 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <title>Apache Flink: Flink 0.10.1 released</title> - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="/css/flink.css"> - <link rel="stylesheet" href="/css/syntax.css"> - - <!-- Blog RSS feed --> - <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <!-- We need to load Jquery in the header for custom google analytics event tracking--> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="/"> - <img alt="Apache Flink" src="/img/navbar-brand-logo.jpg" width="78px" height="40px"> - </a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <!-- Overview --> - <li><a href="/index.html">Overview</a></li> - - <!-- Features --> - <li><a href="/features.html">Features</a></li> - - <!-- Downloads --> - <li><a href="/downloads.html">Downloads</a></li> - - <!-- FAQ --> - <li><a href="/faq.html">FAQ</a></li> - - - <!-- Quickstart --> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Quickstart <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html";>Setup</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/run_example_quickstart.html";>Example: Wikipedia Edit Stream</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/java_api_quickstart.html";>Java API</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/scala_api_quickstart.html";>Scala API</a></li> - </ul> - </li> - - <!-- Documentation --> - <li class="dropdown"> - <a href="" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Documentation <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Latest stable release --> - <li role="presentation" class="dropdown-header"><strong>Latest Release</strong> (Stable)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1";>1.1 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/java"; class="active">1.1 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/scala/index.html"; class="active">1.1 ScalaDocs</a></li> --> - - <!-- Snapshot docs --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Snapshot</strong> (Development)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2";>1.2 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/java"; class="active">1.2 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/scala/index.html"; class="active">1.2 ScalaDocs</a></li> --> - - <!-- Wiki --> - <li class="divider"></li> - <li><a href="/visualizer/"><small><span class="glyphicon glyphicon-new-window"></span></small> Plan Visualizer</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - - </ul> - - <ul class="nav navbar-nav navbar-right"> - <!-- Blog --> - <li class=" active hidden-md hidden-sm"><a href="/blog/">Blog</a></li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Community <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Community --> - <li role="presentation" class="dropdown-header"><strong>Community</strong></li> - <li><a href="/community.html#mailing-lists">Mailing Lists</a></li> - <li><a href="/community.html#irc">IRC</a></li> - <li><a href="/community.html#stack-overflow">Stack Overflow</a></li> - <li><a href="/community.html#issue-tracker">Issue Tracker</a></li> - <li><a href="/community.html#third-party-packages">Third Party Packages</a></li> - <li><a href="/community.html#source-code">Source Code</a></li> - <li><a href="/community.html#people">People</a></li> - <li><a href="/poweredby.html">Powered by Flink</a></li> - - <!-- Contribute --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="/how-to-contribute.html">How to Contribute</a></li> - <li><a href="/contribute-code.html">Contribute Code</a></li> - <li><a href="/contribute-documentation.html">Contribute Documentation</a></li> - <li><a href="/improve-website.html">Improve the Website</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Improvement+Proposals";><small><span class="glyphicon glyphicon-new-window"></span></small> Flink Improvement Proposals (Design Docs)</a></li> - </ul> - </li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Project <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Project --> - <li role="presentation" class="dropdown-header"><strong>Project</strong></li> - <li><a href="/slides.html">Slides</a></li> - <li><a href="/material.html">Material</a></li> - <li><a href="https://twitter.com/apacheflink";><small><span class="glyphicon glyphicon-new-window"></span></small> Twitter</a></li> - <li><a href="https://github.com/apache/flink";><small><span class="glyphicon glyphicon-new-window"></span></small> GitHub</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - </ul> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-8 col-sm-offset-2"> - <div class="row"> - <h1>Flink 0.10.1 released</h1> - - <article> - <p>27 Nov 2015</p> - -<p>Today, the Flink community released the first bugfix release of the 0.10 series of Flink.</p> - -<p>We recommend all users updating to this release, by bumping the version of your Flink dependencies and updating the binaries on the server.</p> - -<h2 id="issues-fixed">Issues fixed</h2> - -<ul class="list-unstyled"> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2879";>FLINK-2879</a>] - Links in documentation are broken -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2938";>FLINK-2938</a>] - Streaming docs not in sync with latest state changes -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2942";>FLINK-2942</a>] - Dangling operators in web UI's program visualization (non-deterministic) -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2967";>FLINK-2967</a>] - TM address detection might not always detect the right interface on slow networks / overloaded JMs -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2977";>FLINK-2977</a>] - Cannot access HBase in a Kerberos secured Yarn cluster -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2987";>FLINK-2987</a>] - Flink 0.10 fails to start on YARN 2.6.0 -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2989";>FLINK-2989</a>] - Job Cancel button doesn't work on Yarn -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3005";>FLINK-3005</a>] - Commons-collections object deserialization remote command execution vulnerability -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3011";>FLINK-3011</a>] - Cannot cancel failing/restarting streaming job from the command line -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3019";>FLINK-3019</a>] - CLI does not list running/restarting jobs -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3020";>FLINK-3020</a>] - Local streaming execution: set number of task manager slots to the maximum parallelism -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3024";>FLINK-3024</a>] - TimestampExtractor Does not Work When returning Long.MIN_VALUE -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3032";>FLINK-3032</a>] - Flink does not start on Hadoop 2.7.1 (HDP), due to class conflict -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3043";>FLINK-3043</a>] - Kafka Connector description in Streaming API guide is wrong/outdated -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3047";>FLINK-3047</a>] - Local batch execution: set number of task manager slots to the maximum parallelism -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3052";>FLINK-3052</a>] - Optimizer does not push properties out of bulk iterations -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2966";>FLINK-2966</a>] - Improve the way job duration is reported on web frontend. -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-2974";>FLINK-2974</a>] - Add periodic offset commit to Kafka Consumer if checkpointing is disabled -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3028";>FLINK-3028</a>] - Cannot cancel restarting job via web frontend -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3040";>FLINK-3040</a>] - Add docs describing how to configure State Backends -</li> -<li>[<a href="https://issues.apache.org/jira/browse/FLINK-3041";>FLINK-3041</a>] - Twitter Streaming Description section of Streaming Programming guide refers to an incorrect example 'TwitterLocal' -</li> -</ul> - - - </article> - </div> - - <div class="row"> - <div id="disqus_thread"></div> - <script type="text/javascript"> - /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ - var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname - - /* * * DON'T EDIT BELOW THIS LINE * * */ - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); - </script> - </div> - </div> -</div> - - <hr /> - <div class="footer text-center"> - <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> - <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> - <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> - </div> - - </div><!-- /.container --> - - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="/js/codetabs.js"></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - </body> -</html> http://git-wip-us.apache.org/repos/asf/flink-web/blob/d8883b04/content/news/2015/12/04/Introducing-windows.html ---------------------------------------------------------------------- diff --git a/content/news/2015/12/04/Introducing-windows.html b/content/news/2015/12/04/Introducing-windows.html deleted file mode 100644 index 1ab0bca..0000000 --- a/content/news/2015/12/04/Introducing-windows.html +++ /dev/null @@ -1,358 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags --> - <title>Apache Flink: Introducing Stream Windows in Apache Flink</title> - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <!-- Bootstrap --> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css";> - <link rel="stylesheet" href="/css/flink.css"> - <link rel="stylesheet" href="/css/syntax.css"> - - <!-- Blog RSS feed --> - <link href="/blog/feed.xml" rel="alternate" type="application/rss+xml" title="Apache Flink Blog: RSS feed" /> - - <!-- jQuery (necessary for Bootstrap's JavaScript plugins) --> - <!-- We need to load Jquery in the header for custom google analytics event tracking--> - <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js";></script> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!-- WARNING: Respond.js doesn't work if you view the page via file:// --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - <body> - - - <!-- Top navbar. --> - <nav class="navbar navbar-default navbar-fixed-top"> - <div class="container"> - <!-- The logo. --> - <div class="navbar-header"> - <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - <div class="navbar-logo"> - <a href="/"> - <img alt="Apache Flink" src="/img/navbar-brand-logo.jpg" width="78px" height="40px"> - </a> - </div> - </div><!-- /.navbar-header --> - - <!-- The navigation links. --> - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - - <!-- Overview --> - <li><a href="/index.html">Overview</a></li> - - <!-- Features --> - <li><a href="/features.html">Features</a></li> - - <!-- Downloads --> - <li><a href="/downloads.html">Downloads</a></li> - - <!-- FAQ --> - <li><a href="/faq.html">FAQ</a></li> - - - <!-- Quickstart --> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Quickstart <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/setup_quickstart.html";>Setup</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/run_example_quickstart.html";>Example: Wikipedia Edit Stream</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/java_api_quickstart.html";>Java API</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/quickstart/scala_api_quickstart.html";>Scala API</a></li> - </ul> - </li> - - <!-- Documentation --> - <li class="dropdown"> - <a href="" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false"><small><span class="glyphicon glyphicon-new-window"></span></small> Documentation <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Latest stable release --> - <li role="presentation" class="dropdown-header"><strong>Latest Release</strong> (Stable)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1";>1.1 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/java"; class="active">1.1 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.1/api/scala/index.html"; class="active">1.1 ScalaDocs</a></li> --> - - <!-- Snapshot docs --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Snapshot</strong> (Development)</li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2";>1.2 Documentation</a></li> - <li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/java"; class="active">1.2 Javadocs</a></li> - <!--<li><a href="http://ci.apache.org/projects/flink/flink-docs-release-1.2/api/scala/index.html"; class="active">1.2 ScalaDocs</a></li> --> - - <!-- Wiki --> - <li class="divider"></li> - <li><a href="/visualizer/"><small><span class="glyphicon glyphicon-new-window"></span></small> Plan Visualizer</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - - </ul> - - <ul class="nav navbar-nav navbar-right"> - <!-- Blog --> - <li class=" active hidden-md hidden-sm"><a href="/blog/">Blog</a></li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Community <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Community --> - <li role="presentation" class="dropdown-header"><strong>Community</strong></li> - <li><a href="/community.html#mailing-lists">Mailing Lists</a></li> - <li><a href="/community.html#irc">IRC</a></li> - <li><a href="/community.html#stack-overflow">Stack Overflow</a></li> - <li><a href="/community.html#issue-tracker">Issue Tracker</a></li> - <li><a href="/community.html#third-party-packages">Third Party Packages</a></li> - <li><a href="/community.html#source-code">Source Code</a></li> - <li><a href="/community.html#people">People</a></li> - <li><a href="/poweredby.html">Powered by Flink</a></li> - - <!-- Contribute --> - <li class="divider"></li> - <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li> - <li><a href="/how-to-contribute.html">How to Contribute</a></li> - <li><a href="/contribute-code.html">Contribute Code</a></li> - <li><a href="/contribute-documentation.html">Contribute Documentation</a></li> - <li><a href="/improve-website.html">Improve the Website</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Improvement+Proposals";><small><span class="glyphicon glyphicon-new-window"></span></small> Flink Improvement Proposals (Design Docs)</a></li> - </ul> - </li> - - <li class="dropdown hidden-md hidden-sm"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Project <span class="caret"></span></a> - <ul class="dropdown-menu" role="menu"> - <!-- Project --> - <li role="presentation" class="dropdown-header"><strong>Project</strong></li> - <li><a href="/slides.html">Slides</a></li> - <li><a href="/material.html">Material</a></li> - <li><a href="https://twitter.com/apacheflink";><small><span class="glyphicon glyphicon-new-window"></span></small> Twitter</a></li> - <li><a href="https://github.com/apache/flink";><small><span class="glyphicon glyphicon-new-window"></span></small> GitHub</a></li> - <li><a href="https://cwiki.apache.org/confluence/display/FLINK/Apache+Flink+Home";><small><span class="glyphicon glyphicon-new-window"></span></small> Wiki</a></li> - </ul> - </li> - </ul> - </div><!-- /.navbar-collapse --> - </div><!-- /.container --> - </nav> - - - <!-- Main content. --> - <div class="container"> - - -<div class="row"> - <div class="col-sm-8 col-sm-offset-2"> - <div class="row"> - <h1>Introducing Stream Windows in Apache Flink</h1> - - <article> - <p>04 Dec 2015 by Fabian Hueske (<a href="https://twitter.com/fhueske";>@fhueske</a>)</p> - -<p>The data analysis space is witnessing an evolution from batch to stream processing for many use cases. Although batch can be handled as a special case of stream processing, analyzing never-ending streaming data often requires a shift in the mindset and comes with its own terminology (for example, âwindowingâ and âat-least-onceâ/âexactly-onceâ processing). This shift and the new terminology can be quite confusing for people being new to the space of stream processing. Apache Flink is a production-ready stream processor with an easy-to-use yet very expressive API to define advanced stream analysis programs. Flinkâs API features very flexible window definitions on data streams which let it stand out among other open source stream processors.</p> - -<p>In this blog post, we discuss the concept of windows for stream processing, present Flinkâs built-in windows, and explain its support for custom windowing semantics.</p> - -<h2 id="what-are-windows-and-what-are-they-good-for">What are windows and what are they good for?</h2> - -<p>Consider the example of a traffic sensor that counts every 15 seconds the number of vehicles passing a certain location. The resulting stream could look like:</p> - -<center> -<img src="/img/blog/window-intro/window-stream.png" style="width:75%;margin:15px" /> -</center> - -<p>If you would like to know, how many vehicles passed that location, you would simply sum the individual counts. However, the nature of a sensor stream is that it continuously produces data. Such a stream never ends and it is not possible to compute a final sum that can be returned. Instead, it is possible to compute rolling sums, i.e., return for each input event an updated sum record. This would yield a new stream of partial sums.</p> - -<center> -<img src="/img/blog/window-intro/window-rolling-sum.png" style="width:75%;margin:15px" /> -</center> - -<p>However, a stream of partial sums might not be what we are looking for, because it constantly updates the count and even more important, some information such as variation over time is lost. Hence, we might want to rephrase our question and ask for the number of cars that pass the location every minute. This requires us to group the elements of the stream into finite sets, each set corresponding to sixty seconds. This operation is called a <em>tumbling windows</em> operation.</p> - -<center> -<img src="/img/blog/window-intro/window-tumbling-window.png" style="width:75%;margin:15px" /> -</center> - -<p>Tumbling windows discretize a stream into non-overlapping windows. For certain applications it is important that windows are not disjunct because an application might require smoothed aggregates. For example, we can compute every thirty seconds the number of cars passed in the last minute. Such windows are called <em>sliding windows</em>.</p> - -<center> -<img src="/img/blog/window-intro/window-sliding-window.png" style="width:75%;margin:15px" /> -</center> - -<p>Defining windows on a data stream as discussed before is a non-parallel operation. This is because each element of a stream must be processed by the same window operator that decides which windows the element should be added to. Windows on a full stream are called <em>AllWindows</em> in Flink. For many applications, a data stream needs to be grouped into multiple logical streams on each of which a window operator can be applied. Think for example about a stream of vehicle counts from multiple traffic sensors (instead of only one sensor as in our previous example), where each sensor monitors a different location. By grouping the stream by sensor id, we can compute windowed traffic statistics for each location in parallel. In Flink, we call such partitioned windows simply <em>Windows</em>, as they are the common case for distributed streams. The following figure shows tumbling windows that collect two elements over a stream of <code>(sensorId, count)</code> pair elements.</p> - -<center> -<img src="/img/blog/window-intro/windows-keyed.png" style="width:75%;margin:15px" /> -</center> - -<p>Generally speaking, a window defines a finite set of elements on an unbounded stream. This set can be based on time (as in our previous examples), element counts, a combination of counts and time, or some custom logic to assign elements to windows. Flinkâs DataStream API provides concise operators for the most common window operations as well as a generic windowing mechanism that allows users to define very custom windowing logic. In the following we present Flinkâs time and count windows before discussing its windowing mechanism in detail.</p> - -<h2 id="time-windows">Time Windows</h2> - -<p>As their name suggests, time windows group stream elements by time. For example, a tumbling time window of one minute collects elements for one minute and applies a function on all elements in the window after one minute passed.</p> - -<p>Defining tumbling and sliding time windows in Apache Flink is very easy:</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// Stream of (sensorId, carCnt)</span> -<span class="k">val</span> <span class="n">vehicleCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="o">...</span> - -<span class="k">val</span> <span class="n">tumblingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> - <span class="c1">// key stream by sensorId</span> - <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// tumbling time window of 1 minute length</span> - <span class="o">.</span><span class="n">timeWindow</span><span class="o">(</span><span class="nc">Time</span><span class="o">.</span><span class="n">minutes</span><span class="o">(</span><span class="mi">1</span><span class="o">))</span> - <span class="c1">// compute sum over carCnt</span> - <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> - -<span class="k">val</span> <span class="n">slidingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> - <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// sliding time window of 1 minute length and 30 secs trigger interval</span> - <span class="o">.</span><span class="n">timeWindow</span><span class="o">(</span><span class="nc">Time</span><span class="o">.</span><span class="n">minutes</span><span class="o">(</span><span class="mi">1</span><span class="o">),</span> <span class="nc">Time</span><span class="o">.</span><span class="n">seconds</span><span class="o">(</span><span class="mi">30</span><span class="o">))</span> - <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span></code></pre></div> - -<p>There is one aspect that we havenât discussed yet, namely the exact meaning of â<em>collects elements for one minute</em>â which boils down to the question, â<em>How does the stream processor interpret time?</em>â.</p> - -<p>Apache Flink features three different notions of time, namely <em>processing time</em>, <em>event time</em>, and <em>ingestion time</em>.</p> - -<ol> - <li>In <strong>processing time</strong>, windows are defined with respect to the wall clock of the machine that builds and processes a window, i.e., a one minute processing time window collects elements for exactly one minute.</li> - <li>In <strong>event time</strong>, windows are defined with respect to timestamps that are attached to each event record. This is common for many types of events, such as log entries, sensor data, etc, where the timestamp usually represents the time at which the event occurred. Event time has several benefits over processing time. First of all, it decouples the program semantics from the actual serving speed of the source and the processing performance of system. Hence you can process historic data, which is served at maximum speed, and continuously produced data with the same program. It also prevents semantically incorrect results in case of backpressure or delays due to failure recovery. Second, event time windows compute correct results, even if events arrive out-of-order of their timestamp which is common if a data stream gathers events from distributed sources.</li> - <li><strong>Ingestion time</strong> is a hybrid of processing and event time. It assigns wall clock timestamps to records as soon as they arrive in the system (at the source) and continues processing with event time semantics based on the attached timestamps.</li> -</ol> - -<h2 id="count-windows">Count Windows</h2> - -<p>Apache Flink also features count windows. A tumbling count window of 100 will collect 100 events in a window and evaluate the window when the 100th element has been added.</p> - -<p>In Flinkâs DataStream API, tumbling and sliding count windows are defined as follows:</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// Stream of (sensorId, carCnt)</span> -<span class="k">val</span> <span class="n">vehicleCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="o">...</span> - -<span class="k">val</span> <span class="n">tumblingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> - <span class="c1">// key stream by sensorId</span> - <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// tumbling count window of 100 elements size</span> - <span class="o">.</span><span class="n">countWindow</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span> - <span class="c1">// compute the carCnt sum </span> - <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span> - -<span class="k">val</span> <span class="n">slidingCnts</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[(</span><span class="kt">Int</span>, <span class="kt">Int</span><span class="o">)]</span> <span class="k">=</span> <span class="n">vehicleCnts</span> - <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="mi">0</span><span class="o">)</span> - <span class="c1">// sliding count window of 100 elements size and 10 elements trigger interval</span> - <span class="o">.</span><span class="n">countWindow</span><span class="o">(</span><span class="mi">100</span><span class="o">,</span> <span class="mi">10</span><span class="o">)</span> - <span class="o">.</span><span class="n">sum</span><span class="o">(</span><span class="mi">1</span><span class="o">)</span></code></pre></div> - -<h2 id="dissecting-flinks-windowing-mechanics">Dissecting Flinkâs windowing mechanics</h2> - -<p>Flinkâs built-in time and count windows cover a wide range of common window use cases. However, there are of course applications that require custom windowing logic that cannot be addressed by Flinkâs built-in windows. In order to support also applications that need very specific windowing semantics, the DataStream API exposes interfaces for the internals of its windowing mechanics. These interfaces give very fine-grained control about the way that windows are built and evaluated.</p> - -<p>The following figure depicts Flinkâs windowing mechanism and introduces the components being involved.</p> - -<center> -<img src="/img/blog/window-intro/window-mechanics.png" style="width:90%;margin:15px" /> -</center> - -<p>Elements that arrive at a window operator are handed to a <code>WindowAssigner</code>. The WindowAssigner assigns elements to one or more windows, possibly creating new windows. A <code>Window</code> itself is just an identifier for a list of elements and may provide some optional meta information, such as begin and end time in case of a <code>TimeWindow</code>. Note that an element can be added to multiple windows, which also means that multiple windows can exist at the same time.</p> - -<p>Each window owns a <code>Trigger</code> that decides when the window is evaluated or purged. The trigger is called for each element that is inserted into the window and when a previously registered timer times out. On each event, a trigger can decide to fire (i.e., evaluate), purge (remove the window and discard its content), or fire and then purge the window. A trigger that just fires evaluates the window and keeps it as it is, i.e., all elements remain in the window and are evaluated again when the triggers fires the next time. A window can be evaluated several times and exists until it is purged. Note that a window consumes memory until it is purged.</p> - -<p>When a Trigger fires, the list of window elements can be given to an optional <code>Evictor</code>. The evictor can iterate through the list and decide to cut off some elements from the start of the list, i.e., remove some of the elements that entered the window first. The remaining elements are given to an evaluation function. If no Evictor was defined, the Trigger hands all the window elements directly to the evaluation function.</p> - -<p>The evaluation function receives the elements of a window (possibly filtered by an Evictor) and computes one or more result elements for the window. The DataStream API accepts different types of evaluation functions, including predefined aggregation functions such as <code>sum()</code>, <code>min()</code>, <code>max()</code>, as well as a <code>ReduceFunction</code>, <code>FoldFunction</code>, or <code>WindowFunction</code>. A WindowFunction is the most generic evaluation function and receives the window object (i.e, the meta data of the window), the list of window elements, and the window key (in case of a keyed window) as parameters.</p> - -<p>These are the components that constitute Flinkâs windowing mechanics. We now show step-by-step how to implement custom windowing logic with the DataStream API. We start with a stream of type <code>DataStream[IN]</code> and key it using a key selector function that extracts a key of type <code>KEY</code> to obtain a <code>KeyedStream[IN, KEY]</code>.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="k">val</span> <span class="n">input</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[</span><span class="kt">IN</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span> - -<span class="c1">// created a keyed stream using a key selector function</span> -<span class="k">val</span> <span class="n">keyed</span><span class="k">:</span> <span class="kt">KeyedStream</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">KEY</span><span class="o">]</span> <span class="k">=</span> <span class="n">input</span> - <span class="o">.</span><span class="n">keyBy</span><span class="o">(</span><span class="n">myKeySel</span><span class="k">:</span> <span class="o">(</span><span class="kt">IN</span><span class="o">)</span> <span class="o">=></span> <span class="nc">KEY</span><span class="o">)</span></code></pre></div> - -<p>We apply a <code>WindowAssigner[IN, WINDOW]</code> that creates windows of type <code>WINDOW</code> resulting in a <code>WindowedStream[IN, KEY, WINDOW]</code>. In addition, a <code>WindowAssigner</code> also provides a default <code>Trigger</code> implementation.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// create windowed stream using a WindowAssigner</span> -<span class="k">var</span> <span class="n">windowed</span><span class="k">:</span> <span class="kt">WindowedStream</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">KEY</span>, <span class="kt">WINDOW</span><span class="o">]</span> <span class="k">=</span> <span class="n">keyed</span> - <span class="o">.</span><span class="n">window</span><span class="o">(</span><span class="n">myAssigner</span><span class="k">:</span> <span class="kt">WindowAssigner</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> - -<p>We can explicitly specify a <code>Trigger</code> to overwrite the default <code>Trigger</code> provided by the <code>WindowAssigner</code>. Note that specifying a triggers does not add an additional trigger condition but replaces the current trigger.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// override the default trigger of the WindowAssigner</span> -<span class="n">windowed</span> <span class="k">=</span> <span class="n">windowed</span> - <span class="o">.</span><span class="n">trigger</span><span class="o">(</span><span class="n">myTrigger</span><span class="k">:</span> <span class="kt">Trigger</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> - -<p>We may want to specify an optional <code>Evictor</code> as follows.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// specify an optional evictor</span> -<span class="n">windowed</span> <span class="k">=</span> <span class="n">windowed</span> - <span class="o">.</span><span class="n">evictor</span><span class="o">(</span><span class="n">myEvictor</span><span class="k">:</span> <span class="kt">Evictor</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> - -<p>Finally, we apply a <code>WindowFunction</code> that returns elements of type <code>OUT</code> to obtain a <code>DataStream[OUT]</code>.</p> - -<div class="highlight"><pre><code class="language-scala"><span class="c1">// apply window function to windowed stream</span> -<span class="k">val</span> <span class="n">output</span><span class="k">:</span> <span class="kt">DataStream</span><span class="o">[</span><span class="kt">OUT</span><span class="o">]</span> <span class="k">=</span> <span class="n">windowed</span> - <span class="o">.</span><span class="n">apply</span><span class="o">(</span><span class="n">myWinFunc</span><span class="k">:</span> <span class="kt">WindowFunction</span><span class="o">[</span><span class="kt">IN</span>, <span class="kt">OUT</span>, <span class="kt">KEY</span>, <span class="kt">WINDOW</span><span class="o">])</span></code></pre></div> - -<p>With Flinkâs internal windowing mechanics and its exposure through the DataStream API it is possible to implement very custom windowing logic such as session windows or windows that emit early results if the values exceed a certain threshold.</p> - -<h2 id="conclusion">Conclusion</h2> - -<p>Support for various types of windows over continuous data streams is a must-have for modern stream processors. Apache Flink is a stream processor with a very strong feature set, including a very flexible mechanism to build and evaluate windows over continuous data streams. Flink provides pre-defined window operators for common uses cases as well as a toolbox that allows to define very custom windowing logic. The Flink community will add more pre-defined window operators as we learn the requirements from our users.</p> - - </article> - </div> - - <div class="row"> - <div id="disqus_thread"></div> - <script type="text/javascript"> - /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */ - var disqus_shortname = 'stratosphere-eu'; // required: replace example with your forum shortname - - /* * * DON'T EDIT BELOW THIS LINE * * */ - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); - </script> - </div> - </div> -</div> - - <hr /> - <div class="footer text-center"> - <p>Copyright © 2014-2016 <a href="http://apache.org";>The Apache Software Foundation</a>. All Rights Reserved.</p> - <p>Apache Flink, Apache, and the Apache feather logo are either registered trademarks or trademarks of The Apache Software Foundation.</p> - <p><a href="/privacy-policy.html">Privacy Policy</a> · <a href="/blog/feed.xml">RSS feed</a></p> - </div> - - </div><!-- /.container --> - - <!-- Include all compiled plugins (below), or include individual files as needed --> - <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js";></script> - <script src="/js/codetabs.js"></script> - - <!-- Google Analytics --> - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', 'UA-52545728-1', 'auto'); - ga('send', 'pageview'); - </script> - </body> -</html>
